예제 #1
0
// 添加请求到队列
func (self *Matrix) Push(req *context.Request) {
	sdl.RLock()
	defer sdl.RUnlock()

	if sdl.status == status.STOP ||
		self.maxPage >= 0 ||
		// 当req不可重复下载时,已存在成功记录则返回
		!req.IsReloadable() && !UpsertSuccess(req) {
		return
	}

	// 大致限制加入队列的请求量,并发情况下应该会比maxPage多
	atomic.AddInt64(&self.maxPage, 1)

	priority := req.GetPriority()

	// 初始化该蜘蛛下该优先级队列
	if _, found := self.reqs[priority]; !found {
		self.priorities = append(self.priorities, priority)
		sort.Ints(self.priorities) // 从小到大排序
		self.reqs[priority] = []*context.Request{}
	}

	// 添加请求到队列
	self.reqs[priority] = append(self.reqs[priority], req)
}