// 添加请求到队列
func (self *scheduler) Push(req *context.Request) {
// 初始化该蜘蛛的队列
spiderId, ok := req.GetSpiderId()
if !ok {
return
}
self.RLock()
defer self.RUnlock()
if self.status == status.STOP {
return
}
// 当req不可重复时，有重复则返回
if !req.GetDuplicatable() && self.Deduplicate(req.GetUrl()+req.GetMethod()) {
return
}
// 初始化该蜘蛛下该优先级队列
priority := req.GetPriority()
if !self.foundPriority(spiderId, priority) {
self.addPriority(spiderId, priority)
}
defer func() {
recover()
}()
// 添加请求到队列
self.queue[spiderId][priority] = append(self.queue[spiderId][priority], req)
}
// 添加请求到队列
func (self *scheduler) Push(req *context.Request) {
self.pushMutex.Lock()
defer self.pushMutex.Unlock()
if self.status == status.STOP {
return
}
// 当req不可重复时，有重复则返回
if !req.GetDuplicatable() && self.Deduplicate(req.GetUrl()+req.GetMethod()) {
return
}
self.SrcManage.Push(req)
}