// 添加请求到队列 func (self *scheduler) Push(req *context.Request) { // 初始化该蜘蛛的队列 spiderId, ok := req.GetSpiderId() if !ok { return } self.RLock() defer self.RUnlock() if self.status == status.STOP { return } // 当req不可重复时,有重复则返回 if !req.GetDuplicatable() && self.Deduplicate(req.GetUrl()+req.GetMethod()) { return } // 初始化该蜘蛛下该优先级队列 priority := req.GetPriority() if !self.foundPriority(spiderId, priority) { self.addPriority(spiderId, priority) } defer func() { recover() }() // 添加请求到队列 self.queue[spiderId][priority] = append(self.queue[spiderId][priority], req) }
func (self *SrcManage) Push(req *context.Request) { spiderId, ok := req.GetSpiderId() if !ok { return } // 初始化该蜘蛛的队列 if _, ok := self.queue[spiderId]; !ok { self.mutex[spiderId] = new(sync.Mutex) self.queue[spiderId] = make(map[int][]*context.Request) } priority := req.GetPriority() // 登记该蜘蛛下该优先级队列 if _, ok := self.queue[spiderId][priority]; !ok { self.uIndex(spiderId, priority) } // 添加请求到队列 self.queue[spiderId][priority] = append(self.queue[spiderId][priority], req) }