Golang Task.DeduplicationTarget Examples

Example #1

File: app.go Project: smypai/pholcus

// 服务器模式下，生成task并添加至库
func (self *Logic) addNewTask() (tasksNum, spidersNum int) {
	length := self.SpiderQueue.Len()
	t := distribute.Task{}

	// 从配置读取字段
	t.ThreadNum = self.AppConf.ThreadNum
	t.Pausetime = self.AppConf.Pausetime
	t.OutType = self.AppConf.OutType
	t.DockerCap = self.AppConf.DockerCap
	t.DockerQueueCap = self.AppConf.DockerQueueCap
	t.InheritDeduplication = self.AppConf.InheritDeduplication
	t.DeduplicationTarget = self.AppConf.DeduplicationTarget
	t.MaxPage = self.AppConf.MaxPage
	t.Keywords = self.AppConf.Keywords

	for i, sp := range self.SpiderQueue.GetAll() {

		t.Spiders = append(t.Spiders, map[string]string{"name": sp.GetName(), "keyword": sp.GetKeyword()})
		spidersNum++

		// 每十个蜘蛛存为一个任务
		if i > 0 && i%10 == 0 && length > 10 {
			// 存入
			one := t
			self.TaskJar.Push(&one)
			// logs.Log.Notice(" *     [新增任务]   详情： %#v", *t)

			tasksNum++

			// 清空spider
			t.Spiders = []map[string]string{}
		}
	}

	if len(t.Spiders) != 0 {
		// 存入
		one := t
		self.TaskJar.Push(&one)
		tasksNum++
	}
	return
}