func New(id int) Crawler {
return &crawler{
id: id,
Pipeline: pipeline.New(),
Downloader: downloader.SurferDownloader,
}
}
func New(id int) Crawler {
return &crawler{
id: id,
Pipeline: pipeline.New(),
Downloader: downloader.SurferDownloader,
srcManage: [2]uint{},
}
}
func (self *crawler) Init(sp *spider.Spider) Crawler {
self.Spider = sp.ReqmatrixInit()
self.Pipeline = pipeline.New(sp)
self.pause[0] = cache.Task.Pausetime / 2
if self.pause[0] > 0 {
self.pause[1] = self.pause[0] * 3
} else {
self.pause[1] = 1
}
return self
}