func New(id int) Crawler {
return &crawler{
id: id,
Pipeline: pipeline.New(),
Downloader: downloader.NewSurfer(0),
srcManage: [2]uint{},
}
}
func (self *crawler) Init(sp *spider.Spider) Crawler {
self.Pipeline.Init(sp)
self.Spider = sp
self.Downloader = downloader.NewSurfer(
time.Duration((self.Spider.Pausetime[1]+self.Spider.Pausetime[0])/2)*time.Millisecond,
self.Spider.Proxy,
)
self.srcManage = [2]uint{}
return self
}