Esempio n. 1
0
func New(id int) Crawler {
	return &crawler{
		id:         id,
		Pipeline:   pipeline.New(),
		Downloader: downloader.NewSurfer(0),
		srcManage:  [2]uint{},
	}
}
Esempio n. 2
0
func (self *crawler) Init(sp *spider.Spider) Crawler {
	self.Pipeline.Init(sp)
	self.Spider = sp
	self.Downloader = downloader.NewSurfer(
		time.Duration((self.Spider.Pausetime[1]+self.Spider.Pausetime[0])/2)*time.Millisecond,
		self.Spider.Proxy,
	)
	self.srcManage = [2]uint{}
	return self
}