func (s *Spider) RegisterDownload(downloaders []downloader.Downloader) *Spider {
s.rawChan = make(chan *raw.Raw, len(downloaders))
for _, _downloader := range downloaders {
_downloader.SetCallBack(s.rawChan)
}
s.downloadPool = downloader.NewPool(downloaders)
return s
}
// register register all components which wasn't registered.
func (s *Spider) register() {
if s.pagePool == nil {
var pageProcessors []page.PageProcessor
for i := uint(0); i < s.threadNum; i++ {
pageProcessor := page.NewPageProcessor()
pageProcessors = append(pageProcessors, pageProcessor)
}
s.pagePool = page.NewPool(pageProcessors)
}
if s.downloadPool == nil {
var downloaders []downloader.Downloader
for i := uint(0); i < s.threadNum; i++ {
_downloader := downloader.New(fmt.Sprintf("down %d", i))
_downloader.SetCallBack(s.rawChan)
downloaders = append(downloaders, _downloader)
}
s.downloadPool = downloader.NewPool(downloaders)
}
s.download()
s.page()
s.analyse()
if s._scheduler == nil {
s._scheduler = scheduler.New()
}
s._scheduler.SetMaxDepth(s.depth)
for _, req := range s.reqs {
s._scheduler.Add(req)
}
if s.OnWatch {
s.Watch()
}
time.Sleep(1 * time.Second)
}