func (ctrl *Controller) Go() {
basic.Config.StartUrl = ctrl.StartUrl
basic.InitConfig()
ctrl.Downloader = downloader.NewDownloader() //初始化各组件，下同
ctrl.Analyzer = analyzer.NewAnalyzer()
ctrl.Processor = processor.NewProcessor()
ctrl.Channel = middleware.NewChannel()
ctrl.WorkPool = middleware.NewWorkPool()
ctrl.StopSignal = NewStopSignal()
//准备第一次请求
primaryreq, err := http.NewRequest(basic.Config.RequestMethod, basic.Config.StartUrl, nil)
basic.Check(err)
basereq := basic.NewRequest(primaryreq, 0)
ctrl.Channel.ReqChan() <- *basereq
//利用goroutine使三个组件同时工作，启动监视器监视
wg.Add(4)
go ctrl.DownloaderManager()
go ctrl.AnalyzerManager()
go ctrl.ProcessorManager()
go ctrl.Monitors()
wg.Wait()
//fmt.Println(len(ctrl.Processor.GetVurl()))
//fmt.Println(ctrl.Processor.GetVurl())
}
func (self *Processor) DealLink(link basic.Link) (*basic.Request, bool) {
//url判重
if _, visited := self.Vurl[link.GetLink()]; visited {
return nil, false
}
self.Vurl[link.GetLink()] = true //放入字典
httpReq, err := http.NewRequest(basic.Config.RequestMethod, link.GetLink(), nil)
basic.Check(err)
request := basic.NewRequest(httpReq, link.GetIndex()) //转化为构造请求
return request, true
}
