Beispiel #1
0
func (ctrl *Controller) Go() {
	basic.Config.StartUrl = ctrl.StartUrl
	basic.InitConfig()
	ctrl.Downloader = downloader.NewDownloader() //初始化各组件,下同
	ctrl.Analyzer = analyzer.NewAnalyzer()
	ctrl.Processor = processor.NewProcessor()
	ctrl.Channel = middleware.NewChannel()
	ctrl.WorkPool = middleware.NewWorkPool()
	ctrl.StopSignal = NewStopSignal()

	//准备第一次请求
	primaryreq, err := http.NewRequest(basic.Config.RequestMethod, basic.Config.StartUrl, nil)
	basic.Check(err)
	basereq := basic.NewRequest(primaryreq, 0)
	ctrl.Channel.ReqChan() <- *basereq

	//利用goroutine使三个组件同时工作,启动监视器监视
	wg.Add(4)
	go ctrl.DownloaderManager()
	go ctrl.AnalyzerManager()
	go ctrl.ProcessorManager()
	go ctrl.Monitors()

	wg.Wait()
	//fmt.Println(len(ctrl.Processor.GetVurl()))
	//fmt.Println(ctrl.Processor.GetVurl())
}
Beispiel #2
0
func (self *Processor) DealLink(link basic.Link) (*basic.Request, bool) {
	//url判重
	if _, visited := self.Vurl[link.GetLink()]; visited {
		return nil, false
	}
	self.Vurl[link.GetLink()] = true //放入字典
	httpReq, err := http.NewRequest(basic.Config.RequestMethod, link.GetLink(), nil)
	basic.Check(err)
	request := basic.NewRequest(httpReq, link.GetIndex()) //转化为构造请求
	return request, true
}