func main() {
	file, _ := os.Create("crawl_baidubaike_and_store_it_output.txt")
	defer file.Close()

	engine.NewEngine("crawl_baidubaike_and_store_it").
		AddPipeline(pipeline.NewFilePipeline(file)).
		SetProcesser(NewMyProcesser()).
		SetStartUrls(getUrlsFromFile("test.url")).
		Start()
}
Пример #2
0
func (this *QuickEngine) GetEngine() *Engine {
	e := NewEngine(this.quickEngineConfig.TaskName).
		SetProcesser(NewQuickEngineProcesser(this.quickEngineConfig)).
		SetStartUrls(this.quickEngineConfig.StartUrls).
		SetConfig(this.quickEngineConfig.ToCommonConfig())

	if this.quickEngineConfig.LogFile != "" {
		var err error
		this.logFile, err = os.OpenFile(this.quickEngineConfig.LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		log.SetOutput(this.logFile)
		if err != nil {
			log.Fatal("cannot create log file: %s\n", this.quickEngineConfig.LogFile)
		}
	}

	if this.file != nil {
		return e.AddPipeline(pipeline.NewFilePipeline(this.file))
	} else if this.quickEngineConfig.OutputFile != "" {
		this.file, _ = os.Create(this.quickEngineConfig.OutputFile)
		return e.AddPipeline(pipeline.NewFilePipeline(this.file))
	}
	return e.AddPipeline(pipeline.NewConsolePipeline())

}