func main() {
file, _ := os.Create("crawl_baidubaike_and_store_it_output.txt")
defer file.Close()
engine.NewEngine("crawl_baidubaike_and_store_it").
AddPipeline(pipeline.NewFilePipeline(file)).
SetProcesser(NewMyProcesser()).
SetStartUrls(getUrlsFromFile("test.url")).
Start()
}
func (this *QuickEngine) GetEngine() *Engine {
e := NewEngine(this.quickEngineConfig.TaskName).
SetProcesser(NewQuickEngineProcesser(this.quickEngineConfig)).
SetStartUrls(this.quickEngineConfig.StartUrls).
SetConfig(this.quickEngineConfig.ToCommonConfig())
if this.quickEngineConfig.LogFile != "" {
var err error
this.logFile, err = os.OpenFile(this.quickEngineConfig.LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
log.SetOutput(this.logFile)
if err != nil {
log.Fatal("cannot create log file: %s\n", this.quickEngineConfig.LogFile)
}
}
if this.file != nil {
return e.AddPipeline(pipeline.NewFilePipeline(this.file))
} else if this.quickEngineConfig.OutputFile != "" {
this.file, _ = os.Create(this.quickEngineConfig.OutputFile)
return e.AddPipeline(pipeline.NewFilePipeline(this.file))
}
return e.AddPipeline(pipeline.NewConsolePipeline())
}