func NewQuickEngineConfig(fileName string) *QuickEngineConfig {
c := &QuickEngineConfig{}
t := reflect.TypeOf(&c.Config)
v := reflect.ValueOf(&c.Config)
config := common.NewConfig()
for i := 0; i < t.Elem().NumField(); i++ {
field := t.Elem().Field(i)
value := v.Elem().FieldByName(field.Name)
funcName := "Get" + field.Name
switch value.Type().Kind() {
case reflect.Int:
defaultValue := reflect.ValueOf(config).MethodByName(funcName).Call([]reflect.Value{})
value.SetInt(defaultValue[0].Int())
case reflect.String:
defaultValue := reflect.ValueOf(config).MethodByName(funcName).Call([]reflect.Value{})
value.SetString(fmt.Sprintf("%s", defaultValue[0].Interface()))
case reflect.Bool:
defaultValue := reflect.ValueOf(config).MethodByName(funcName).Call([]reflect.Value{})
value.SetBool(defaultValue[0].Bool())
}
}
file, _ := ioutil.ReadFile(fileName)
json.Unmarshal(file, c)
return c
}
func main() {
engine.NewEngine("inject_useragent").
SetStartUrl("http://my-user-agent.com/").
SetProcesser(NewMyProcesser()).
AddPlugin(plugin.NewUserAgentPlugin()).
AddPipeline(pipeline.NewConsolePipeline()).
SetConfig(common.NewConfig().SetHeaders(map[string]string{"User-Agent": "golang spider"})).
Start()
}
func NewEngine(taskName string) *Engine {
e := &Engine{taskName: taskName}
e.config = common.NewConfig()
e.resourceManager = common.NewResourceManager(e.config.GetConcurrency())
e.retryCache = make(map[[md5.Size]byte]int)
e.scheduler = scheduler.NewScheduler()
e.downloader = downloader.NewHttpDownloader()
e.processer = processer.NewLazyProcesser()
return e
}
func main() {
config := common.NewConfig().
SetConcurrency(1000).
SetWaitTime(10 * time.Millisecond).
SetPollingTime(10 * time.Millisecond)
engine.NewEngine("crawl_rate").
SetStartUrls(genUrls()).
SetProcesser(NewMyProcesser()).
SetConfig(config).
Start()
}
func (this *QuickEngineConfig) ToCommonConfig() *common.Config {
e := common.NewConfig().SetConcurrency(this.Config.Concurrency).
SetPollingTime(this.stringToDuration(this.Config.PollingTime)).
SetWaitTime(this.stringToDuration(this.Config.WaitTime)).
SetDownloadTimeout(this.stringToDuration(this.Config.DownloadTimeout)).
SetConnectionTimeout(this.stringToDuration(this.Config.ConnectionTimeout)).
SetMaxIdleConnsPerHost(this.Config.MaxIdleConnsPerHost).
SetMaxRetryTimes(this.Config.MaxRetryTimes).
SetMaxDepth(this.Config.MaxDepth).
SetLogging(this.Config.Logging).
SetHeaders(this.Config.Headers).
SetSucc(this.Config.Succ)
return e
}