func NewQuickEngineConfig(fileName string) *QuickEngineConfig { c := &QuickEngineConfig{} t := reflect.TypeOf(&c.Config) v := reflect.ValueOf(&c.Config) config := common.NewConfig() for i := 0; i < t.Elem().NumField(); i++ { field := t.Elem().Field(i) value := v.Elem().FieldByName(field.Name) funcName := "Get" + field.Name switch value.Type().Kind() { case reflect.Int: defaultValue := reflect.ValueOf(config).MethodByName(funcName).Call([]reflect.Value{}) value.SetInt(defaultValue[0].Int()) case reflect.String: defaultValue := reflect.ValueOf(config).MethodByName(funcName).Call([]reflect.Value{}) value.SetString(fmt.Sprintf("%s", defaultValue[0].Interface())) case reflect.Bool: defaultValue := reflect.ValueOf(config).MethodByName(funcName).Call([]reflect.Value{}) value.SetBool(defaultValue[0].Bool()) } } file, _ := ioutil.ReadFile(fileName) json.Unmarshal(file, c) return c }
func main() { engine.NewEngine("inject_useragent"). SetStartUrl("http://my-user-agent.com/"). SetProcesser(NewMyProcesser()). AddPlugin(plugin.NewUserAgentPlugin()). AddPipeline(pipeline.NewConsolePipeline()). SetConfig(common.NewConfig().SetHeaders(map[string]string{"User-Agent": "golang spider"})). Start() }
func NewEngine(taskName string) *Engine { e := &Engine{taskName: taskName} e.config = common.NewConfig() e.resourceManager = common.NewResourceManager(e.config.GetConcurrency()) e.retryCache = make(map[[md5.Size]byte]int) e.scheduler = scheduler.NewScheduler() e.downloader = downloader.NewHttpDownloader() e.processer = processer.NewLazyProcesser() return e }
func main() { config := common.NewConfig(). SetConcurrency(1000). SetWaitTime(10 * time.Millisecond). SetPollingTime(10 * time.Millisecond) engine.NewEngine("crawl_rate"). SetStartUrls(genUrls()). SetProcesser(NewMyProcesser()). SetConfig(config). Start() }
func (this *QuickEngineConfig) ToCommonConfig() *common.Config { e := common.NewConfig().SetConcurrency(this.Config.Concurrency). SetPollingTime(this.stringToDuration(this.Config.PollingTime)). SetWaitTime(this.stringToDuration(this.Config.WaitTime)). SetDownloadTimeout(this.stringToDuration(this.Config.DownloadTimeout)). SetConnectionTimeout(this.stringToDuration(this.Config.ConnectionTimeout)). SetMaxIdleConnsPerHost(this.Config.MaxIdleConnsPerHost). SetMaxRetryTimes(this.Config.MaxRetryTimes). SetMaxDepth(this.Config.MaxDepth). SetLogging(this.Config.Logging). SetHeaders(this.Config.Headers). SetSucc(this.Config.Succ) return e }