Exemplo n.º 1
0
func NewQuickEngineConfig(fileName string) *QuickEngineConfig {
	c := &QuickEngineConfig{}
	t := reflect.TypeOf(&c.Config)
	v := reflect.ValueOf(&c.Config)
	config := common.NewConfig()

	for i := 0; i < t.Elem().NumField(); i++ {
		field := t.Elem().Field(i)
		value := v.Elem().FieldByName(field.Name)
		funcName := "Get" + field.Name
		switch value.Type().Kind() {
		case reflect.Int:
			defaultValue := reflect.ValueOf(config).MethodByName(funcName).Call([]reflect.Value{})
			value.SetInt(defaultValue[0].Int())
		case reflect.String:
			defaultValue := reflect.ValueOf(config).MethodByName(funcName).Call([]reflect.Value{})
			value.SetString(fmt.Sprintf("%s", defaultValue[0].Interface()))
		case reflect.Bool:
			defaultValue := reflect.ValueOf(config).MethodByName(funcName).Call([]reflect.Value{})
			value.SetBool(defaultValue[0].Bool())
		}
	}

	file, _ := ioutil.ReadFile(fileName)
	json.Unmarshal(file, c)
	return c
}
func main() {
	engine.NewEngine("inject_useragent").
		SetStartUrl("http://my-user-agent.com/").
		SetProcesser(NewMyProcesser()).
		AddPlugin(plugin.NewUserAgentPlugin()).
		AddPipeline(pipeline.NewConsolePipeline()).
		SetConfig(common.NewConfig().SetHeaders(map[string]string{"User-Agent": "golang spider"})).
		Start()
}
Exemplo n.º 3
0
func NewEngine(taskName string) *Engine {
	e := &Engine{taskName: taskName}
	e.config = common.NewConfig()

	e.resourceManager = common.NewResourceManager(e.config.GetConcurrency())
	e.retryCache = make(map[[md5.Size]byte]int)

	e.scheduler = scheduler.NewScheduler()
	e.downloader = downloader.NewHttpDownloader()
	e.processer = processer.NewLazyProcesser()
	return e
}
Exemplo n.º 4
0
func main() {
	config := common.NewConfig().
		SetConcurrency(1000).
		SetWaitTime(10 * time.Millisecond).
		SetPollingTime(10 * time.Millisecond)

	engine.NewEngine("crawl_rate").
		SetStartUrls(genUrls()).
		SetProcesser(NewMyProcesser()).
		SetConfig(config).
		Start()
}
Exemplo n.º 5
0
func (this *QuickEngineConfig) ToCommonConfig() *common.Config {
	e := common.NewConfig().SetConcurrency(this.Config.Concurrency).
		SetPollingTime(this.stringToDuration(this.Config.PollingTime)).
		SetWaitTime(this.stringToDuration(this.Config.WaitTime)).
		SetDownloadTimeout(this.stringToDuration(this.Config.DownloadTimeout)).
		SetConnectionTimeout(this.stringToDuration(this.Config.ConnectionTimeout)).
		SetMaxIdleConnsPerHost(this.Config.MaxIdleConnsPerHost).
		SetMaxRetryTimes(this.Config.MaxRetryTimes).
		SetMaxDepth(this.Config.MaxDepth).
		SetLogging(this.Config.Logging).
		SetHeaders(this.Config.Headers).
		SetSucc(this.Config.Succ)
	return e
}