Example #1
0
func ReadParsingConfig(s ServerContext, w http.ResponseWriter, r *http.Request) {
	name := mux.Vars(r)["name"]
	repo := s.GetRepository()
	combainerCfg := repo.GetCombainerConfig()
	var parsingCfg configs.ParsingConfig
	cfg, err := repo.GetParsingConfig(name)
	if err != nil {
		fmt.Fprintf(w, "%s", err)
		return
	}

	err = cfg.Decode(&parsingCfg)
	if err != nil {
		fmt.Fprintf(w, "%s", err)
		return
	}

	parsingCfg.UpdateByCombainerConfig(&combainerCfg)
	aggregationConfigs, err := GetAggregationConfigs(repo, &parsingCfg)
	if err != nil {
		log.Errorf("Unable to read aggregation configs: %s", err)
		return
	}

	data, err := common.Encode(&parsingCfg)
	if err != nil {
		fmt.Fprintf(w, "%s", err)
		return
	}

	fmt.Fprintf(w, "============ %s ============\n", name)
	fmt.Fprintf(w, "%s", data)
	for aggname, v := range *aggregationConfigs {
		fmt.Fprintf(w, "============ %s ============\n", aggname)
		d, err := common.Encode(&v)
		if err != nil {
			fmt.Fprintf(w, "%s", err)
			return
		}
		fmt.Fprintf(w, "%s\n", d)
	}
}
Example #2
0
func Parsing(task Task) (err error) {
	log := cocaine.NewLogger()
	log.Info("Start ", task)
	defer log.Close()

	//Wrap it
	log.Debug("Create configuration manager")
	cfgManager := cocaine.NewService(common.CFGMANAGER)
	defer cfgManager.Close()

	log.Debug("Fetch configuration file")
	res := <-cfgManager.Call("enqueue", "parsing", task.Config)
	if err = res.Err(); err != nil {
		return
	}
	var rawCfg []byte
	if err = res.Extract(&rawCfg); err != nil {
		return
	}
	var cfg common.ParsingConfig
	common.Encode(rawCfg, &cfg)

	res = <-cfgManager.Call("enqueue", "common", "")
	if err = res.Err(); err != nil {
		return
	}
	if err = res.Extract(&rawCfg); err != nil {
		return
	}
	var combainerCfg common.CombainerConfig
	common.Encode(rawCfg, &combainerCfg)

	common.MapUpdate(&(combainerCfg.CloudCfg.DF), &(cfg.DF))
	cfg.DF = combainerCfg.CloudCfg.DF
	common.MapUpdate(&(combainerCfg.CloudCfg.DS), &(cfg.DS))
	cfg.DS = combainerCfg.CloudCfg.DS
	common.MapUpdate(&(combainerCfg.CloudCfg.DG), &(cfg.DG))
	cfg.DG = combainerCfg.CloudCfg.DG

	fetcherType, err := common.GetType(cfg.DF)
	if err != nil {
		return
	}

	log.Info(fmt.Sprintf("Use %s for fetching data", fetcherType))
	fetcher := cocaine.NewService(fetcherType)
	defer fetcher.Close()

	fetcherTask := common.FetcherTask{
		Target:    task.Host,
		StartTime: task.PrevTime,
		EndTime:   task.CurrTime,
	}

	ft := struct {
		Config map[string]interface{} "Config"
		Task   common.FetcherTask     "Task"
	}{cfg.DF, fetcherTask}

	js, _ := common.Pack(ft)

	res = <-fetcher.Call("enqueue", "get", js)
	if err = res.Err(); err != nil {
		return
	}

	var t []byte
	if err = res.Extract(&t); err != nil {
		return
	}

	// ParsingApp stage
	parserApp := cocaine.NewService(common.PARSINGAPP)
	defer parserApp.Close()
	taskToParser, err := common.Pack([]interface{}{cfg.Parser, t})
	if err != nil {
		return
	}
	res = <-parserApp.Call("enqueue", "parse", taskToParser)
	if err = res.Err(); err != nil {
		return
	}
	var z interface{}
	if err = res.Extract(&z); err != nil {
		return
	}

	// Datagrid stage
	dgType, err := common.GetType(cfg.DG)
	if err != nil {
		return
	}

	log.Info(fmt.Sprintf("Use %s for handle data", dgType))
	datagrid := cocaine.NewService(dgType)
	defer datagrid.Close()
	taskToDatagrid, err := common.Pack([]interface{}{cfg.DG, z})
	if err != nil {
		return
	}
	res = <-datagrid.Call("enqueue", "put", taskToDatagrid)
	if err = res.Err(); err != nil {
		return
	}
	var token string
	if err = res.Extract(&token); err != nil {
		return
	}
	log.Info(token)
	defer func() {
		taskToDatagrid, _ = common.Pack([]interface{}{cfg.DG, token})
		<-datagrid.Call("enqueue", "drop", taskToDatagrid)
	}()

	log.Info("Stop ", task)
	return nil
}