func Parsing(ctx *ParsingContext, task tasks.ParsingTask) (*ParsingResult, error) { logger.Infof("%s start parsing", task.Id) var ( blob []byte err error payload interface{} wg sync.WaitGroup ) blob, err = fetchDataFromTarget(&task) if err != nil { logger.Errf("%s error `%v` occured while fetching data", task.Id, err) return nil, err } if !task.ParsingConfig.SkipParsingStage() { logger.Infof("%s Send data to parsing", task.Id) parsingApp, err := ctx.Resolver.Resolve(ctx.Ctx, common.PARSINGAPP) if err != nil { logger.Errf("%s error `%v` occured while resolving %s", task.Id, err, common.PARSINGAPP) return nil, err } taskToParser, _ := common.Pack([]interface{}{task.Id, task.ParsingConfig.Parser, blob}) if err := parsingApp.Do("enqueue", "parse", taskToParser).Wait(ctx.Ctx, &blob); err != nil { logger.Errf("%s error `%v` occured while parsing data", task.Id, err) return nil, err } } payload = blob if !task.ParsingConfig.Raw { logger.Debugf("%s Use %s for handle data", task.Id, common.DATABASEAPP) datagrid, err := ctx.Resolver.Resolve(ctx.Ctx, common.DATABASEAPP) if err != nil { logger.Errf("%s unable to get DG %v", task.Id, err) return nil, err } var token string if err := datagrid.Do("enqueue", "put", blob).Wait(ctx.Ctx, &token); err != nil { logger.Errf("%s unable to put data to DG %v", task.Id, err) return nil, err } defer func() { taskToDatagrid, _ := common.Pack([]interface{}{token}) datagrid.Do("enqueue", "drop", taskToDatagrid) logger.Debugf("%s Drop table", task.Id) }() payload = token } pr := NewParsingResult() for aggLogName, aggCfg := range task.AggregationConfigs { for k, v := range aggCfg.Data { aggType, err := v.Type() if err != nil { logger.Errf("no type in configuration: %s %s %v", aggLogName, k, v) return nil, err } logger.Debugf("%s Send to %s %s type %s %v", task.Id, aggLogName, k, aggType, v) wg.Add(1) go func(name string, dataName string, v interface{}, configName string) { defer wg.Done() app, err := ctx.Resolver.Resolve(ctx.Ctx, name) if err != nil { logger.Errf("%s %s %s", task.Id, name, err) return } // Task structure t, _ := common.Pack(map[string]interface{}{ "config": v, "token": payload, "prevtime": task.PrevTime, "currtime": task.CurrTime, "id": task.Id, }) var rawRes []byte if err := app.Do("enqueue", "aggregate_host", t).Wait(ctx.Ctx, &rawRes); err != nil { logger.Errf("%s Failed task: %v", task.Id, err) return } logger.Debugf("result for %s %s: %v", configName, dataName, rawRes) pr.Put(configName, dataName, rawRes) }(aggType, k, v, aggLogName) } } wg.Wait() logger.Infof("%s Done", task.Id) return pr, nil }
func Parsing(task Task) (err error) { log := cocaine.NewLogger() log.Info("Start ", task) defer log.Close() //Wrap it log.Debug("Create configuration manager") cfgManager := cocaine.NewService(common.CFGMANAGER) defer cfgManager.Close() log.Debug("Fetch configuration file") res := <-cfgManager.Call("enqueue", "parsing", task.Config) if err = res.Err(); err != nil { return } var rawCfg []byte if err = res.Extract(&rawCfg); err != nil { return } var cfg common.ParsingConfig common.Encode(rawCfg, &cfg) res = <-cfgManager.Call("enqueue", "common", "") if err = res.Err(); err != nil { return } if err = res.Extract(&rawCfg); err != nil { return } var combainerCfg common.CombainerConfig common.Encode(rawCfg, &combainerCfg) common.MapUpdate(&(combainerCfg.CloudCfg.DF), &(cfg.DF)) cfg.DF = combainerCfg.CloudCfg.DF common.MapUpdate(&(combainerCfg.CloudCfg.DS), &(cfg.DS)) cfg.DS = combainerCfg.CloudCfg.DS common.MapUpdate(&(combainerCfg.CloudCfg.DG), &(cfg.DG)) cfg.DG = combainerCfg.CloudCfg.DG fetcherType, err := common.GetType(cfg.DF) if err != nil { return } log.Info(fmt.Sprintf("Use %s for fetching data", fetcherType)) fetcher := cocaine.NewService(fetcherType) defer fetcher.Close() fetcherTask := common.FetcherTask{ Target: task.Host, StartTime: task.PrevTime, EndTime: task.CurrTime, } ft := struct { Config map[string]interface{} "Config" Task common.FetcherTask "Task" }{cfg.DF, fetcherTask} js, _ := common.Pack(ft) res = <-fetcher.Call("enqueue", "get", js) if err = res.Err(); err != nil { return } var t []byte if err = res.Extract(&t); err != nil { return } // ParsingApp stage parserApp := cocaine.NewService(common.PARSINGAPP) defer parserApp.Close() taskToParser, err := common.Pack([]interface{}{cfg.Parser, t}) if err != nil { return } res = <-parserApp.Call("enqueue", "parse", taskToParser) if err = res.Err(); err != nil { return } var z interface{} if err = res.Extract(&z); err != nil { return } // Datagrid stage dgType, err := common.GetType(cfg.DG) if err != nil { return } log.Info(fmt.Sprintf("Use %s for handle data", dgType)) datagrid := cocaine.NewService(dgType) defer datagrid.Close() taskToDatagrid, err := common.Pack([]interface{}{cfg.DG, z}) if err != nil { return } res = <-datagrid.Call("enqueue", "put", taskToDatagrid) if err = res.Err(); err != nil { return } var token string if err = res.Extract(&token); err != nil { return } log.Info(token) defer func() { taskToDatagrid, _ = common.Pack([]interface{}{cfg.DG, token}) <-datagrid.Call("enqueue", "drop", taskToDatagrid) }() log.Info("Stop ", task) return nil }