func Parsing(ctx *ParsingContext, task tasks.ParsingTask) (*ParsingResult, error) {
logger.Infof("%s start parsing", task.Id)
var (
blob []byte
err error
payload interface{}
wg sync.WaitGroup
)
blob, err = fetchDataFromTarget(&task)
if err != nil {
logger.Errf("%s error `%v` occured while fetching data", task.Id, err)
return nil, err
}
if !task.ParsingConfig.SkipParsingStage() {
logger.Infof("%s Send data to parsing", task.Id)
parsingApp, err := ctx.Resolver.Resolve(ctx.Ctx, common.PARSINGAPP)
if err != nil {
logger.Errf("%s error `%v` occured while resolving %s",
task.Id, err, common.PARSINGAPP)
return nil, err
}
taskToParser, _ := common.Pack([]interface{}{task.Id, task.ParsingConfig.Parser, blob})
if err := parsingApp.Do("enqueue", "parse", taskToParser).Wait(ctx.Ctx, &blob); err != nil {
logger.Errf("%s error `%v` occured while parsing data", task.Id, err)
return nil, err
}
}
payload = blob
if !task.ParsingConfig.Raw {
logger.Debugf("%s Use %s for handle data", task.Id, common.DATABASEAPP)
datagrid, err := ctx.Resolver.Resolve(ctx.Ctx, common.DATABASEAPP)
if err != nil {
logger.Errf("%s unable to get DG %v", task.Id, err)
return nil, err
}
var token string
if err := datagrid.Do("enqueue", "put", blob).Wait(ctx.Ctx, &token); err != nil {
logger.Errf("%s unable to put data to DG %v", task.Id, err)
return nil, err
}
defer func() {
taskToDatagrid, _ := common.Pack([]interface{}{token})
datagrid.Do("enqueue", "drop", taskToDatagrid)
logger.Debugf("%s Drop table", task.Id)
}()
payload = token
}
pr := NewParsingResult()
for aggLogName, aggCfg := range task.AggregationConfigs {
for k, v := range aggCfg.Data {
aggType, err := v.Type()
if err != nil {
logger.Errf("no type in configuration: %s %s %v", aggLogName, k, v)
return nil, err
}
logger.Debugf("%s Send to %s %s type %s %v", task.Id, aggLogName, k, aggType, v)
wg.Add(1)
go func(name string, dataName string, v interface{}, configName string) {
defer wg.Done()
app, err := ctx.Resolver.Resolve(ctx.Ctx, name)
if err != nil {
logger.Errf("%s %s %s", task.Id, name, err)
return
}
// Task structure
t, _ := common.Pack(map[string]interface{}{
"config": v,
"token": payload,
"prevtime": task.PrevTime,
"currtime": task.CurrTime,
"id": task.Id,
})
var rawRes []byte
if err := app.Do("enqueue", "aggregate_host", t).Wait(ctx.Ctx, &rawRes); err != nil {
logger.Errf("%s Failed task: %v", task.Id, err)
return
}
logger.Debugf("result for %s %s: %v", configName, dataName, rawRes)
pr.Put(configName, dataName, rawRes)
}(aggType, k, v, aggLogName)
}
}
wg.Wait()
logger.Infof("%s Done", task.Id)
return pr, nil
}
func Parsing(task Task) (err error) {
log := cocaine.NewLogger()
log.Info("Start ", task)
defer log.Close()
//Wrap it
log.Debug("Create configuration manager")
cfgManager := cocaine.NewService(common.CFGMANAGER)
defer cfgManager.Close()
log.Debug("Fetch configuration file")
res := <-cfgManager.Call("enqueue", "parsing", task.Config)
if err = res.Err(); err != nil {
return
}
var rawCfg []byte
if err = res.Extract(&rawCfg); err != nil {
return
}
var cfg common.ParsingConfig
common.Encode(rawCfg, &cfg)
res = <-cfgManager.Call("enqueue", "common", "")
if err = res.Err(); err != nil {
return
}
if err = res.Extract(&rawCfg); err != nil {
return
}
var combainerCfg common.CombainerConfig
common.Encode(rawCfg, &combainerCfg)
common.MapUpdate(&(combainerCfg.CloudCfg.DF), &(cfg.DF))
cfg.DF = combainerCfg.CloudCfg.DF
common.MapUpdate(&(combainerCfg.CloudCfg.DS), &(cfg.DS))
cfg.DS = combainerCfg.CloudCfg.DS
common.MapUpdate(&(combainerCfg.CloudCfg.DG), &(cfg.DG))
cfg.DG = combainerCfg.CloudCfg.DG
fetcherType, err := common.GetType(cfg.DF)
if err != nil {
return
}
log.Info(fmt.Sprintf("Use %s for fetching data", fetcherType))
fetcher := cocaine.NewService(fetcherType)
defer fetcher.Close()
fetcherTask := common.FetcherTask{
Target: task.Host,
StartTime: task.PrevTime,
EndTime: task.CurrTime,
}
ft := struct {
Config map[string]interface{} "Config"
Task common.FetcherTask "Task"
}{cfg.DF, fetcherTask}
js, _ := common.Pack(ft)
res = <-fetcher.Call("enqueue", "get", js)
if err = res.Err(); err != nil {
return
}
var t []byte
if err = res.Extract(&t); err != nil {
return
}
// ParsingApp stage
parserApp := cocaine.NewService(common.PARSINGAPP)
defer parserApp.Close()
taskToParser, err := common.Pack([]interface{}{cfg.Parser, t})
if err != nil {
return
}
res = <-parserApp.Call("enqueue", "parse", taskToParser)
if err = res.Err(); err != nil {
return
}
var z interface{}
if err = res.Extract(&z); err != nil {
return
}
// Datagrid stage
dgType, err := common.GetType(cfg.DG)
if err != nil {
return
}
log.Info(fmt.Sprintf("Use %s for handle data", dgType))
datagrid := cocaine.NewService(dgType)
defer datagrid.Close()
taskToDatagrid, err := common.Pack([]interface{}{cfg.DG, z})
if err != nil {
return
}
res = <-datagrid.Call("enqueue", "put", taskToDatagrid)
if err = res.Err(); err != nil {
return
}
var token string
if err = res.Extract(&token); err != nil {
return
}
log.Info(token)
defer func() {
taskToDatagrid, _ = common.Pack([]interface{}{cfg.DG, token})
<-datagrid.Call("enqueue", "drop", taskToDatagrid)
}()
log.Info("Stop ", task)
return nil
}