func fetchDataFromTarget(task *tasks.ParsingTask) ([]byte, error) { fetcherType, err := task.ParsingConfig.DataFetcher.Type() if err != nil { return nil, err } logger.Debugf("%s use %s for fetching data", task.Id, fetcherType) fetcher, err := NewFetcher(fetcherType, task.ParsingConfig.DataFetcher) if err != nil { return nil, err } fetcherTask := tasks.FetcherTask{ Target: task.Host, CommonTask: task.CommonTask, } blob, err := fetcher.Fetch(&fetcherTask) if err != nil { return nil, err } logger.Debugf("%s Fetch %d bytes from %s: %s", task.Id, len(blob), task.Host, blob) return blob, nil }
func NewAgaveSender(config AgaveConfig) (as IAgaveSender, err error) { logger.Debugf("%s AgaveConfig: %s", config.Id, config) as = &AgaveSender{ AgaveConfig: config, } return as, nil }
func (r *RazladkiSender) send(data tasks.DataType, timestamp uint64) (*RazladkiResult, error) { logger.Debugf("%s Data to send: %v", r.id, data) result := RazladkiResult{ Timestamp: timestamp, Params: make(map[string]Param), Alarms: make(map[string]Alarm), } for aggname, title := range r.Items { var root, metricname string items := strings.SplitN(aggname, ".", 2) if len(items) > 1 { root, metricname = items[0], items[1] } else { root = items[0] } for subgroup, value := range data[root] { rv := reflect.ValueOf(value) switch rv.Kind() { case reflect.Slice, reflect.Array: // unsupported case reflect.Map: if len(metricname) == 0 { continue } key := reflect.ValueOf(metricname) mapVal := rv.MapIndex(key) if !mapVal.IsValid() { continue } value := reflect.ValueOf(mapVal.Interface()) switch value.Kind() { case reflect.Slice, reflect.Array: // unsupported case reflect.Map: // unsupported default: name := fmt.Sprintf("%s_%s", subgroup, metricname) result.Push(name, common.InterfaceToString(value.Interface()), title) } default: if len(metricname) != 0 { continue } name := fmt.Sprintf("%s_%s", subgroup, root) result.Push(name, common.InterfaceToString(value), title) } } } return &result, nil }
func Send(request *cocaine.Request, response *cocaine.Response) { raw := <-request.Read() var task Task err := common.Unpack(raw, &task) if err != nil { response.ErrorMsg(-100, err.Error()) return } logger.Debugf("%s Task: %v", task.Id, task) task.Config.Id = task.Id task.Config.Hosts, err = getAgaveHosts() if err != nil { response.ErrorMsg(-100, err.Error()) return } if len(task.Config.Fields) == 0 { task.Config.Fields = DEFAULT_FIELDS } if task.Config.Step == 0 { task.Config.Step = DEFAULT_STEP } logger.Debugf("%s Fields: %v Step: %d", task.Id, task.Config.Fields, task.Config.Step) as, err := agave.NewAgaveSender(task.Config) if err != nil { logger.Errf("%s Unexpected error %s", task.Id, err) response.ErrorMsg(-100, err.Error()) response.Close() return } as.Send(task.Data) response.Write("OK") response.Close() }
func (as *AgaveSender) sendPoint(url string) { for _, host := range as.Hosts { req, _ := http.NewRequest("GET", fmt.Sprintf("http://%s%s", host, url), nil) req.Header = DEFAULT_HEADERS logger.Debugf("%s %s", as.Id, req.URL) resp, err := AgaveHttpClient.Do(req) if err != nil { logger.Errf("%s Unable to do request %s", as.Id, err) continue } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { logger.Errf("%s %s %d %s", as.Id, req.URL, resp.StatusCode, err) continue } logger.Infof("%s %s %d %s", as.Id, req.URL, resp.StatusCode, body) } }
func Parsing(ctx *ParsingContext, task tasks.ParsingTask) (*ParsingResult, error) { logger.Infof("%s start parsing", task.Id) var ( blob []byte err error payload interface{} wg sync.WaitGroup ) blob, err = fetchDataFromTarget(&task) if err != nil { logger.Errf("%s error `%v` occured while fetching data", task.Id, err) return nil, err } if !task.ParsingConfig.SkipParsingStage() { logger.Infof("%s Send data to parsing", task.Id) parsingApp, err := ctx.Resolver.Resolve(ctx.Ctx, common.PARSINGAPP) if err != nil { logger.Errf("%s error `%v` occured while resolving %s", task.Id, err, common.PARSINGAPP) return nil, err } taskToParser, _ := common.Pack([]interface{}{task.Id, task.ParsingConfig.Parser, blob}) if err := parsingApp.Do("enqueue", "parse", taskToParser).Wait(ctx.Ctx, &blob); err != nil { logger.Errf("%s error `%v` occured while parsing data", task.Id, err) return nil, err } } payload = blob if !task.ParsingConfig.Raw { logger.Debugf("%s Use %s for handle data", task.Id, common.DATABASEAPP) datagrid, err := ctx.Resolver.Resolve(ctx.Ctx, common.DATABASEAPP) if err != nil { logger.Errf("%s unable to get DG %v", task.Id, err) return nil, err } var token string if err := datagrid.Do("enqueue", "put", blob).Wait(ctx.Ctx, &token); err != nil { logger.Errf("%s unable to put data to DG %v", task.Id, err) return nil, err } defer func() { taskToDatagrid, _ := common.Pack([]interface{}{token}) datagrid.Do("enqueue", "drop", taskToDatagrid) logger.Debugf("%s Drop table", task.Id) }() payload = token } pr := NewParsingResult() for aggLogName, aggCfg := range task.AggregationConfigs { for k, v := range aggCfg.Data { aggType, err := v.Type() if err != nil { logger.Errf("no type in configuration: %s %s %v", aggLogName, k, v) return nil, err } logger.Debugf("%s Send to %s %s type %s %v", task.Id, aggLogName, k, aggType, v) wg.Add(1) go func(name string, dataName string, v interface{}, configName string) { defer wg.Done() app, err := ctx.Resolver.Resolve(ctx.Ctx, name) if err != nil { logger.Errf("%s %s %s", task.Id, name, err) return } // Task structure t, _ := common.Pack(map[string]interface{}{ "config": v, "token": payload, "prevtime": task.PrevTime, "currtime": task.CurrTime, "id": task.Id, }) var rawRes []byte if err := app.Do("enqueue", "aggregate_host", t).Wait(ctx.Ctx, &rawRes); err != nil { logger.Errf("%s Failed task: %v", task.Id, err) return } logger.Debugf("result for %s %s: %v", configName, dataName, rawRes) pr.Put(configName, dataName, rawRes) }(aggType, k, v, aggLogName) } } wg.Wait() logger.Infof("%s Done", task.Id) return pr, nil }
func Aggregate(ctx *AggregationContext, task tasks.AggregationTask) error { logger.Debugf("%s Start aggregation", task.Id) return nil }
func (as *AgaveSender) send(data tasks.DataType) (map[string][]string, error) { // Repack data by subgroups logger.Debugf("%s Data to send: %v", as.Id, data) var repacked map[string][]string = make(map[string][]string) for _, aggname := range as.Items { var root, metricname string items := strings.SplitN(aggname, ".", 2) if len(items) > 1 { root, metricname = items[0], items[1] } else { root = items[0] } for subgroup, value := range data[root] { rv := reflect.ValueOf(value) switch rv.Kind() { case reflect.Slice, reflect.Array: if len(metricname) != 0 { // we expect neted map here continue } if len(as.Fields) == 0 || len(as.Fields) != rv.Len() { logger.Errf("%s Unable to send a slice. Fields len %d, len of value %d", as.Id, len(as.Fields), rv.Len()) continue } forJoin := make([]string, 0, len(as.Fields)) for i, field := range as.Fields { forJoin = append(forJoin, fmt.Sprintf("%s:%s", field, common.InterfaceToString(rv.Index(i).Interface()))) } repacked[subgroup] = append(repacked[subgroup], strings.Join(forJoin, "+")) case reflect.Map: if len(metricname) == 0 { continue } key := reflect.ValueOf(metricname) mapVal := rv.MapIndex(key) if !mapVal.IsValid() { continue } value := reflect.ValueOf(mapVal.Interface()) switch value.Kind() { case reflect.Slice, reflect.Array: if len(as.Fields) == 0 || len(as.Fields) != value.Len() { logger.Errf("%s Unable to send a slice. Fields len %d, len of value %d", as.Id, len(as.Fields), rv.Len()) continue } forJoin := make([]string, 0, len(as.Fields)) for i, field := range as.Fields { forJoin = append(forJoin, fmt.Sprintf("%s:%s", field, common.InterfaceToString(value.Index(i).Interface()))) } repacked[subgroup] = append(repacked[subgroup], strings.Join(forJoin, "+")) case reflect.Map: //unsupported default: repacked[subgroup] = append(repacked[subgroup], fmt.Sprintf("%s:%s", metricname, common.InterfaceToString(value.Interface()))) } // } default: if len(metricname) != 0 { // we expect neted map here continue } repacked[subgroup] = append(repacked[subgroup], fmt.Sprintf("%s:%s", root, common.InterfaceToString(value))) } } } return repacked, nil }