Esempio n. 1
0
func fetchDataFromTarget(task *tasks.ParsingTask) ([]byte, error) {
	fetcherType, err := task.ParsingConfig.DataFetcher.Type()
	if err != nil {
		return nil, err
	}

	logger.Debugf("%s use %s for fetching data", task.Id, fetcherType)
	fetcher, err := NewFetcher(fetcherType, task.ParsingConfig.DataFetcher)
	if err != nil {
		return nil, err
	}

	fetcherTask := tasks.FetcherTask{
		Target:     task.Host,
		CommonTask: task.CommonTask,
	}

	blob, err := fetcher.Fetch(&fetcherTask)
	if err != nil {
		return nil, err
	}

	logger.Debugf("%s Fetch %d bytes from %s: %s", task.Id, len(blob), task.Host, blob)
	return blob, nil
}
Esempio n. 2
0
func NewAgaveSender(config AgaveConfig) (as IAgaveSender, err error) {
	logger.Debugf("%s AgaveConfig: %s", config.Id, config)
	as = &AgaveSender{
		AgaveConfig: config,
	}
	return as, nil
}
Esempio n. 3
0
func (r *RazladkiSender) send(data tasks.DataType, timestamp uint64) (*RazladkiResult, error) {
	logger.Debugf("%s Data to send: %v", r.id, data)
	result := RazladkiResult{
		Timestamp: timestamp,
		Params:    make(map[string]Param),
		Alarms:    make(map[string]Alarm),
	}
	for aggname, title := range r.Items {
		var root, metricname string
		items := strings.SplitN(aggname, ".", 2)
		if len(items) > 1 {
			root, metricname = items[0], items[1]
		} else {
			root = items[0]
		}
		for subgroup, value := range data[root] {
			rv := reflect.ValueOf(value)
			switch rv.Kind() {
			case reflect.Slice, reflect.Array:
				// unsupported
			case reflect.Map:
				if len(metricname) == 0 {
					continue
				}

				key := reflect.ValueOf(metricname)
				mapVal := rv.MapIndex(key)
				if !mapVal.IsValid() {
					continue
				}

				value := reflect.ValueOf(mapVal.Interface())

				switch value.Kind() {
				case reflect.Slice, reflect.Array:
					// unsupported
				case reflect.Map:
					// unsupported
				default:
					name := fmt.Sprintf("%s_%s", subgroup, metricname)
					result.Push(name, common.InterfaceToString(value.Interface()), title)
				}
			default:
				if len(metricname) != 0 {
					continue
				}
				name := fmt.Sprintf("%s_%s", subgroup, root)
				result.Push(name, common.InterfaceToString(value), title)
			}
		}
	}
	return &result, nil
}
Esempio n. 4
0
func Send(request *cocaine.Request, response *cocaine.Response) {
	raw := <-request.Read()

	var task Task
	err := common.Unpack(raw, &task)
	if err != nil {
		response.ErrorMsg(-100, err.Error())
		return
	}
	logger.Debugf("%s Task: %v", task.Id, task)

	task.Config.Id = task.Id
	task.Config.Hosts, err = getAgaveHosts()
	if err != nil {
		response.ErrorMsg(-100, err.Error())
		return
	}

	if len(task.Config.Fields) == 0 {
		task.Config.Fields = DEFAULT_FIELDS
	}

	if task.Config.Step == 0 {
		task.Config.Step = DEFAULT_STEP
	}

	logger.Debugf("%s Fields: %v Step: %d", task.Id, task.Config.Fields, task.Config.Step)

	as, err := agave.NewAgaveSender(task.Config)
	if err != nil {
		logger.Errf("%s Unexpected error %s", task.Id, err)
		response.ErrorMsg(-100, err.Error())
		response.Close()
		return
	}
	as.Send(task.Data)
	response.Write("OK")
	response.Close()
}
Esempio n. 5
0
func (as *AgaveSender) sendPoint(url string) {
	for _, host := range as.Hosts {
		req, _ := http.NewRequest("GET",
			fmt.Sprintf("http://%s%s", host, url),
			nil)
		req.Header = DEFAULT_HEADERS

		logger.Debugf("%s %s", as.Id, req.URL)
		resp, err := AgaveHttpClient.Do(req)
		if err != nil {
			logger.Errf("%s Unable to do request %s", as.Id, err)
			continue
		}
		defer resp.Body.Close()

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			logger.Errf("%s %s %d %s", as.Id, req.URL, resp.StatusCode, err)
			continue
		}

		logger.Infof("%s %s %d %s", as.Id, req.URL, resp.StatusCode, body)
	}
}
Esempio n. 6
0
func Parsing(ctx *ParsingContext, task tasks.ParsingTask) (*ParsingResult, error) {
	logger.Infof("%s start parsing", task.Id)

	var (
		blob    []byte
		err     error
		payload interface{}
		wg      sync.WaitGroup
	)

	blob, err = fetchDataFromTarget(&task)
	if err != nil {
		logger.Errf("%s error `%v` occured while fetching data", task.Id, err)
		return nil, err
	}

	if !task.ParsingConfig.SkipParsingStage() {
		logger.Infof("%s Send data to parsing", task.Id)

		parsingApp, err := ctx.Resolver.Resolve(ctx.Ctx, common.PARSINGAPP)
		if err != nil {
			logger.Errf("%s error `%v` occured while resolving %s",
				task.Id, err, common.PARSINGAPP)
			return nil, err
		}

		taskToParser, _ := common.Pack([]interface{}{task.Id, task.ParsingConfig.Parser, blob})
		if err := parsingApp.Do("enqueue", "parse", taskToParser).Wait(ctx.Ctx, &blob); err != nil {
			logger.Errf("%s error `%v` occured while parsing data", task.Id, err)
			return nil, err
		}
	}

	payload = blob

	if !task.ParsingConfig.Raw {
		logger.Debugf("%s Use %s for handle data", task.Id, common.DATABASEAPP)

		datagrid, err := ctx.Resolver.Resolve(ctx.Ctx, common.DATABASEAPP)
		if err != nil {
			logger.Errf("%s unable to get DG %v", task.Id, err)
			return nil, err
		}

		var token string
		if err := datagrid.Do("enqueue", "put", blob).Wait(ctx.Ctx, &token); err != nil {
			logger.Errf("%s unable to put data to DG %v", task.Id, err)
			return nil, err
		}

		defer func() {
			taskToDatagrid, _ := common.Pack([]interface{}{token})
			datagrid.Do("enqueue", "drop", taskToDatagrid)
			logger.Debugf("%s Drop table", task.Id)
		}()
		payload = token
	}

	pr := NewParsingResult()

	for aggLogName, aggCfg := range task.AggregationConfigs {
		for k, v := range aggCfg.Data {
			aggType, err := v.Type()
			if err != nil {
				logger.Errf("no type in configuration: %s %s %v", aggLogName, k, v)
				return nil, err
			}

			logger.Debugf("%s Send to %s %s type %s %v", task.Id, aggLogName, k, aggType, v)

			wg.Add(1)
			go func(name string, dataName string, v interface{}, configName string) {
				defer wg.Done()

				app, err := ctx.Resolver.Resolve(ctx.Ctx, name)
				if err != nil {
					logger.Errf("%s %s %s", task.Id, name, err)
					return
				}

				// Task structure
				t, _ := common.Pack(map[string]interface{}{
					"config":   v,
					"token":    payload,
					"prevtime": task.PrevTime,
					"currtime": task.CurrTime,
					"id":       task.Id,
				})

				var rawRes []byte
				if err := app.Do("enqueue", "aggregate_host", t).Wait(ctx.Ctx, &rawRes); err != nil {
					logger.Errf("%s Failed task: %v", task.Id, err)
					return
				}

				logger.Debugf("result for %s %s: %v", configName, dataName, rawRes)

				pr.Put(configName, dataName, rawRes)

			}(aggType, k, v, aggLogName)
		}
	}
	wg.Wait()

	logger.Infof("%s Done", task.Id)
	return pr, nil
}
Esempio n. 7
0
func Aggregate(ctx *AggregationContext, task tasks.AggregationTask) error {
	logger.Debugf("%s Start aggregation", task.Id)
	return nil
}
Esempio n. 8
0
func (as *AgaveSender) send(data tasks.DataType) (map[string][]string, error) {
	// Repack data by subgroups
	logger.Debugf("%s Data to send: %v", as.Id, data)
	var repacked map[string][]string = make(map[string][]string)
	for _, aggname := range as.Items {
		var root, metricname string
		items := strings.SplitN(aggname, ".", 2)
		if len(items) > 1 {
			root, metricname = items[0], items[1]
		} else {
			root = items[0]
		}
		for subgroup, value := range data[root] {
			rv := reflect.ValueOf(value)
			switch rv.Kind() {
			case reflect.Slice, reflect.Array:
				if len(metricname) != 0 {
					// we expect neted map here
					continue
				}
				if len(as.Fields) == 0 || len(as.Fields) != rv.Len() {
					logger.Errf("%s Unable to send a slice. Fields len %d, len of value %d", as.Id, len(as.Fields), rv.Len())
					continue
				}

				forJoin := make([]string, 0, len(as.Fields))
				for i, field := range as.Fields {
					forJoin = append(forJoin, fmt.Sprintf("%s:%s", field, common.InterfaceToString(rv.Index(i).Interface())))
				}

				repacked[subgroup] = append(repacked[subgroup], strings.Join(forJoin, "+"))
			case reflect.Map:
				if len(metricname) == 0 {
					continue
				}

				key := reflect.ValueOf(metricname)
				mapVal := rv.MapIndex(key)
				if !mapVal.IsValid() {
					continue
				}

				value := reflect.ValueOf(mapVal.Interface())

				switch value.Kind() {
				case reflect.Slice, reflect.Array:
					if len(as.Fields) == 0 || len(as.Fields) != value.Len() {
						logger.Errf("%s Unable to send a slice. Fields len %d, len of value %d", as.Id, len(as.Fields), rv.Len())
						continue
					}
					forJoin := make([]string, 0, len(as.Fields))
					for i, field := range as.Fields {
						forJoin = append(forJoin, fmt.Sprintf("%s:%s", field, common.InterfaceToString(value.Index(i).Interface())))
					}
					repacked[subgroup] = append(repacked[subgroup], strings.Join(forJoin, "+"))
				case reflect.Map:
					//unsupported
				default:
					repacked[subgroup] = append(repacked[subgroup], fmt.Sprintf("%s:%s", metricname, common.InterfaceToString(value.Interface())))
				}

				// }
			default:
				if len(metricname) != 0 {
					// we expect neted map here
					continue
				}
				repacked[subgroup] = append(repacked[subgroup], fmt.Sprintf("%s:%s", root, common.InterfaceToString(value)))
			}
		}
	}

	return repacked, nil
}