func fetchDataFromTarget(task *tasks.ParsingTask) ([]byte, error) {
fetcherType, err := task.ParsingConfig.DataFetcher.Type()
if err != nil {
return nil, err
}
logger.Debugf("%s use %s for fetching data", task.Id, fetcherType)
fetcher, err := NewFetcher(fetcherType, task.ParsingConfig.DataFetcher)
if err != nil {
return nil, err
}
fetcherTask := tasks.FetcherTask{
Target: task.Host,
CommonTask: task.CommonTask,
}
blob, err := fetcher.Fetch(&fetcherTask)
if err != nil {
return nil, err
}
logger.Debugf("%s Fetch %d bytes from %s: %s", task.Id, len(blob), task.Host, blob)
return blob, nil
}
func NewAgaveSender(config AgaveConfig) (as IAgaveSender, err error) {
logger.Debugf("%s AgaveConfig: %s", config.Id, config)
as = &AgaveSender{
AgaveConfig: config,
}
return as, nil
}
func (r *RazladkiSender) send(data tasks.DataType, timestamp uint64) (*RazladkiResult, error) {
logger.Debugf("%s Data to send: %v", r.id, data)
result := RazladkiResult{
Timestamp: timestamp,
Params: make(map[string]Param),
Alarms: make(map[string]Alarm),
}
for aggname, title := range r.Items {
var root, metricname string
items := strings.SplitN(aggname, ".", 2)
if len(items) > 1 {
root, metricname = items[0], items[1]
} else {
root = items[0]
}
for subgroup, value := range data[root] {
rv := reflect.ValueOf(value)
switch rv.Kind() {
case reflect.Slice, reflect.Array:
// unsupported
case reflect.Map:
if len(metricname) == 0 {
continue
}
key := reflect.ValueOf(metricname)
mapVal := rv.MapIndex(key)
if !mapVal.IsValid() {
continue
}
value := reflect.ValueOf(mapVal.Interface())
switch value.Kind() {
case reflect.Slice, reflect.Array:
// unsupported
case reflect.Map:
// unsupported
default:
name := fmt.Sprintf("%s_%s", subgroup, metricname)
result.Push(name, common.InterfaceToString(value.Interface()), title)
}
default:
if len(metricname) != 0 {
continue
}
name := fmt.Sprintf("%s_%s", subgroup, root)
result.Push(name, common.InterfaceToString(value), title)
}
}
}
return &result, nil
}
func Send(request *cocaine.Request, response *cocaine.Response) {
raw := <-request.Read()
var task Task
err := common.Unpack(raw, &task)
if err != nil {
response.ErrorMsg(-100, err.Error())
return
}
logger.Debugf("%s Task: %v", task.Id, task)
task.Config.Id = task.Id
task.Config.Hosts, err = getAgaveHosts()
if err != nil {
response.ErrorMsg(-100, err.Error())
return
}
if len(task.Config.Fields) == 0 {
task.Config.Fields = DEFAULT_FIELDS
}
if task.Config.Step == 0 {
task.Config.Step = DEFAULT_STEP
}
logger.Debugf("%s Fields: %v Step: %d", task.Id, task.Config.Fields, task.Config.Step)
as, err := agave.NewAgaveSender(task.Config)
if err != nil {
logger.Errf("%s Unexpected error %s", task.Id, err)
response.ErrorMsg(-100, err.Error())
response.Close()
return
}
as.Send(task.Data)
response.Write("OK")
response.Close()
}
func (as *AgaveSender) sendPoint(url string) {
for _, host := range as.Hosts {
req, _ := http.NewRequest("GET",
fmt.Sprintf("http://%s%s", host, url),
nil)
req.Header = DEFAULT_HEADERS
logger.Debugf("%s %s", as.Id, req.URL)
resp, err := AgaveHttpClient.Do(req)
if err != nil {
logger.Errf("%s Unable to do request %s", as.Id, err)
continue
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
logger.Errf("%s %s %d %s", as.Id, req.URL, resp.StatusCode, err)
continue
}
logger.Infof("%s %s %d %s", as.Id, req.URL, resp.StatusCode, body)
}
}
func Parsing(ctx *ParsingContext, task tasks.ParsingTask) (*ParsingResult, error) {
logger.Infof("%s start parsing", task.Id)
var (
blob []byte
err error
payload interface{}
wg sync.WaitGroup
)
blob, err = fetchDataFromTarget(&task)
if err != nil {
logger.Errf("%s error `%v` occured while fetching data", task.Id, err)
return nil, err
}
if !task.ParsingConfig.SkipParsingStage() {
logger.Infof("%s Send data to parsing", task.Id)
parsingApp, err := ctx.Resolver.Resolve(ctx.Ctx, common.PARSINGAPP)
if err != nil {
logger.Errf("%s error `%v` occured while resolving %s",
task.Id, err, common.PARSINGAPP)
return nil, err
}
taskToParser, _ := common.Pack([]interface{}{task.Id, task.ParsingConfig.Parser, blob})
if err := parsingApp.Do("enqueue", "parse", taskToParser).Wait(ctx.Ctx, &blob); err != nil {
logger.Errf("%s error `%v` occured while parsing data", task.Id, err)
return nil, err
}
}
payload = blob
if !task.ParsingConfig.Raw {
logger.Debugf("%s Use %s for handle data", task.Id, common.DATABASEAPP)
datagrid, err := ctx.Resolver.Resolve(ctx.Ctx, common.DATABASEAPP)
if err != nil {
logger.Errf("%s unable to get DG %v", task.Id, err)
return nil, err
}
var token string
if err := datagrid.Do("enqueue", "put", blob).Wait(ctx.Ctx, &token); err != nil {
logger.Errf("%s unable to put data to DG %v", task.Id, err)
return nil, err
}
defer func() {
taskToDatagrid, _ := common.Pack([]interface{}{token})
datagrid.Do("enqueue", "drop", taskToDatagrid)
logger.Debugf("%s Drop table", task.Id)
}()
payload = token
}
pr := NewParsingResult()
for aggLogName, aggCfg := range task.AggregationConfigs {
for k, v := range aggCfg.Data {
aggType, err := v.Type()
if err != nil {
logger.Errf("no type in configuration: %s %s %v", aggLogName, k, v)
return nil, err
}
logger.Debugf("%s Send to %s %s type %s %v", task.Id, aggLogName, k, aggType, v)
wg.Add(1)
go func(name string, dataName string, v interface{}, configName string) {
defer wg.Done()
app, err := ctx.Resolver.Resolve(ctx.Ctx, name)
if err != nil {
logger.Errf("%s %s %s", task.Id, name, err)
return
}
// Task structure
t, _ := common.Pack(map[string]interface{}{
"config": v,
"token": payload,
"prevtime": task.PrevTime,
"currtime": task.CurrTime,
"id": task.Id,
})
var rawRes []byte
if err := app.Do("enqueue", "aggregate_host", t).Wait(ctx.Ctx, &rawRes); err != nil {
logger.Errf("%s Failed task: %v", task.Id, err)
return
}
logger.Debugf("result for %s %s: %v", configName, dataName, rawRes)
pr.Put(configName, dataName, rawRes)
}(aggType, k, v, aggLogName)
}
}
wg.Wait()
logger.Infof("%s Done", task.Id)
return pr, nil
}
func Aggregate(ctx *AggregationContext, task tasks.AggregationTask) error {
logger.Debugf("%s Start aggregation", task.Id)
return nil
}
func (as *AgaveSender) send(data tasks.DataType) (map[string][]string, error) {
// Repack data by subgroups
logger.Debugf("%s Data to send: %v", as.Id, data)
var repacked map[string][]string = make(map[string][]string)
for _, aggname := range as.Items {
var root, metricname string
items := strings.SplitN(aggname, ".", 2)
if len(items) > 1 {
root, metricname = items[0], items[1]
} else {
root = items[0]
}
for subgroup, value := range data[root] {
rv := reflect.ValueOf(value)
switch rv.Kind() {
case reflect.Slice, reflect.Array:
if len(metricname) != 0 {
// we expect neted map here
continue
}
if len(as.Fields) == 0 || len(as.Fields) != rv.Len() {
logger.Errf("%s Unable to send a slice. Fields len %d, len of value %d", as.Id, len(as.Fields), rv.Len())
continue
}
forJoin := make([]string, 0, len(as.Fields))
for i, field := range as.Fields {
forJoin = append(forJoin, fmt.Sprintf("%s:%s", field, common.InterfaceToString(rv.Index(i).Interface())))
}
repacked[subgroup] = append(repacked[subgroup], strings.Join(forJoin, "+"))
case reflect.Map:
if len(metricname) == 0 {
continue
}
key := reflect.ValueOf(metricname)
mapVal := rv.MapIndex(key)
if !mapVal.IsValid() {
continue
}
value := reflect.ValueOf(mapVal.Interface())
switch value.Kind() {
case reflect.Slice, reflect.Array:
if len(as.Fields) == 0 || len(as.Fields) != value.Len() {
logger.Errf("%s Unable to send a slice. Fields len %d, len of value %d", as.Id, len(as.Fields), rv.Len())
continue
}
forJoin := make([]string, 0, len(as.Fields))
for i, field := range as.Fields {
forJoin = append(forJoin, fmt.Sprintf("%s:%s", field, common.InterfaceToString(value.Index(i).Interface())))
}
repacked[subgroup] = append(repacked[subgroup], strings.Join(forJoin, "+"))
case reflect.Map:
//unsupported
default:
repacked[subgroup] = append(repacked[subgroup], fmt.Sprintf("%s:%s", metricname, common.InterfaceToString(value.Interface())))
}
// }
default:
if len(metricname) != 0 {
// we expect neted map here
continue
}
repacked[subgroup] = append(repacked[subgroup], fmt.Sprintf("%s:%s", root, common.InterfaceToString(value)))
}
}
}
return repacked, nil
}