func openTSDB(config ConfigApp, buffer *bytes.Buffer) error {
logCtx := log.WithFields(log.Fields{
"in": "openTSDB",
"ctx": "Metric format error",
})
maxMetrics := 10
if i, err := strconv.Atoi(os.Getenv("STATSDAEMON_MAXMETRICS")); err == nil && i > 0 {
maxMetrics = i
}
if config.OpenTSDBAddress != "-" && config.OpenTSDBAddress != "" {
datapoints := []tsdb.DataPoint{}
TSDB := tsdb.TSDB{}
server := tsdb.Server{}
serverAdress := strings.Split(config.OpenTSDBAddress, ":")
if len(serverAdress) != 2 {
return fmt.Errorf("Incorrect openTSDB server address %v", serverAdress)
}
port, err := strconv.ParseUint(serverAdress[1], 10, 32)
if err != nil {
return err
}
server.Host = serverAdress[0]
server.Port = uint(port)
TSDB.Servers = append(TSDB.Servers, server)
metrics := strings.Split(buffer.String(), "\n")
metrics = removeEmptyLines(metrics)
num := len(metrics)
currentMetricsNum := 0
for idx, mtr := range metrics {
// target format: cpu.load 12.50 112345566 host=dev,zone=west
data := strings.Split(mtr, " ")
if len(data) >= 3 {
metric := tsdb.Metric{}
value := tsdb.Value{}
tags := tsdb.Tags{}
timestamp := tsdb.Time{}
datapoint := tsdb.DataPoint{}
// parse value
// K/V values NOT allowed for OpenTSDB as metric
// TODO - consider setting them as tags
val, err := strconv.ParseFloat(data[1], 64)
if err != nil {
// continue on error in one metric
logCtx.WithField("after", "ParseFloat").Errorf("Only float/integer values allowed. Got: %s in line \"%s\". Error: %s", data[1], data, err)
Stat.ErrorIncr()
continue
}
value.Set(val)
// parse timestamp
err = timestamp.Parse(data[2])
if err != nil {
logCtx.WithField("after", "Parse").Errorf("Timestamp expected. Got: %s in line \"%s\". Error: %s", data[2], data, err)
Stat.ErrorIncr()
continue
}
// parse metric/bucket
metricName := data[0]
err = metric.Set(metricName)
if err != nil {
logCtx.WithField("after", "Set").Errorf("Metric name expected. Got: %s in line \"%s\". Error: %s", data[0], data, err)
Stat.ErrorIncr()
continue
}
if len(data) == 4 {
combinedTagsSlice := strings.Split(data[3], ",")
if len(combinedTagsSlice) > 0 {
for _, e := range combinedTagsSlice {
strSlice := strings.Split(e, "=")
if len(strSlice) == 2 {
if strSlice[0] == "" || strSlice[1] == "" {
logCtx.WithField("after", "Split").Errorf("Tag expected. Got: %s in line \"%s\"", e, data)
Stat.ErrorIncr()
continue
}
tags.Set(strSlice[0], strSlice[1])
}
}
}
}
datapoint.Value = &value
datapoint.Metric = &metric
datapoint.Tags = &tags
datapoint.Timestamp = ×tamp
datapoints = append(datapoints, datapoint)
currentMetricsNum++
// FIXME - heuristic that 10 is low enough to be accepted by OpenTSDB or env variable STATSDAEMON_MAXMETRICS
if (currentMetricsNum%maxMetrics == 0) || idx == num-1 {
log.Printf("currentMetricsNum: %d", currentMetricsNum)
out, err := TSDB.Put(datapoints)
if len(out.Errors) > 0 || err != nil {
sout := []string{}
for _, elem := range out.Errors {
sout = append(sout, elem.Error)
}
log.Printf("OpenTSDB.Put return: %s, error: %s", strings.Join(sout, "; "), err)
}
if err != nil {
return err
}
datapoints = []tsdb.DataPoint{}
}
} else {
logCtx.WithField("after", "Split").Errorf("Buffer format. Expected \"metric value timestamp\". Got \"%s\"", mtr)
Stat.ErrorIncr()
}
}
// log.Printf("datapoints len: %d, data: %s\n", len(datapoints), StructPrettyPrint(datapoints))
// TSDB.Servers = append(TSDB.Servers, server)
// _, err = TSDB.Put(datapoints)
// if err != nil {
// return err
// }
logCtx = log.WithFields(log.Fields{
"in": "openTSDB",
"ctx": "success writing to OpenTSDB",
})
logCtx.WithField("after", "Put").Infof("sent %d stats to %s", currentMetricsNum, config.OpenTSDBAddress)
return nil
}
return fmt.Errorf("No valid OpenTSDB address: %s", config.OpenTSDBAddress)
}
func submit(deadline time.Time) error {
var buffer bytes.Buffer
var num int64
now := time.Now().Unix()
num += processCounters(&buffer, now)
num += processGauges(&buffer, now)
num += processTimers(&buffer, now, percentThreshold)
if *graphiteAddress != "-" {
client, err := net.Dial("tcp", *graphiteAddress)
if err != nil {
if *debug {
log.Printf("WARNING: resetting counters when in debug mode")
processCounters(&buffer, now)
processGauges(&buffer, now)
processTimers(&buffer, now, percentThreshold)
}
return spew.Errorf("dialing %s failed - %s", *graphiteAddress, err)
}
defer client.Close()
err = client.SetDeadline(deadline)
if err != nil {
return spew.Errorf("could not set deadline: %v", err)
}
if num == 0 {
return nil
}
if *debug {
for _, line := range bytes.Split(buffer.Bytes(), []byte("\n")) {
if len(line) == 0 {
continue
}
log.Printf("DEBUG: %s", line)
}
}
_, err = client.Write(buffer.Bytes())
if err != nil {
return spew.Errorf("failed to write stats - %s", err)
}
log.Printf("sent %d stats to %s", num, *graphiteAddress)
}
if *openTSDBAddress != "-" {
datapoints := []tsdb.DataPoint{}
TSDB := tsdb.TSDB{}
server := tsdb.Server{}
serverAdress := strings.Split(*openTSDBAddress, ":")
if len(serverAdress) != 2 {
return spew.Errorf("Error: Incorrect openTSDB server address")
}
port, err := strconv.ParseUint(serverAdress[1], 10, 32)
if err != nil {
return spew.Errorf("%d %v: '%#v'", getCaller(), err, serverAdress[1])
}
server.Host = serverAdress[0]
server.Port = uint(port)
TSDB.Servers = append(TSDB.Servers, server)
metrics := strings.Split(buffer.String(), "\n")
for _, mtr := range metrics {
data := strings.Split(mtr, " ")
if len(data) == 3 {
metric := tsdb.Metric{}
value := tsdb.Value{}
tags := tsdb.Tags{}
timestamp := tsdb.Time{}
datapoint := tsdb.DataPoint{}
val, err := strconv.ParseFloat(data[1], 64)
if err != nil {
return spew.Errorf("%d %v: '%#v'", getCaller(), err, data[1])
}
value.Set(val)
err = timestamp.Parse(data[2])
if err != nil {
return spew.Errorf("%d %v: '%#v'", getCaller(), err, data[2])
}
metricAndTags := strings.Split(data[0], "?")
if metricAndTags[0] != data[0] {
err = metric.Set(metricAndTags[0])
if err != nil {
return spew.Errorf("%d %v: '%#v'", getCaller(), err, metricAndTags[0])
}
for _, tagVal := range strings.Split(metricAndTags[1], "&") {
arrTagVal := strings.Split(tagVal, "=")
if len(arrTagVal) != 2 {
return spew.Errorf("Error: Incorrect metric format %s: '%s'", "tagVal", tagVal)
}
tags.Set(arrTagVal[0], arrTagVal[1])
}
} else {
metricAndTags := strings.Split(data[0], "._t_")
if len(metricAndTags) != 2 {
return spew.Errorf("Error: Incorrect metric format %s: '%s'", "data[0]", data[0])
}
err = metric.Set(metricAndTags[0])
if err != nil {
return spew.Errorf("%v : '%#v'", err, metricAndTags[0])
}
arrTagVal := strings.Split(metricAndTags[1], ".")
if len(arrTagVal) != 2 {
return spew.Errorf("Error: Incorrect metric format %s: '%s'", "metricAndTags[1]", metricAndTags[1])
}
tags.Set(arrTagVal[0], arrTagVal[1])
}
datapoint.Value = &value
datapoint.Metric = &metric
datapoint.Tags = &tags
datapoint.Timestamp = ×tamp
datapoints = append(datapoints, datapoint)
if len(datapoints) > 10 {
_, err = TSDB.Put(datapoints)
if err != nil {
return spew.Errorf("%d %v: '%#v'", getCaller(), err, datapoints)
}
datapoints = []tsdb.DataPoint{}
}
}
}
if len(datapoints) > 0 {
_, err = TSDB.Put(datapoints)
if err != nil {
return spew.Errorf("%d %v: '%#v'", getCaller(), err, datapoints)
}
}
log.Printf("sent %d stats to %s", num, *openTSDBAddress)
}
return nil
}