// Graph takes an OpenTSDB request data structure and queries OpenTSDB. Use the
// json parameter to pass JSON. Use the b64 parameter to pass base64-encoded
// JSON.
func Graph(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
j := []byte(r.FormValue("json"))
if bs := r.FormValue("b64"); bs != "" {
b, err := base64.StdEncoding.DecodeString(bs)
if err != nil {
return nil, err
}
j = b
}
if len(j) == 0 {
return nil, fmt.Errorf("either json or b64 required")
}
oreq, err := opentsdb.RequestFromJSON(j)
if err != nil {
return nil, err
}
if ads_v := r.FormValue("autods"); ads_v != "" {
ads_i, err := strconv.Atoi(ads_v)
if err != nil {
return nil, err
}
if err := oreq.AutoDownsample(ads_i); err != nil {
return nil, err
}
}
ar := make(map[int]bool)
for _, v := range r.Form["autorate"] {
if i, err := strconv.Atoi(v); err == nil {
ar[i] = true
}
}
queries := make([]string, len(oreq.Queries))
var start, end string
var startT, endT time.Time
if s, ok := oreq.Start.(string); ok && strings.Contains(s, "-ago") {
startT, err = opentsdb.ParseTime(s)
if err != nil {
return nil, err
}
start = strings.TrimSuffix(s, "-ago")
}
if s, ok := oreq.End.(string); ok && strings.Contains(s, "-ago") {
endT, err = opentsdb.ParseTime(s)
if err != nil {
return nil, err
}
end = strings.TrimSuffix(s, "-ago")
}
if start == "" && end == "" {
s, sok := oreq.Start.(int64)
e, eok := oreq.End.(int64)
if sok && eok {
start = fmt.Sprintf("%vs", e-s)
startT = time.Unix(s, 0)
endT = time.Unix(e, 0)
if err != nil {
return nil, err
}
}
}
if endT.Equal(time.Time{}) {
endT = time.Now().UTC()
}
m_units := make(map[string]string)
for i, q := range oreq.Queries {
if ar[i] {
meta, err := schedule.MetadataMetrics(q.Metric)
if err != nil {
return nil, err
}
if meta == nil {
return nil, fmt.Errorf("no metadata for %s: cannot use auto rate", q)
}
if meta.Unit != "" {
m_units[q.Metric] = meta.Unit
}
if meta.Rate != "" {
switch meta.Rate {
case metadata.Gauge:
// ignore
case metadata.Rate:
q.Rate = true
case metadata.Counter:
q.Rate = true
q.RateOptions = opentsdb.RateOptions{
Counter: true,
ResetValue: 1,
}
default:
return nil, fmt.Errorf("unknown metadata rate: %s", meta.Rate)
}
}
}
queries[i] = fmt.Sprintf(`q("%v", "%v", "%v")`, q, start, end)
if !schedule.SystemConf.GetTSDBContext().Version().FilterSupport() {
if err := schedule.Search.Expand(q); err != nil {
return nil, err
}
}
}
var tr opentsdb.ResponseSet
b, _ := json.MarshalIndent(oreq, "", " ")
t.StepCustomTiming("tsdb", "query", string(b), func() {
h := schedule.SystemConf.GetTSDBHost()
if h == "" {
err = fmt.Errorf("tsdbHost not set")
return
}
tr, err = oreq.Query(h)
})
if err != nil {
return nil, err
}
cs, err := makeChart(tr, m_units)
if err != nil {
return nil, err
}
if _, present := r.Form["png"]; present {
c := chart.ScatterChart{
Title: fmt.Sprintf("%v - %v", oreq.Start, queries),
}
c.XRange.Time = true
if min, err := strconv.ParseFloat(r.FormValue("min"), 64); err == nil {
c.YRange.MinMode.Fixed = true
c.YRange.MinMode.Value = min
}
if max, err := strconv.ParseFloat(r.FormValue("max"), 64); err == nil {
c.YRange.MaxMode.Fixed = true
c.YRange.MaxMode.Value = max
}
for ri, r := range cs {
pts := make([]chart.EPoint, len(r.Data))
for idx, v := range r.Data {
pts[idx].X = v[0]
pts[idx].Y = v[1]
}
slice.Sort(pts, func(i, j int) bool {
return pts[i].X < pts[j].X
})
c.AddData(r.Name, pts, chart.PlotStyleLinesPoints, sched.Autostyle(ri))
}
w.Header().Set("Content-Type", "image/svg+xml")
white := color.RGBA{0xff, 0xff, 0xff, 0xff}
const width = 800
const height = 600
s := svg.New(w)
s.Start(width, height)
s.Rect(0, 0, width, height, "fill: #ffffff")
sgr := svgg.AddTo(s, 0, 0, width, height, "", 12, white)
c.Plot(sgr)
s.End()
return nil, nil
}
var a []annotate.Annotation
warnings := []string{}
if schedule.SystemConf.AnnotateEnabled() {
a, err = annotateBackend.GetAnnotations(&startT, &endT)
if err != nil {
warnings = append(warnings, fmt.Sprintf("unable to get annotations: %v", err))
}
}
return struct {
Queries []string
Series []*chartSeries
Annotations []annotate.Annotation
Warnings []string
}{
queries,
cs,
a,
warnings,
}, nil
}
func main() {
flag.Parse()
if *tsdbHost == "" {
flag.PrintDefaults()
log.Fatal("host must be supplied")
}
putUrl := (&url.URL{Scheme: "http", Host: *tsdbHost, Path: "api/put"}).String()
if *ruleFlag == "" {
flag.PrintDefaults()
log.Fatal("rule must be supplied")
}
rules, err := denormalize.ParseDenormalizationRules(*ruleFlag)
if err != nil {
log.Fatal(err)
}
if len(rules) > 1 {
log.Fatal("Please specify only one rule")
}
var rule *denormalize.DenormalizationRule
var metric string
for k, v := range rules {
metric = k
rule = v
}
query := &opentsdb.Query{Metric: metric, Aggregator: "avg"}
query.Tags, err = queryForAggregateTags(query)
if err != nil {
log.Fatal(err)
}
startDate, err := opentsdb.ParseTime(*start)
if err != nil {
log.Fatal(err)
}
endDate := time.Now().UTC()
if *end != "" {
endDate, err = opentsdb.ParseTime(*end)
if err != nil {
log.Fatal(err)
}
}
backfill := func(batchStart, batchEnd time.Time) (err error) {
startTimeString := batchStart.Format(opentsdb.TSDBTimeFormat)
endTimeString := batchEnd.Format(opentsdb.TSDBTimeFormat)
defer func() {
if err != nil {
log.Fatalf("Error on batch %s - %s. %v \n", startTimeString, endTimeString, err)
}
}()
req := opentsdb.Request{Start: startTimeString, End: endTimeString, Queries: []*opentsdb.Query{query}}
resp, err := req.Query(*tsdbHost)
if err != nil {
return err
}
dps := []*opentsdb.DataPoint{}
for _, r := range resp {
for t, p := range r.DPS {
timeStamp, err := strconv.ParseInt(t, 10, 64)
if err != nil {
return err
}
dp := &opentsdb.DataPoint{
Timestamp: timeStamp,
Metric: r.Metric,
Tags: r.Tags,
Value: p,
}
err = rule.Translate(dp)
if err != nil {
return err
}
dps = append(dps, dp)
}
}
fmt.Printf("%s - %s: %d dps\n", startTimeString, endTimeString, len(dps))
total := 0
for len(dps) > 0 {
count := len(dps)
if len(dps) > *batchSize {
count = *batchSize
}
putResp, err := collect.SendDataPoints(dps[:count], putUrl)
if err != nil {
return err
}
defer putResp.Body.Close()
if putResp.StatusCode != 204 {
return fmt.Errorf("Non 204 status code from opentsdb: %d", putResp.StatusCode)
}
dps = dps[count:]
total += count
}
fmt.Printf("Relayed %d data points.\n", total)
return nil
}
// walk backwards a day at a time
curEnd := endDate
for curEnd.After(startDate) {
curStart := curEnd.Add(-24 * time.Hour)
if curStart.Before(startDate) {
curStart = startDate
}
backfill(curStart, curEnd)
curEnd = curEnd.Add(-24 * time.Hour)
}
}