func (t *Tsdb) Flush() {
t.Lock()
if len(t.Metrics) == 0 {
t.Unlock()
return
}
metrics := make([]*schema.MetricData, len(t.Metrics))
copy(metrics, t.Metrics)
t.Metrics = t.Metrics[:0]
t.Unlock()
// Write the metrics to our HTTP server.
log.Debug("writing %d metrics to API", len(metrics))
batches := schema.Reslice(metrics, maxMetricsPerFlush*2)
for _, batch := range batches {
id := time.Now().UnixNano()
body, err := msg.CreateMsg(batch, id, msg.FormatMetricDataArrayMsgp)
if err != nil {
log.Error(3, "unable to convert metrics to MetricDataArrayMsgp.", "error", err)
return
}
t.dataChan <- tsdbData{Path: "metrics", Body: body}
log.Debug("%d metrics queud for delivery", len(batch))
}
}
func (route *GrafanaNet) run() {
metrics := make([]*schema.MetricData, 0, route.flushMaxNum)
ticker := time.NewTicker(route.flushMaxWait)
client := &http.Client{
Timeout: route.timeout,
}
if !route.sslVerify {
// this transport should be the equivalent of Go's DefaultTransport
client.Transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
// except we removed ExpectContinueTimeout cause it doesn't seem very useful, requires go 1.6, and prevents using http2.
// and except for this
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
}
b := &backoff.Backoff{
Min: 100 * time.Millisecond,
Max: time.Minute,
Factor: 1.5,
Jitter: true,
}
flush := func() {
if len(metrics) == 0 {
return
}
mda := schema.MetricDataArray(metrics)
data, err := msg.CreateMsg(mda, 0, msg.FormatMetricDataArrayMsgp)
if err != nil {
panic(err)
}
for {
pre := time.Now()
req, err := http.NewRequest("POST", route.addr, bytes.NewBuffer(data))
if err != nil {
panic(err)
}
req.Header.Add("Authorization", "Bearer "+route.apiKey)
req.Header.Add("Content-Type", "rt-metric-binary")
resp, err := client.Do(req)
diff := time.Since(pre)
if err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 {
b.Reset()
log.Info("GrafanaNet sent %d metrics in %s -msg size %d", len(metrics), diff, len(data))
route.numOut.Inc(int64(len(metrics)))
route.tickFlushSize.Update(int64(len(data)))
route.durationTickFlush.Update(diff)
metrics = metrics[:0]
resp.Body.Close()
break
}
route.numErrFlush.Inc(1)
dur := b.Duration()
if err != nil {
log.Warning("GrafanaNet failed to submit data: %s will try again in %s (this attempt took %s)", err, dur, diff)
} else {
buf := make([]byte, 300)
n, _ := resp.Body.Read(buf)
log.Warning("GrafanaNet failed to submit data: http %d - %s will try again in %s (this attempt took %s)", resp.StatusCode, buf[:n], dur, diff)
resp.Body.Close()
}
time.Sleep(dur)
}
}
for {
select {
case buf := <-route.buf:
route.numBuffered.Dec(1)
md := parseMetric(buf, route.schemas)
if md == nil {
continue
}
md.SetId()
metrics = append(metrics, md)
if len(metrics) == route.flushMaxNum {
flush()
}
case <-ticker.C:
flush()
}
}
}