// LogFmtMetricsEmitter emits the metrics in logfmt compatible formats every d
// duration using the provided logger. source is added to the line as
// log_shuttle_stats_source if not empty.
func LogFmtMetricsEmitter(r metrics.Registry, source string, d time.Duration, l *log.Logger) {
if d == 0 {
return
}
for _ = range time.Tick(d) {
ctx := slog.Context{}
if source != "" {
ctx["log_shuttle_stats_source"] = source
}
r.Each(func(name string, i interface{}) {
switch metric := i.(type) {
case metrics.Counter:
countDifference(ctx, name, metric.Count())
case metrics.Gauge:
ctx[name] = metric.Value()
case metrics.GaugeFloat64:
ctx[name] = metric.Value()
case metrics.Healthcheck:
metric.Check()
ctx[name] = metric.Error()
case metrics.Histogram:
s := metric.Snapshot()
ps := s.Percentiles(percentiles)
countDifference(ctx, name, s.Count())
ctx[name+".min"] = s.Min()
ctx[name+".max"] = s.Max()
ctx[name+".mean"] = s.Mean()
ctx[name+".stddev"] = s.StdDev()
for i, pn := range percentileNames {
ctx[name+"."+pn] = ps[i]
}
case metrics.Meter:
s := metric.Snapshot()
countDifference(ctx, name, s.Count())
ctx[name+".rate.1min"] = s.Rate1()
ctx[name+".rate.5min"] = s.Rate5()
ctx[name+".rate.15min"] = s.Rate15()
ctx[name+".rate.mean"] = s.RateMean()
case metrics.Timer:
s := metric.Snapshot()
ps := s.Percentiles(percentiles)
countDifference(ctx, name, s.Count())
ctx[name+".min"] = sec(float64(s.Min()))
ctx[name+".max"] = sec(float64(s.Max()))
ctx[name+".mean"] = sec(s.Mean())
ctx[name+".stddev"] = sec(s.StdDev())
for i, pn := range percentileNames {
ctx[name+"."+pn] = sec(ps[i])
}
ctx[name+".rate.1min"] = fmt.Sprintf("%.3f", s.Rate1())
ctx[name+".rate.5min"] = fmt.Sprintf("%.3f", s.Rate5())
ctx[name+".rate.15min"] = fmt.Sprintf("%.3f", s.Rate15())
ctx[name+".rate.mean"] = fmt.Sprintf("%.3f", s.RateMean())
}
})
l.Println(ctx)
}
}
func logForever(r gmetrics.Registry, d time.Duration) {
for {
r.Each(func(name string, i interface{}) {
switch m := i.(type) {
case gmetrics.Counter:
glog.Infof("counter %s\n", name)
glog.Infof(" count: %9d\n", m.Count())
case gmetrics.Gauge:
glog.Infof("gauge %s\n", name)
glog.Infof(" value: %9d\n", m.Value())
case gmetrics.Healthcheck:
m.Check()
glog.Infof("healthcheck %s\n", name)
glog.Infof(" error: %v\n", m.Error())
case gmetrics.Histogram:
ps := m.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
glog.Infof("histogram %s\n", name)
glog.Infof(" count: %9d\n", m.Count())
glog.Infof(" min: %9d\n", m.Min())
glog.Infof(" max: %9d\n", m.Max())
glog.Infof(" mean: %12.2f\n", m.Mean())
glog.Infof(" stddev: %12.2f\n", m.StdDev())
glog.Infof(" median: %12.2f\n", ps[0])
glog.Infof(" 75%%: %12.2f\n", ps[1])
glog.Infof(" 95%%: %12.2f\n", ps[2])
glog.Infof(" 99%%: %12.2f\n", ps[3])
glog.Infof(" 99.9%%: %12.2f\n", ps[4])
case gmetrics.Meter:
glog.Infof("meter %s\n", name)
glog.Infof(" count: %9d\n", m.Count())
glog.Infof(" 1-min rate: %12.2f\n", m.Rate1())
glog.Infof(" 5-min rate: %12.2f\n", m.Rate5())
glog.Infof(" 15-min rate: %12.2f\n", m.Rate15())
glog.Infof(" mean rate: %12.2f\n", m.RateMean())
case gmetrics.Timer:
ps := m.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
glog.Infof("timer %s\n", name)
glog.Infof(" count: %9d\n", m.Count())
glog.Infof(" min: %9d\n", m.Min())
glog.Infof(" max: %9d\n", m.Max())
glog.Infof(" mean: %12.2f\n", m.Mean())
glog.Infof(" stddev: %12.2f\n", m.StdDev())
glog.Infof(" median: %12.2f\n", ps[0])
glog.Infof(" 75%%: %12.2f\n", ps[1])
glog.Infof(" 95%%: %12.2f\n", ps[2])
glog.Infof(" 99%%: %12.2f\n", ps[3])
glog.Infof(" 99.9%%: %12.2f\n", ps[4])
glog.Infof(" 1-min rate: %12.2f\n", m.Rate1())
glog.Infof(" 5-min rate: %12.2f\n", m.Rate5())
glog.Infof(" 15-min rate: %12.2f\n", m.Rate15())
glog.Infof(" mean rate: %12.2f\n", m.RateMean())
}
})
time.Sleep(d)
}
}
// publish_expvars: expose each metric in the given registry to expvars
func publish_expvars(r metrics.Registry) {
du := float64(time.Nanosecond)
percentiles := []float64{0.50, 0.75, 0.95, 0.99, 0.999}
r.Each(func(name string, i interface{}) {
switch m := i.(type) {
case metrics.Counter:
expvar.Publish(fmt.Sprintf("%s.Count", name), expvar.Func(func() interface{} {
return m.Count()
}))
case metrics.Meter:
expvar.Publish(fmt.Sprintf("%s.Rate1", name), expvar.Func(func() interface{} {
return m.Rate1()
}))
expvar.Publish(fmt.Sprintf("%s.Rate5", name), expvar.Func(func() interface{} {
return m.Rate5()
}))
expvar.Publish(fmt.Sprintf("%s.Rate15", name), expvar.Func(func() interface{} {
return m.Rate15()
}))
expvar.Publish(fmt.Sprintf("%s.Mean", name), expvar.Func(func() interface{} {
return m.RateMean()
}))
case metrics.Histogram:
expvar.Publish(fmt.Sprintf("%s.Count", name), expvar.Func(func() interface{} {
return m.Count()
}))
expvar.Publish(fmt.Sprintf("%s.Mean", name), expvar.Func(func() interface{} {
return m.Mean()
}))
expvar.Publish(fmt.Sprintf("%s.Min", name), expvar.Func(func() interface{} {
return m.Min()
}))
expvar.Publish(fmt.Sprintf("%s.Max", name), expvar.Func(func() interface{} {
return m.Max()
}))
expvar.Publish(fmt.Sprintf("%s.StdDev", name), expvar.Func(func() interface{} {
return m.StdDev()
}))
expvar.Publish(fmt.Sprintf("%s.Variance", name), expvar.Func(func() interface{} {
return m.Variance()
}))
for _, p := range percentiles {
expvar.Publish(fmt.Sprintf("%s.Percentile%2.3f", name, p), expvar.Func(func() interface{} {
return m.Percentile(p)
}))
}
case metrics.Timer:
expvar.Publish(fmt.Sprintf("%s.Rate1", name), expvar.Func(func() interface{} {
return m.Rate1()
}))
expvar.Publish(fmt.Sprintf("%s.Rate5", name), expvar.Func(func() interface{} {
return m.Rate5()
}))
expvar.Publish(fmt.Sprintf("%s.Rate15", name), expvar.Func(func() interface{} {
return m.Rate15()
}))
expvar.Publish(fmt.Sprintf("%s.RateMean", name), expvar.Func(func() interface{} {
return m.RateMean()
}))
expvar.Publish(fmt.Sprintf("%s.Mean", name), expvar.Func(func() interface{} {
return du * m.Mean()
}))
expvar.Publish(fmt.Sprintf("%s.Min", name), expvar.Func(func() interface{} {
return int64(du) * m.Min()
}))
expvar.Publish(fmt.Sprintf("%s.Max", name), expvar.Func(func() interface{} {
return int64(du) * m.Max()
}))
expvar.Publish(fmt.Sprintf("%s.StdDev", name), expvar.Func(func() interface{} {
return du * m.StdDev()
}))
expvar.Publish(fmt.Sprintf("%s.Variance", name), expvar.Func(func() interface{} {
return du * m.Variance()
}))
for _, p := range percentiles {
expvar.Publish(fmt.Sprintf("%s.Percentile%2.3f", name, p), expvar.Func(func() interface{} {
return m.Percentile(p)
}))
}
}
})
expvar.Publish("time", expvar.Func(now))
}
func dumpRegistry(registry gometrics.Registry) {
logger.Info("Dumping metrics registry")
registry.Each(func(name string, metric interface{}) {
dumpMetric(name, metric)
})
}
func registerGCStats(r metrics.Registry) {
g := metrics.NewGauge()
r.Register("local.GCStats.LastGCDuration", g)
}
func registerRusageStats(r metrics.Registry) {
for _, metric := range []string{"UserTime", "SystemTime", "MaxRss", "InBlock", "OuBlock"} {
g := metrics.NewGauge()
r.Register("rusage."+metric, g)
}
}
func getGauge(r metrics.Registry, name string) metrics.Gauge {
if v := r.Get(name); v != nil {
return v.(metrics.Gauge)
}
return nil
}
func GoogleCloudMonitoring(r metrics.Registry, d time.Duration, client *http.Client, project string) {
reporter := newReporter(client, project)
for _ = range time.Tick(d) {
r.Each(reporter.report)
}
}
func metricsData(registry metrics.Registry, cfg *config.Config) []*cloudwatch.MetricDatum {
counters, gagues, histos, meters, timers := 0, 0, 0, 0, 0
countersOut, gaguesOut, histosOut, metersOut, timersOut := 0, 0, 0, 0, 0
data := []*cloudwatch.MetricDatum{}
timestamp := aws.Time(time.Now())
aDatum := func(name string) *cloudwatch.MetricDatum {
return &cloudwatch.MetricDatum{
MetricName: aws.String(name),
Timestamp: timestamp,
Dimensions: dimensions(cfg),
}
}
//rough port from the graphite reporter
registry.Each(func(name string, i interface{}) {
switch metric := i.(type) {
case metrics.Counter:
counters += 1
count := float64(metric.Count())
if cfg.Filter.ShouldReport(name, count) {
datum := aDatum(name)
datum.Unit = aws.String(cloudwatch.StandardUnitCount)
datum.Value = aws.Float64(count)
data = append(data, datum)
countersOut += 1
}
if cfg.ResetCountersOnReport {
metric.Clear()
}
case metrics.Gauge:
gagues += 1
value := float64(metric.Value())
if cfg.Filter.ShouldReport(name, value) {
datum := aDatum(name)
datum.Unit = aws.String(cloudwatch.StandardUnitCount)
datum.Value = aws.Float64(float64(value))
data = append(data, datum)
gaguesOut += 1
}
case metrics.GaugeFloat64:
gagues += 1
value := float64(metric.Value())
if cfg.Filter.ShouldReport(name, value) {
datum := aDatum(name)
datum.Unit = aws.String(cloudwatch.StandardUnitCount)
datum.Value = aws.Float64(value)
data = append(data, datum)
gaguesOut += 1
}
case metrics.Histogram:
histos += 1
h := metric.Snapshot()
value := float64(h.Count())
if cfg.Filter.ShouldReport(name, value) {
for _, p := range cfg.Filter.Percentiles(name) {
log.Printf("%+v", h)
pname := fmt.Sprintf("%s-perc%.3f", name, p)
pvalue := h.Percentile(p)
if cfg.Filter.ShouldReport(pname, pvalue) {
datum := aDatum(pname)
datum.Unit = aws.String(cloudwatch.StandardUnitCount)
datum.Value = aws.Float64(pvalue)
data = append(data, datum)
histosOut += 1
}
}
}
case metrics.Meter:
meters += 1
m := metric.Snapshot()
dataz := map[string]float64{
fmt.Sprintf("%s.count", name): float64(m.Count()),
fmt.Sprintf("%s.one-minute", name): m.Rate1(),
fmt.Sprintf("%s.five-minute", name): m.Rate5(),
fmt.Sprintf("%s.fifteen-minute", name): m.Rate15(),
fmt.Sprintf("%s.mean", name): m.RateMean(),
}
for n, v := range dataz {
if cfg.Filter.ShouldReport(n, v) {
datum := aDatum(n)
datum.Value = aws.Float64(v)
data = append(data, datum)
metersOut += 1
}
}
case metrics.Timer:
timers += 1
t := metric.Snapshot()
if t.Count() == 0 {
return
}
dataz := map[string]float64{
fmt.Sprintf("%s.count", name): float64(t.Count()),
fmt.Sprintf("%s.one-minute", name): t.Rate1(),
fmt.Sprintf("%s.five-minute", name): t.Rate5(),
fmt.Sprintf("%s.fifteen-minute", name): t.Rate15(),
fmt.Sprintf("%s.mean", name): t.RateMean(),
}
for n, v := range dataz {
if cfg.Filter.ShouldReport(n, v) {
datum := aDatum(n)
datum.Value = aws.Float64(v)
data = append(data, datum)
timersOut += 1
}
}
for _, p := range cfg.Filter.Percentiles(name) {
pname := fmt.Sprintf("%s-perc%.3f", name, p)
pvalue := t.Percentile(p)
if cfg.Filter.ShouldReport(pname, pvalue) {
datum := aDatum(pname)
datum.Value = aws.Float64(pvalue)
data = append(data, datum)
timersOut += 1
}
}
}
})
total := counters + gagues + histos + meters + timers
totalOut := countersOut + gaguesOut + histosOut + metersOut + timersOut
log.Printf("component=cloudwatch-reporter fn=metricsData at=sources total=%d counters=%d gagues=%d histos=%d meters=%d timers=%d", total, counters, gagues, histos, meters, timers)
log.Printf("component=cloudwatch-reporter fn=metricsData at=targets total=%d counters=%d gagues=%d histos=%d meters=%d timers=%d", totalOut, countersOut, gaguesOut, histosOut, metersOut, timersOut)
return data
}