func NewGoInternalMetrics(prefix string) *GoInternalMetrics {
goint := &GoInternalMetrics{
Alloc: gmetrics.NewGauge(),
TotalAlloc: gmetrics.NewGauge(),
NumGoroutine: gmetrics.NewGauge(),
}
gmetrics.Register(prefix+".goint.alloc", goint.Alloc)
gmetrics.Register(prefix+".goint.total_alloc", goint.TotalAlloc)
gmetrics.Register(prefix+".goint.go_routines", goint.NumGoroutine)
return goint
}
func NewProcessMemoryMetrics(prefix string) *ProcessMemoryMetrics {
mem := &ProcessMemoryMetrics{
Resident: gmetrics.NewGauge(),
Shared: gmetrics.NewGauge(),
PageFaults: gmetrics.NewMeter(),
}
gmetrics.Register(prefix+".mem.resident", mem.Resident)
gmetrics.Register(prefix+".mem.shared", mem.Shared)
gmetrics.Register(prefix+".mem.pagefaults", mem.PageFaults)
return mem
}
func NewHttp() *Http {
return &Http{
Txns: util.NewBroadcast(),
reqGauge: metrics.NewGauge(),
reqMeter: metrics.NewMeter(),
reqTimer: metrics.NewTimer(),
}
}
func NewClientMetrics() *ClientMetrics {
return &ClientMetrics{
connGauge: metrics.NewGauge(),
connMeter: metrics.NewMeter(),
connTimer: metrics.NewTimer(),
proxySetupTimer: metrics.NewTimer(),
bytesIn: metrics.NewHistogram(metrics.NewExpDecaySample(sampleSize, sampleAlpha)),
bytesOut: metrics.NewHistogram(metrics.NewExpDecaySample(sampleSize, sampleAlpha)),
bytesInCount: metrics.NewCounter(),
bytesOutCount: metrics.NewCounter(),
}
}
func (smb standardBucket) Gauge(name string) Gauge {
smb.Lock()
defer smb.Unlock()
gauge, ok := smb.gauges[name]
if !ok {
gauge = gometrics.NewGauge()
smb.gauges[name] = gauge
smb.registry.Register(name, gauge)
}
return gauge
}
// initializeFieldTagPath traverses the given struct trying to initialize
// metric values. The "metric" struct tag is used to determine the name of the
// metrics for each struct field. If there is no "metric" struct tag, the
// lowercased struct field name is used for the metric name. The name is
// prefixed with tags from previous struct fields if any, separated by a dot.
// For example:
//
// Messages struct {
// Smtp struct {
// Latency metrics.Timer `metric:"latency"`
// } `metric:"smtp"`
// Http struct {
// Latency metrics.Timer `metric:"latency"`
// } `metric:"http"`
// } `metric:"messages"`
//
// yields timers with names "messages.smtp.latency" and "messages.http.latency"
// respectively.
//
// If there is no metric tag for a field it is skipped and assumed it is used
// for other purposes such as configuration.
func (m *MetricTags) initializeFieldTagPath(fieldType reflect.Value, prefix string) {
for i := 0; i < fieldType.NumField(); i++ {
val := fieldType.Field(i)
field := fieldType.Type().Field(i)
tag := field.Tag.Get("metric")
if tag == "" {
// If tag isn't found, derive tag from the lower case name of
// the field.
tag = strings.ToLower(field.Name)
}
if prefix != "" {
tag = prefix + m.separator + tag
}
if field.Type.Kind() == reflect.Struct {
// Recursively traverse an embedded struct
m.initializeFieldTagPath(val, tag)
} else if field.Type.Kind() == reflect.Map && field.Type.Key().Kind() == reflect.String {
// If this is a map[string]Something, then use the string key as bucket name and recursively generate the metrics below
for _, k := range val.MapKeys() {
m.initializeFieldTagPath(val.MapIndex(k).Elem(), tag+m.separator+k.String())
}
} else {
// Found a field, initialize
switch field.Type.String() {
case "metrics.Counter":
c := metrics.NewCounter()
metrics.Register(tag, c)
val.Set(reflect.ValueOf(c))
case "metrics.Timer":
t := metrics.NewTimer()
metrics.Register(tag, t)
val.Set(reflect.ValueOf(t))
case "metrics.Meter":
m := metrics.NewMeter()
metrics.Register(tag, m)
val.Set(reflect.ValueOf(m))
case "metrics.Gauge":
g := metrics.NewGauge()
metrics.Register(tag, g)
val.Set(reflect.ValueOf(g))
case "metrics.Histogram":
s := metrics.NewUniformSample(1028)
h := metrics.NewHistogram(s)
metrics.Register(tag, h)
val.Set(reflect.ValueOf(h))
}
}
}
}
// SaveGauge indicates that we want to store an internal representation of the value of this gauge so that we can query
// it (via GetGauge).
func (i *Instrumentation) SaveGauge(bucket string) error {
i.mtx.Lock()
defer i.mtx.Unlock()
if _, exists := i.savedGauges[bucket]; exists {
return fmt.Errorf("Gauge bucket %s already registered", bucket)
}
g := metrics.NewGauge()
i.registry.Register(bucket, g)
i.savedGauges[bucket] = g
return nil
}
func (cw *ClusterWatcher) Watch(stop chan interface{}) error {
if cw.DataDogAPIKey != "" {
cw.errorsGauge = metrics.NewGauge()
cw.startedGauge = metrics.NewGauge()
cw.passivatedGauge = metrics.NewGauge()
cw.warningsGauge = metrics.NewGauge()
metrics.Register("arken.environments.stats.errors", cw.errorsGauge)
metrics.Register("arken.environments.stats.started", cw.startedGauge)
metrics.Register("arken.environments.stats.passivated", cw.passivatedGauge)
metrics.Register("arken.environments.stats.warning", cw.warningsGauge)
host, _ := os.Hostname()
cw.dog = datadog.New(host, cw.DataDogAPIKey)
go cw.dog.DefaultReporter().Start(30 * time.Second)
}
go cw.updateMetrics()
return cw.watchServiceKeys(stop)
}
func NewCachingProxy(target Filter, size int) *CachingProxy {
_, ok := target.(CacheKey)
if !ok {
log.Fatalf("%s should implement CacheKey interface\n", target)
return nil
}
total := metrics.NewCounter()
hit := metrics.NewCounter()
cached := metrics.NewGauge()
targetName := fmt.Sprintf("%s", target)
metrics.Register("CachingProxy("+targetName+")-Total", total)
metrics.Register("CachingProxy("+targetName+")-Hit", hit)
metrics.Register("CachingProxy("+targetName+")-Cached", cached)
return &CachingProxy{target, &sync.RWMutex{}, list.New(), size, total, hit, cached}
}
// conf.Environment, conf.Region
func NewProxyMetrics(env string, region string) ProxyMetrics {
prefix := buildPrefix(env, region)
pm := ProxyMetrics{
Msgs: gmetrics.NewMeter(),
MsgReply: gmetrics.NewMeter(),
MsgForward: gmetrics.NewMeter(),
MsgBodySize: gmetrics.NewHistogram(gmetrics.NewExpDecaySample(1028, 0.015)),
Connects: gmetrics.NewMeter(),
Connections: gmetrics.NewGauge(),
}
gmetrics.Register(prefix+".proxy.msgs", pm.Msgs)
gmetrics.Register(prefix+".proxy.msg_reply", pm.Msgs)
gmetrics.Register(prefix+".proxy.msg_forward", pm.Msgs)
gmetrics.Register(prefix+".proxy.msg_body_size", pm.MsgBodySize)
gmetrics.Register(prefix+".proxy.connects", pm.Connects)
gmetrics.Register(prefix+".proxy.connections", pm.Connections)
return pm
}
func addRuntimeMetrics(p *newrelic.Plugin) {
p.AddMetric(newrelic.NewMetric("Runtime/Events/Goroutines", "goroutines", func() (float64, error) { return float64(runtime.NumGoroutine()), nil }))
var lastNumCgoCall int64
p.AddMetric(newrelic.NewMetric("Runtime/Events/CGO Calls", "calls", func() (float64, error) {
currentNumCgoCall := runtime.NumCgoCall()
result := float64(currentNumCgoCall - lastNumCgoCall)
lastNumCgoCall = currentNumCgoCall
return result, nil
}))
threads := metrics.NewGauge()
fdsize := metrics.NewGauge()
vmpeak := metrics.NewGauge()
vmsize := metrics.NewGauge()
rsspeak := metrics.NewGauge()
rsssize := metrics.NewGauge()
pr := &procReader{
sampleRate: time.Second * 5,
procMap: make(map[string]string),
threads: threads,
fdsize: fdsize,
vmpeak: vmpeak,
vmsize: vmsize,
rsspeak: rsspeak,
rsssize: rsssize,
}
pr.Run()
nrmetrics.AddGaugeMetric(p, threads, nrmetrics.MetricConfig{Name: "Runtime/Threads", Unit: "threads", Count: true})
nrmetrics.AddGaugeMetric(p, fdsize, nrmetrics.MetricConfig{Name: "Runtime/FD Size", Unit: "file descriptor slots", Count: true})
nrmetrics.AddGaugeMetric(p, vmpeak, nrmetrics.MetricConfig{Name: "Runtime/Peak Virt Mem", Unit: "bytes", Count: true})
nrmetrics.AddGaugeMetric(p, vmsize, nrmetrics.MetricConfig{Name: "Runtime/Virt Mem", Unit: "bytes", Count: true})
nrmetrics.AddGaugeMetric(p, rsspeak, nrmetrics.MetricConfig{Name: "Runtime/RSS Peak", Unit: "bytes", Count: true})
nrmetrics.AddGaugeMetric(p, rsssize, nrmetrics.MetricConfig{Name: "Runtime/RSS Size", Unit: "bytes", Count: true})
}
func addMemoryMetrics(p *newrelic.Plugin) {
allocInUse := metrics.NewGauge()
allocTotal := metrics.NewGauge()
sys := metrics.NewGauge()
pointerLookups := metrics.NewMeter()
mallocs := metrics.NewMeter()
frees := metrics.NewMeter()
allocHeap := metrics.NewGauge()
heapSys := metrics.NewGauge()
heapIdle := metrics.NewGauge()
heapInUse := metrics.NewGauge()
heapReleased := metrics.NewGauge()
heapObjects := metrics.NewGauge()
stackInUse := metrics.NewGauge()
stackSys := metrics.NewGauge()
mspanInUse := metrics.NewGauge()
mspanSys := metrics.NewGauge()
mcacheInUse := metrics.NewGauge()
mcacheSys := metrics.NewGauge()
buckHashSys := metrics.NewGauge()
gcSys := metrics.NewGauge()
otherSys := metrics.NewGauge()
mr := &memReader{
sampleRate: time.Minute,
allocInUse: allocInUse,
allocTotal: allocTotal,
pointerLookups: pointerLookups,
mallocs: mallocs,
frees: frees,
allocHeap: allocHeap,
heapSys: heapSys,
heapIdle: heapIdle,
heapInUse: heapInUse,
heapReleased: heapReleased,
heapObjects: heapObjects,
stackInUse: stackInUse,
stackSys: stackSys,
mspanInUse: mspanInUse,
mspanSys: mspanSys,
mcacheInUse: mcacheInUse,
mcacheSys: mcacheSys,
buckHashSys: buckHashSys,
gcSys: gcSys,
otherSys: otherSys,
}
mr.Run()
nrmetrics.AddGaugeMetric(p, allocTotal, nrmetrics.MetricConfig{Name: "Memory/Summary/Allocated Over Time", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, sys, nrmetrics.MetricConfig{Name: "Memory/Summary/System/Total", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, heapSys, nrmetrics.MetricConfig{Name: "Memory/System/Heap", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, stackSys, nrmetrics.MetricConfig{Name: "Memory/System/Stack", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, mspanSys, nrmetrics.MetricConfig{Name: "Memory/System/MSpan", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, mcacheSys, nrmetrics.MetricConfig{Name: "Memory/System/MCache", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, buckHashSys, nrmetrics.MetricConfig{Name: "Memory/System/BuckHash", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, gcSys, nrmetrics.MetricConfig{Name: "Memory/System/GC", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, otherSys, nrmetrics.MetricConfig{Name: "Memory/System/Other", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, allocInUse, nrmetrics.MetricConfig{Name: "Memory/Summary/In Use/Total", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, heapInUse, nrmetrics.MetricConfig{Name: "Memory/In Use/Heap", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, stackInUse, nrmetrics.MetricConfig{Name: "Memory/In Use/Stack", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, mspanInUse, nrmetrics.MetricConfig{Name: "Memory/In Use/MSpan", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, mcacheInUse, nrmetrics.MetricConfig{Name: "Memory/In Use/MCache", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, allocHeap, nrmetrics.MetricConfig{Name: "Memory/Heap/Allocated", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, heapIdle, nrmetrics.MetricConfig{Name: "Memory/Heap/Idle", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, heapInUse, nrmetrics.MetricConfig{Name: "Memory/Heap/In Use", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, heapReleased, nrmetrics.MetricConfig{Name: "Memory/Heap/Released", Unit: "bytes", Value: true})
nrmetrics.AddGaugeMetric(p, heapObjects, nrmetrics.MetricConfig{Name: "Memory/Events/Allocated Heap Objects", Unit: "objects", Value: true})
nrmetrics.AddMeterMetric(p, pointerLookups, nrmetrics.MetricConfig{
Name: "Memory/Events/Pointer Lookups",
Unit: "lookups",
Rate1: true,
Rate5: true,
Rate15: true,
})
nrmetrics.AddMeterMetric(p, mallocs, nrmetrics.MetricConfig{
Name: "Memory/Events/Mallocs",
Unit: "mallocs",
Rate1: true,
Rate5: true,
Rate15: true,
})
nrmetrics.AddMeterMetric(p, frees, nrmetrics.MetricConfig{
Name: "Memory/Events/Frees",
Unit: "frees",
Rate1: true,
Rate5: true,
Rate15: true,
})
}
func registerGCStats(r metrics.Registry) {
g := metrics.NewGauge()
r.Register("local.GCStats.LastGCDuration", g)
}
func registerRusageStats(r metrics.Registry) {
for _, metric := range []string{"UserTime", "SystemTime", "MaxRss", "InBlock", "OuBlock"} {
g := metrics.NewGauge()
r.Register("rusage."+metric, g)
}
}
func (m *StreamingMetrics) Register(op string) {
m.OpGroups[op] = MetricsGroup{metrics.NewCounter(), metrics.NewCounter(), metrics.NewGauge()}
}
