// buildMetrics creates the actual metrics for the given container.
func (d DockerStats) buildMetrics(container *docker.Container, containerStats *docker.Stats, cpuPercentage float64) []metric.Metric {
ret := []metric.Metric{
buildDockerMetric("DockerMemoryUsed", metric.Gauge, float64(containerStats.MemoryStats.Usage)),
buildDockerMetric("DockerMemoryLimit", metric.Gauge, float64(containerStats.MemoryStats.Limit)),
buildDockerMetric("DockerCpuPercentage", metric.Gauge, cpuPercentage),
buildDockerMetric("DockerCpuThrottledPeriods", metric.CumulativeCounter, float64(containerStats.CPUStats.ThrottlingData.ThrottledPeriods)),
buildDockerMetric("DockerCpuThrottledNanoseconds", metric.CumulativeCounter, float64(containerStats.CPUStats.ThrottlingData.ThrottledTime)),
}
for netiface := range containerStats.Networks {
// legacy format
txb := buildDockerMetric("DockerTxBytes", metric.CumulativeCounter, float64(containerStats.Networks[netiface].TxBytes))
txb.AddDimension("iface", netiface)
ret = append(ret, txb)
rxb := buildDockerMetric("DockerRxBytes", metric.CumulativeCounter, float64(containerStats.Networks[netiface].RxBytes))
rxb.AddDimension("iface", netiface)
ret = append(ret, rxb)
}
additionalDimensions := map[string]string{
"container_id": container.ID,
"container_name": strings.TrimPrefix(container.Name, "/"),
}
metric.AddToAll(&ret, additionalDimensions)
ret = append(ret, buildDockerMetric("DockerContainerCount", metric.Counter, 1))
metric.AddToAll(&ret, d.extractDimensions(container))
return ret
}
// parseUWSGIMetrics11 will parse UWSGI metrics under the assumption of
// the response header containing a Metrics-Schema version 'uwsgi.1.1'.
func parseUWSGIMetrics11(raw *[]byte) ([]metric.Metric, error) {
parsed := new(uwsgiJSONFormat1X)
err := json.Unmarshal(*raw, parsed)
if err != nil {
return []metric.Metric{}, err
}
results := []metric.Metric{}
appendIt := func(metrics []metric.Metric, typeDimVal string) {
metric.AddToAll(&metrics, map[string]string{"type": typeDimVal})
results = append(results, metrics...)
}
appendIt(convertToMetrics(&parsed.Gauges, metric.Gauge), "gauge")
appendIt(convertToMetrics(&parsed.Meters, metric.Gauge), "meter")
appendIt(convertToMetrics(&parsed.Counters, metric.Counter), "counter")
appendIt(convertToMetrics(&parsed.Histograms, metric.Gauge), "histogram")
appendIt(convertToMetrics(&parsed.Timers, metric.Gauge), "timer")
// This is necessary as Go doesn't allow us to type assert
// map[string]interface{} as map[string]string.
// Basically go doesn't allow type assertions for interface{}'s nested
// inside data structures across the entire structure since it is a linearly
// complex action
for k, v := range parsed.ServiceDims {
metric.AddToAll(&results, map[string]string{k: v.(string)})
}
return results, nil
}
// buildMetrics creates the actual metrics for the given container.
func (d DockerStats) buildMetrics(container *docker.Container, memUsed, memLimit, cpuPercentage float64) []metric.Metric {
ret := []metric.Metric{
buildDockerMetric("DockerMemoryUsed", memUsed),
buildDockerMetric("DockerMemoryLimit", memLimit),
buildDockerMetric("DockerCpuPercentage", cpuPercentage),
}
additionalDimensions := map[string]string{
"container_id": container.ID,
"container_name": strings.TrimPrefix(container.Name, "/"),
}
metric.AddToAll(&ret, additionalDimensions)
metric.AddToAll(&ret, getServiceDimensions(container))
return ret
}
// parseUWSGIMetrics takes the json returned from the endpoint and converts
// it into raw metrics. We first check that the metrics returned have a float value
// otherwise we skip the metric.
func parseUWSGIMetrics(raw *[]byte) ([]metric.Metric, error) {
parsed := new(uwsgiJSONFormat)
err := json.Unmarshal(*raw, parsed)
if err != nil {
return []metric.Metric{}, err
}
results := []metric.Metric{}
appendIt := func(metrics []metric.Metric, typeDimVal string) {
metric.AddToAll(&metrics, map[string]string{"type": typeDimVal})
results = append(results, metrics...)
}
appendIt(convertToMetrics(&parsed.Gauges, metric.Gauge), "gauge")
appendIt(convertToMetrics(&parsed.Meters, metric.Gauge), "meter")
appendIt(convertToMetrics(&parsed.Counters, metric.Counter), "counter")
appendIt(convertToMetrics(&parsed.Histograms, metric.Gauge), "histogram")
appendIt(convertToMetrics(&parsed.Timers, metric.Gauge), "timer")
if len(results) == 0 {
// If parsing using UWSGI format did not work, the output is probably
// in Dropwizard format and should be handled as such.
return parseDropwizardMetric(raw)
}
return results, nil
}
func (n *nerveUWSGICollector) queryService(serviceName string, port int) {
serviceLog := n.log.WithField("service", serviceName)
endpoint := fmt.Sprintf("http://localhost:%d/%s", port, n.queryPath)
serviceLog.Debug("making GET request to ", endpoint)
rawResponse, err := queryEndpoint(endpoint, n.timeout)
if err != nil {
serviceLog.Warn("Failed to query endpoint ", endpoint, ": ", err)
return
}
metrics, err := parseUWSGIMetrics(&rawResponse)
if err != nil {
serviceLog.Warn("Failed to parse response into metrics: ", err)
return
}
metric.AddToAll(&metrics, map[string]string{
"collector": n.Name(),
"service": serviceName,
"port": strconv.Itoa(port),
})
serviceLog.Debug("Sending ", len(metrics), " to channel")
for _, m := range metrics {
n.Channel() <- m
}
}
func (ps ProcStatus) getMetrics(proc procfs.Proc, cmdOutput []string) []metric.Metric {
stat, err := proc.NewStat()
if err != nil {
ps.log.Warn("Error getting stats: ", err)
return nil
}
pid := strconv.Itoa(stat.PID)
dim := map[string]string{
"processName": stat.Comm,
"pid": pid,
}
ret := []metric.Metric{
procStatusPoint("VirtualMemory", float64(stat.VirtualMemory()), dim),
procStatusPoint("ResidentMemory", float64(stat.ResidentMemory()), dim),
procStatusPoint("CPUTime", float64(stat.CPUTime()), dim),
}
if len(cmdOutput) > 0 {
generatedDimensions := ps.extractDimensions(cmdOutput[0])
metric.AddToAll(&ret, generatedDimensions)
}
return ret
}
// parseMetrics parse all the metrics according to the schemaVer.
// Always it tries to parse the metric with a base JSON format
// if it doesnt succed it will try with different parsing functions
func parseMetrics(raw *[]byte, schemaVer string, cumulCounterEnabled bool) ([]metric.Metric, error) {
parsed := new(uwsgiJSONFormat1X)
err := json.Unmarshal(*raw, &parsed)
if err != nil {
return []metric.Metric{}, err
}
results, err := parseUWSGIMetrics(parsed, cumulCounterEnabled)
if err != nil {
return results, err
}
if schemaVer == "uwsgi.1.1" {
// This is necessary as Go doesn't allow us to type assert
// map[string]interface{} as map[string]string.
// Basically go doesn't allow type assertions for interface{}'s nested
// inside data structures across the entire structure since it is a linearly
// complex action
for k, v := range parsed.ServiceDims {
metric.AddToAll(&results, map[string]string{k: v.(string)})
}
} else if schemaVer == "default" && len(results) == 0 {
return parseDropwizardMetrics(raw)
}
return results, nil
}
// buildMetrics creates the actual metrics for the given container.
func (d DockerStats) buildMetrics(container *docker.Container, containerStats *docker.Stats, cpuPercentage float64) []metric.Metric {
ret := []metric.Metric{
buildDockerMetric("DockerRxBytes", metric.CumulativeCounter, float64(containerStats.Network.RxBytes)),
buildDockerMetric("DockerTxBytes", metric.CumulativeCounter, float64(containerStats.Network.TxBytes)),
buildDockerMetric("DockerMemoryUsed", metric.Gauge, float64(containerStats.MemoryStats.Usage)),
buildDockerMetric("DockerMemoryLimit", metric.Gauge, float64(containerStats.MemoryStats.Limit)),
buildDockerMetric("DockerCpuPercentage", metric.Gauge, cpuPercentage),
}
additionalDimensions := map[string]string{
"container_id": container.ID,
"container_name": strings.TrimPrefix(container.Name, "/"),
}
metric.AddToAll(&ret, additionalDimensions)
metric.AddToAll(&ret, d.extractDimensions(container))
return ret
}
// parseUWSGIMetrics takes the json returned from the endpoint and converts
// it into raw metrics. We first check that the metrics returned have a float value
// otherwise we skip the metric.
//
// @cumulCounterEnabled: if true it enables to create meter and timer counters as
// Cumultative Counters instead of Gauges
func parseUWSGIMetrics(parsed *uwsgiJSONFormat1X, cumulCounterEnabled bool) ([]metric.Metric, error) {
results := []metric.Metric{}
appendIt := func(metrics []metric.Metric, typeDimVal string) {
metric.AddToAll(&metrics, map[string]string{"type": typeDimVal})
results = append(results, metrics...)
}
appendIt(convertToMetrics(&parsed.Gauges, metric.Gauge, false), "gauge")
appendIt(convertToMetrics(&parsed.Counters, metric.Counter, false), "counter")
appendIt(convertToMetrics(&parsed.Histograms, metric.Gauge, false), "histogram")
appendIt(convertToMetrics(&parsed.Meters, metric.Gauge, cumulCounterEnabled), "meter")
appendIt(convertToMetrics(&parsed.Timers, metric.Gauge, cumulCounterEnabled), "timer")
return results, nil
}
func extractParsedMetric(parser Parser, parsed *Format) []metric.Metric {
results := []metric.Metric{}
appendIt := func(metrics []metric.Metric, typeDimVal string) {
if !parser.isCCEnabled() {
metric.AddToAll(&metrics, map[string]string{"type": typeDimVal})
}
results = append(results, metrics...)
}
appendIt(parser.parseMapOfMap(parsed.Gauges, metric.Gauge), "gauge")
appendIt(parser.parseMapOfMap(parsed.Counters, metric.Counter), "counter")
appendIt(parser.parseMapOfMap(parsed.Histograms, metric.Gauge), "histogram")
appendIt(parser.parseMapOfMap(parsed.Meters, metric.Gauge), "meter")
appendIt(parser.parseMapOfMap(parsed.Timers, metric.Gauge), "timer")
return results
}
// parseUWSGIMetrics11 will parse UWSGI metrics under the assumption of
// the response header containing a Metrics-Schema version 'uwsgi.1.1'.
func (parser *UWSGIMetric) parseUWSGIMetrics11() ([]metric.Metric, error) {
parsed := new(Format)
err := json.Unmarshal(parser.data, parsed)
if err != nil {
return []metric.Metric{}, err
}
results := extractParsedMetric(parser, parsed)
// This is necessary as Go doesn't allow us to type assert
// map[string]interface{} as map[string]string.
// Basically go doesn't allow type assertions for interface{}'s nested
// inside data structures across the entire structure since it is a linearly
// complex action
for k, v := range parsed.ServiceDims {
metric.AddToAll(&results, map[string]string{k: v.(string)})
}
return results, nil
}
// metricFromMap takes in flattened maps formatted like this::
// {
// "count": 3443,
// "mean_rate": 100
// }
// and metricname and metrictype and returns metrics for each name:rollup pair
func (parser *BaseParser) metricFromMap(metricMap map[string]interface{},
metricName string,
metricType string) []metric.Metric {
results := []metric.Metric{}
dims := make(map[string]string)
for rollup, value := range metricMap {
// First check for dimension set if present
// See uwsgi_metric.go:68 for explanation on the range over value
if rollup == "dimensions" {
for dimName, dimVal := range value.(map[string]interface{}) {
dims[dimName] = dimVal.(string)
}
continue
}
mName := metricName
mType := metricType
matched, _ := regexp.MatchString("m[0-9]+_rate", rollup)
// If cumulCounterEnabled is true:
// 1. change metric type meter.count and timer.count moving them to cumulative counter
// 2. don't send back metered metrics (rollup == 'mXX_rate')
if parser.ccEnabled && matched {
continue
}
if parser.ccEnabled && rollup != "value" {
mName = metricName + "." + rollup
if rollup == "count" {
mType = metric.CumulativeCounter
}
}
tempMetric, ok := parser.createMetricFromDatam(rollup, value, mName, mType)
if ok {
results = append(results, tempMetric)
}
}
metric.AddToAll(&results, dims)
return results
}
func (c *NerveHTTPD) getMetrics(serviceName string, port int) []metric.Metric {
results := []metric.Metric{}
serviceLog := c.log.WithField("service", serviceName)
endpoint := fmt.Sprintf("http://%s:%d/%s", c.host, port, c.queryPath)
serviceLog.Debug("making GET request to ", endpoint)
httpResponse := fetchApacheMetrics(endpoint, port)
if httpResponse.status != 200 {
c.updateFailedStatus(serviceName, port, httpResponse.status)
serviceLog.Warn("Failed to query endpoint ", endpoint, ": ", httpResponse.err)
return results
}
apacheMetrics := extractApacheMetrics(httpResponse.data)
metric.AddToAll(&apacheMetrics, map[string]string{
"service": serviceName,
"port": strconv.Itoa(port),
})
return apacheMetrics
}
// parseUWSGIMetrics10 takes the json returned from the endpoint and converts
// it into raw metrics. We first check that the metrics returned have a float value
// otherwise we skip the metric.
func parseUWSGIMetrics10(raw *[]byte) ([]metric.Metric, error) {
parsed := new(uwsgiJSONFormat1X)
err := json.Unmarshal(*raw, parsed)
if err != nil {
return []metric.Metric{}, err
}
results := []metric.Metric{}
appendIt := func(metrics []metric.Metric, typeDimVal string) {
metric.AddToAll(&metrics, map[string]string{"type": typeDimVal})
results = append(results, metrics...)
}
appendIt(convertToMetrics(&parsed.Gauges, metric.Gauge), "gauge")
appendIt(convertToMetrics(&parsed.Meters, metric.Gauge), "meter")
appendIt(convertToMetrics(&parsed.Counters, metric.Counter), "counter")
appendIt(convertToMetrics(&parsed.Histograms, metric.Gauge), "histogram")
appendIt(convertToMetrics(&parsed.Timers, metric.Gauge), "timer")
return results, nil
}