func (h *histogram) Write(out *dto.Metric) error { his := &dto.Histogram{} buckets := make([]*dto.Bucket, len(h.upperBounds)) his.SampleSum = proto.Float64(math.Float64frombits(atomic.LoadUint64(&h.sumBits))) his.SampleCount = proto.Uint64(atomic.LoadUint64(&h.count)) var count uint64 for i, upperBound := range h.upperBounds { count += atomic.LoadUint64(&h.counts[i]) buckets[i] = &dto.Bucket{ CumulativeCount: proto.Uint64(count), UpperBound: proto.Float64(upperBound), } } his.Bucket = buckets out.Histogram = his out.Label = h.labelPairs return nil }
func (h *constHistogram) Write(out *dto.Metric) error { his := &dto.Histogram{} buckets := make([]*dto.Bucket, 0, len(h.buckets)) his.SampleCount = proto.Uint64(h.count) his.SampleSum = proto.Float64(h.sum) for upperBound, count := range h.buckets { buckets = append(buckets, &dto.Bucket{ CumulativeCount: proto.Uint64(count), UpperBound: proto.Float64(upperBound), }) } if len(buckets) > 0 { sort.Sort(buckSort(buckets)) } his.Bucket = buckets out.Histogram = his out.Label = h.labelPairs return nil }
func (s *summary) Write(out *dto.Metric) error { sum := &dto.Summary{} qs := make([]*dto.Quantile, 0, len(s.objectives)) s.bufMtx.Lock() s.mtx.Lock() // Swap bufs even if hotBuf is empty to set new hotBufExpTime. s.swapBufs(time.Now()) s.bufMtx.Unlock() s.flushColdBuf() sum.SampleCount = proto.Uint64(s.cnt) sum.SampleSum = proto.Float64(s.sum) for _, rank := range s.sortedObjectives { var q float64 if s.headStream.Count() == 0 { q = math.NaN() } else { q = s.headStream.Query(rank) } qs = append(qs, &dto.Quantile{ Quantile: proto.Float64(rank), Value: proto.Float64(q), }) } s.mtx.Unlock() if len(qs) > 0 { sort.Sort(quantSort(qs)) } sum.Quantile = qs out.Summary = sum out.Label = s.labelPairs return nil }
func (s *constSummary) Write(out *dto.Metric) error { sum := &dto.Summary{} qs := make([]*dto.Quantile, 0, len(s.quantiles)) sum.SampleCount = proto.Uint64(s.count) sum.SampleSum = proto.Float64(s.sum) for rank, q := range s.quantiles { qs = append(qs, &dto.Quantile{ Quantile: proto.Float64(rank), Value: proto.Float64(q), }) } if len(qs) > 0 { sort.Sort(quantSort(qs)) } sum.Quantile = qs out.Summary = sum out.Label = s.labelPairs return nil }
// readingValue represents the state where the last byte read (now in // p.currentByte) is the first byte of the sample value (i.e. a float). func (p *Parser) readingValue() stateFn { // When we are here, we have read all the labels, so for the // special case of a summary/histogram, we can finally find out // if the metric already exists. if p.currentMF.GetType() == dto.MetricType_SUMMARY { signature := model.LabelsToSignature(p.currentLabels) if summary := p.summaries[signature]; summary != nil { p.currentMetric = summary } else { p.summaries[signature] = p.currentMetric p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) } } else if p.currentMF.GetType() == dto.MetricType_HISTOGRAM { signature := model.LabelsToSignature(p.currentLabels) if histogram := p.histograms[signature]; histogram != nil { p.currentMetric = histogram } else { p.histograms[signature] = p.currentMetric p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) } } else { p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) } if p.readTokenUntilWhitespace(); p.err != nil { return nil // Unexpected end of input. } value, err := strconv.ParseFloat(p.currentToken.String(), 64) if err != nil { // Create a more helpful error message. p.parseError(fmt.Sprintf("expected float as value, got %q", p.currentToken.String())) return nil } switch p.currentMF.GetType() { case dto.MetricType_COUNTER: p.currentMetric.Counter = &dto.Counter{Value: proto.Float64(value)} case dto.MetricType_GAUGE: p.currentMetric.Gauge = &dto.Gauge{Value: proto.Float64(value)} case dto.MetricType_UNTYPED: p.currentMetric.Untyped = &dto.Untyped{Value: proto.Float64(value)} case dto.MetricType_SUMMARY: // *sigh* if p.currentMetric.Summary == nil { p.currentMetric.Summary = &dto.Summary{} } switch { case p.currentIsSummaryCount: p.currentMetric.Summary.SampleCount = proto.Uint64(uint64(value)) case p.currentIsSummarySum: p.currentMetric.Summary.SampleSum = proto.Float64(value) case !math.IsNaN(p.currentQuantile): p.currentMetric.Summary.Quantile = append( p.currentMetric.Summary.Quantile, &dto.Quantile{ Quantile: proto.Float64(p.currentQuantile), Value: proto.Float64(value), }, ) } case dto.MetricType_HISTOGRAM: // *sigh* if p.currentMetric.Histogram == nil { p.currentMetric.Histogram = &dto.Histogram{} } switch { case p.currentIsHistogramCount: p.currentMetric.Histogram.SampleCount = proto.Uint64(uint64(value)) case p.currentIsHistogramSum: p.currentMetric.Histogram.SampleSum = proto.Float64(value) case !math.IsNaN(p.currentBucket): p.currentMetric.Histogram.Bucket = append( p.currentMetric.Histogram.Bucket, &dto.Bucket{ UpperBound: proto.Float64(p.currentBucket), CumulativeCount: proto.Uint64(uint64(value)), }, ) } default: p.err = fmt.Errorf("unexpected type for metric name %q", p.currentMF.GetName()) } if p.currentByte == '\n' { return p.startOfLine } return p.startTimestamp }
func testParse(t testing.TB) { var scenarios = []struct { in string out []*dto.MetricFamily }{ // 0: Empty lines as input. { in: ` `, out: []*dto.MetricFamily{}, }, // 1: Minimal case. { in: ` minimal_metric 1.234 another_metric -3e3 103948 # Even that: no_labels{} 3 # HELP line for non-existing metric will be ignored. `, out: []*dto.MetricFamily{ &dto.MetricFamily{ Name: proto.String("minimal_metric"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(1.234), }, }, }, }, &dto.MetricFamily{ Name: proto.String("another_metric"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(-3e3), }, TimestampMs: proto.Int64(103948), }, }, }, &dto.MetricFamily{ Name: proto.String("no_labels"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(3), }, }, }, }, }, }, // 2: Counters & gauges, docstrings, various whitespace, escape sequences. { in: ` # A normal comment. # # TYPE name counter name{labelname="val1",basename="basevalue"} NaN name {labelname="val2",basename="base\"v\\al\nue"} 0.23 1234567890 # HELP name two-line\n doc str\\ing # HELP name2 doc str"ing 2 # TYPE name2 gauge name2{labelname="val2" ,basename = "basevalue2" } +Inf 54321 name2{ labelname = "val1" , }-Inf `, out: []*dto.MetricFamily{ &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("two-line\n doc str\\ing"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val1"), }, &dto.LabelPair{ Name: proto.String("basename"), Value: proto.String("basevalue"), }, }, Counter: &dto.Counter{ Value: proto.Float64(math.NaN()), }, }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val2"), }, &dto.LabelPair{ Name: proto.String("basename"), Value: proto.String("base\"v\\al\nue"), }, }, Counter: &dto.Counter{ Value: proto.Float64(.23), }, TimestampMs: proto.Int64(1234567890), }, }, }, &dto.MetricFamily{ Name: proto.String("name2"), Help: proto.String("doc str\"ing 2"), Type: dto.MetricType_GAUGE.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val2"), }, &dto.LabelPair{ Name: proto.String("basename"), Value: proto.String("basevalue2"), }, }, Gauge: &dto.Gauge{ Value: proto.Float64(math.Inf(+1)), }, TimestampMs: proto.Int64(54321), }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val1"), }, }, Gauge: &dto.Gauge{ Value: proto.Float64(math.Inf(-1)), }, }, }, }, }, }, // 3: The evil summary, mixed with other types and funny comments. { in: ` # TYPE my_summary summary my_summary{n1="val1",quantile="0.5"} 110 decoy -1 -2 my_summary{n1="val1",quantile="0.9"} 140 1 my_summary_count{n1="val1"} 42 # Latest timestamp wins in case of a summary. my_summary_sum{n1="val1"} 4711 2 fake_sum{n1="val1"} 2001 # TYPE another_summary summary another_summary_count{n2="val2",n1="val1"} 20 my_summary_count{n2="val2",n1="val1"} 5 5 another_summary{n1="val1",n2="val2",quantile=".3"} -1.2 my_summary_sum{n1="val2"} 08 15 my_summary{n1="val3", quantile="0.2"} 4711 my_summary{n1="val1",n2="val2",quantile="-12.34",} NaN # some # funny comments # HELP # HELP # HELP my_summary # HELP my_summary `, out: []*dto.MetricFamily{ &dto.MetricFamily{ Name: proto.String("fake_sum"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val1"), }, }, Untyped: &dto.Untyped{ Value: proto.Float64(2001), }, }, }, }, &dto.MetricFamily{ Name: proto.String("decoy"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(-1), }, TimestampMs: proto.Int64(-2), }, }, }, &dto.MetricFamily{ Name: proto.String("my_summary"), Type: dto.MetricType_SUMMARY.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val1"), }, }, Summary: &dto.Summary{ SampleCount: proto.Uint64(42), SampleSum: proto.Float64(4711), Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(0.5), Value: proto.Float64(110), }, &dto.Quantile{ Quantile: proto.Float64(0.9), Value: proto.Float64(140), }, }, }, TimestampMs: proto.Int64(2), }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n2"), Value: proto.String("val2"), }, &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val1"), }, }, Summary: &dto.Summary{ SampleCount: proto.Uint64(5), Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(-12.34), Value: proto.Float64(math.NaN()), }, }, }, TimestampMs: proto.Int64(5), }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val2"), }, }, Summary: &dto.Summary{ SampleSum: proto.Float64(8), }, TimestampMs: proto.Int64(15), }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val3"), }, }, Summary: &dto.Summary{ Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(0.2), Value: proto.Float64(4711), }, }, }, }, }, }, &dto.MetricFamily{ Name: proto.String("another_summary"), Type: dto.MetricType_SUMMARY.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n2"), Value: proto.String("val2"), }, &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val1"), }, }, Summary: &dto.Summary{ SampleCount: proto.Uint64(20), Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(0.3), Value: proto.Float64(-1.2), }, }, }, }, }, }, }, }, // 4: The histogram. { in: ` # HELP request_duration_microseconds The response latency. # TYPE request_duration_microseconds histogram request_duration_microseconds_bucket{le="100"} 123 request_duration_microseconds_bucket{le="120"} 412 request_duration_microseconds_bucket{le="144"} 592 request_duration_microseconds_bucket{le="172.8"} 1524 request_duration_microseconds_bucket{le="+Inf"} 2693 request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, out: []*dto.MetricFamily{ { Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ &dto.Bucket{ UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, &dto.Bucket{ UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, &dto.Bucket{ UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, &dto.Bucket{ UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, &dto.Bucket{ UpperBound: proto.Float64(math.Inf(+1)), CumulativeCount: proto.Uint64(2693), }, }, }, }, }, }, }, }, } for i, scenario := range scenarios { out, err := parser.TextToMetricFamilies(strings.NewReader(scenario.in)) if err != nil { t.Errorf("%d. error: %s", i, err) continue } if expected, got := len(scenario.out), len(out); expected != got { t.Errorf( "%d. expected %d MetricFamilies, got %d", i, expected, got, ) } for _, expected := range scenario.out { got, ok := out[expected.GetName()] if !ok { t.Errorf( "%d. expected MetricFamily %q, found none", i, expected.GetName(), ) continue } if expected.String() != got.String() { t.Errorf( "%d. expected MetricFamily %s, got %s", i, expected, got, ) } } } }
func testCreate(t testing.TB) { var scenarios = []struct { in *dto.MetricFamily out string }{ // 0: Counter, NaN as value, timestamp given. { in: &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("two-line\n doc str\\ing"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val1"), }, &dto.LabelPair{ Name: proto.String("basename"), Value: proto.String("basevalue"), }, }, Counter: &dto.Counter{ Value: proto.Float64(math.NaN()), }, }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val2"), }, &dto.LabelPair{ Name: proto.String("basename"), Value: proto.String("basevalue"), }, }, Counter: &dto.Counter{ Value: proto.Float64(.23), }, TimestampMs: proto.Int64(1234567890), }, }, }, out: `# HELP name two-line\n doc str\\ing # TYPE name counter name{labelname="val1",basename="basevalue"} NaN name{labelname="val2",basename="basevalue"} 0.23 1234567890 `, }, // 1: Gauge, some escaping required, +Inf as value, multi-byte characters in label values. { in: &dto.MetricFamily{ Name: proto.String("gauge_name"), Help: proto.String("gauge\ndoc\nstr\"ing"), Type: dto.MetricType_GAUGE.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("name_1"), Value: proto.String("val with\nnew line"), }, &dto.LabelPair{ Name: proto.String("name_2"), Value: proto.String("val with \\backslash and \"quotes\""), }, }, Gauge: &dto.Gauge{ Value: proto.Float64(math.Inf(+1)), }, }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("name_1"), Value: proto.String("Björn"), }, &dto.LabelPair{ Name: proto.String("name_2"), Value: proto.String("佖佥"), }, }, Gauge: &dto.Gauge{ Value: proto.Float64(3.14E42), }, }, }, }, out: `# HELP gauge_name gauge\ndoc\nstr"ing # TYPE gauge_name gauge gauge_name{name_1="val with\nnew line",name_2="val with \\backslash and \"quotes\""} +Inf gauge_name{name_1="Björn",name_2="佖佥"} 3.14e+42 `, }, // 2: Untyped, no help, one sample with no labels and -Inf as value, another sample with one label. { in: &dto.MetricFamily{ Name: proto.String("untyped_name"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(math.Inf(-1)), }, }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("name_1"), Value: proto.String("value 1"), }, }, Untyped: &dto.Untyped{ Value: proto.Float64(-1.23e-45), }, }, }, }, out: `# TYPE untyped_name untyped untyped_name -Inf untyped_name{name_1="value 1"} -1.23e-45 `, }, // 3: Summary. { in: &dto.MetricFamily{ Name: proto.String("summary_name"), Help: proto.String("summary docstring"), Type: dto.MetricType_SUMMARY.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Summary: &dto.Summary{ SampleCount: proto.Uint64(42), SampleSum: proto.Float64(-3.4567), Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(0.5), Value: proto.Float64(-1.23), }, &dto.Quantile{ Quantile: proto.Float64(0.9), Value: proto.Float64(.2342354), }, &dto.Quantile{ Quantile: proto.Float64(0.99), Value: proto.Float64(0), }, }, }, }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("name_1"), Value: proto.String("value 1"), }, &dto.LabelPair{ Name: proto.String("name_2"), Value: proto.String("value 2"), }, }, Summary: &dto.Summary{ SampleCount: proto.Uint64(4711), SampleSum: proto.Float64(2010.1971), Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(0.5), Value: proto.Float64(1), }, &dto.Quantile{ Quantile: proto.Float64(0.9), Value: proto.Float64(2), }, &dto.Quantile{ Quantile: proto.Float64(0.99), Value: proto.Float64(3), }, }, }, }, }, }, out: `# HELP summary_name summary docstring # TYPE summary_name summary summary_name{quantile="0.5"} -1.23 summary_name{quantile="0.9"} 0.2342354 summary_name{quantile="0.99"} 0 summary_name_sum -3.4567 summary_name_count 42 summary_name{name_1="value 1",name_2="value 2",quantile="0.5"} 1 summary_name{name_1="value 1",name_2="value 2",quantile="0.9"} 2 summary_name{name_1="value 1",name_2="value 2",quantile="0.99"} 3 summary_name_sum{name_1="value 1",name_2="value 2"} 2010.1971 summary_name_count{name_1="value 1",name_2="value 2"} 4711 `, }, // 4: Histogram { in: &dto.MetricFamily{ Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ &dto.Bucket{ UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, &dto.Bucket{ UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, &dto.Bucket{ UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, &dto.Bucket{ UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, &dto.Bucket{ UpperBound: proto.Float64(math.Inf(+1)), CumulativeCount: proto.Uint64(2693), }, }, }, }, }, }, out: `# HELP request_duration_microseconds The response latency. # TYPE request_duration_microseconds histogram request_duration_microseconds_bucket{le="100"} 123 request_duration_microseconds_bucket{le="120"} 412 request_duration_microseconds_bucket{le="144"} 592 request_duration_microseconds_bucket{le="172.8"} 1524 request_duration_microseconds_bucket{le="+Inf"} 2693 request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, }, // 5: Histogram with missing +Inf bucket. { in: &dto.MetricFamily{ Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ &dto.Bucket{ UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, &dto.Bucket{ UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, &dto.Bucket{ UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, &dto.Bucket{ UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, }, }, }, }, }, out: `# HELP request_duration_microseconds The response latency. # TYPE request_duration_microseconds histogram request_duration_microseconds_bucket{le="100"} 123 request_duration_microseconds_bucket{le="120"} 412 request_duration_microseconds_bucket{le="144"} 592 request_duration_microseconds_bucket{le="172.8"} 1524 request_duration_microseconds_bucket{le="+Inf"} 2693 request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, }, } for i, scenario := range scenarios { out := bytes.NewBuffer(make([]byte, 0, len(scenario.out))) n, err := MetricFamilyToText(out, scenario.in) if err != nil { t.Errorf("%d. error: %s", i, err) continue } if expected, got := len(scenario.out), n; expected != got { t.Errorf( "%d. expected %d bytes written, got %d", i, expected, got, ) } if expected, got := scenario.out, out.String(); expected != got { t.Errorf( "%d. expected out=%q, got %q", i, expected, got, ) } } }