func fetchMetricFamilies(url string, ch chan<- *dto.MetricFamily) {
defer close(ch)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalf("creating GET request for URL %q failed: %s", url, err)
}
req.Header.Add("Accept", acceptHeader)
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatalf("executing GET request for URL %q failed: %s", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Fatalf("GET request for URL %q returned HTTP status %s", url, resp.Status)
}
mediatype, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
if err == nil && mediatype == "application/vnd.google.protobuf" &&
params["encoding"] == "delimited" &&
params["proto"] == "io.prometheus.client.MetricFamily" {
for {
mf := &dto.MetricFamily{}
if _, err = pbutil.ReadDelimited(resp.Body, mf); err != nil {
if err == io.EOF {
break
}
log.Fatalln("reading metric family protocol buffer failed:", err)
}
ch <- mf
}
} else {
// We could do further content-type checks here, but the
// fallback for now will anyway be the text format
// version 0.0.4, so just go for it and see if it works.
var parser expfmt.TextParser
metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
if err != nil {
log.Fatalln("reading text format failed:", err)
}
for _, mf := range metricFamilies {
ch <- mf
}
}
}
func (c *textFileCollector) parseTextFiles() []*dto.MetricFamily {
error := 0.0
var metricFamilies []*dto.MetricFamily
mtimes := map[string]time.Time{}
// Iterate over files and accumulate their metrics.
files, err := ioutil.ReadDir(c.path)
if err != nil && c.path != "" {
log.Errorf("Error reading textfile collector directory %s: %s", c.path, err)
error = 1.0
}
for _, f := range files {
if !strings.HasSuffix(f.Name(), ".prom") {
continue
}
path := filepath.Join(c.path, f.Name())
file, err := os.Open(path)
if err != nil {
log.Errorf("Error opening %s: %v", path, err)
error = 1.0
continue
}
var parser expfmt.TextParser
parsedFamilies, err := parser.TextToMetricFamilies(file)
if err != nil {
log.Errorf("Error parsing %s: %v", path, err)
error = 1.0
continue
}
// Only set this once it has been parsed, so that
// a failure does not appear fresh.
mtimes[f.Name()] = f.ModTime()
for _, mf := range parsedFamilies {
if mf.Help == nil {
help := fmt.Sprintf("Metric read from %s", path)
mf.Help = &help
}
metricFamilies = append(metricFamilies, mf)
}
}
// Export the mtimes of the successful files.
if len(mtimes) > 0 {
mtimeMetricFamily := dto.MetricFamily{
Name: proto.String("node_textfile_mtime"),
Help: proto.String("Unixtime mtime of textfiles successfully read."),
Type: dto.MetricType_GAUGE.Enum(),
Metric: []*dto.Metric{},
}
// Sorting is needed for predictable output comparison in tests.
filenames := make([]string, 0, len(mtimes))
for filename := range mtimes {
filenames = append(filenames, filename)
}
sort.Strings(filenames)
for _, filename := range filenames {
mtimeMetricFamily.Metric = append(mtimeMetricFamily.Metric,
&dto.Metric{
Label: []*dto.LabelPair{
&dto.LabelPair{
Name: proto.String("file"),
Value: proto.String(filename),
},
},
Gauge: &dto.Gauge{Value: proto.Float64(float64(mtimes[filename].UnixNano()) / 1e9)},
},
)
}
metricFamilies = append(metricFamilies, &mtimeMetricFamily)
}
// Export if there were errors.
metricFamilies = append(metricFamilies, &dto.MetricFamily{
Name: proto.String("node_textfile_scrape_error"),
Help: proto.String("1 if there was an error opening or reading a file, 0 otherwise"),
Type: dto.MetricType_GAUGE.Enum(),
Metric: []*dto.Metric{
&dto.Metric{
Gauge: &dto.Gauge{Value: &error},
},
},
})
return metricFamilies
}
// Parse returns a slice of Metrics from a text representation of a
// metrics
func (p *PrometheusParser) Parse(buf []byte) ([]telegraf.Metric, error) {
var metrics []telegraf.Metric
var parser expfmt.TextParser
// parse even if the buffer begins with a newline
buf = bytes.TrimPrefix(buf, []byte("\n"))
// Read raw data
buffer := bytes.NewBuffer(buf)
reader := bufio.NewReader(buffer)
// Get format
mediatype, params, err := mime.ParseMediaType(p.PromFormat["Content-Type"])
// Prepare output
metricFamilies := make(map[string]*dto.MetricFamily)
if err == nil && mediatype == "application/vnd.google.protobuf" &&
params["encoding"] == "delimited" &&
params["proto"] == "io.prometheus.client.MetricFamily" {
for {
metricFamily := &dto.MetricFamily{}
if _, err = pbutil.ReadDelimited(reader, metricFamily); err != nil {
if err == io.EOF {
break
}
return nil, fmt.Errorf("reading metric family protocol buffer failed: %s", err)
}
metricFamilies[metricFamily.GetName()] = metricFamily
}
} else {
metricFamilies, err = parser.TextToMetricFamilies(reader)
if err != nil {
return nil, fmt.Errorf("reading text format failed: %s", err)
}
// read metrics
for metricName, mf := range metricFamilies {
for _, m := range mf.Metric {
// reading tags
tags := makeLabels(m)
/*
for key, value := range p.DefaultTags {
tags[key] = value
}
*/
// reading fields
fields := make(map[string]interface{})
if mf.GetType() == dto.MetricType_SUMMARY {
// summary metric
fields = makeQuantiles(m)
fields["count"] = float64(m.GetSummary().GetSampleCount())
fields["sum"] = float64(m.GetSummary().GetSampleSum())
} else if mf.GetType() == dto.MetricType_HISTOGRAM {
// historgram metric
fields = makeBuckets(m)
fields["count"] = float64(m.GetHistogram().GetSampleCount())
fields["sum"] = float64(m.GetHistogram().GetSampleSum())
} else {
// standard metric
fields = getNameAndValue(m)
}
// converting to telegraf metric
if len(fields) > 0 {
var t time.Time
if m.TimestampMs != nil && *m.TimestampMs > 0 {
t = time.Unix(0, *m.TimestampMs*1000000)
} else {
t = time.Now()
}
metric, err := telegraf.NewMetric(metricName, tags, fields, t)
if err == nil {
metrics = append(metrics, metric)
}
}
}
}
}
return metrics, err
}
// Push returns an http.Handler which accepts samples over HTTP and stores them
// in the MetricStore. If replace is true, all metrics for the job and instance
// given by the request are deleted before new ones are stored.
//
// The returned handler is already instrumented for Prometheus.
func Push(
ms storage.MetricStore, replace bool,
) func(http.ResponseWriter, *http.Request, httprouter.Params) {
var ps httprouter.Params
var mtx sync.Mutex // Protects ps.
instrumentedHandlerFunc := prometheus.InstrumentHandlerFunc(
"push",
func(w http.ResponseWriter, r *http.Request) {
job := ps.ByName("job")
labelsString := ps.ByName("labels")
mtx.Unlock()
labels, err := splitLabels(labelsString)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if job == "" {
http.Error(w, "job name is required", http.StatusBadRequest)
return
}
labels["job"] = job
if replace {
ms.SubmitWriteRequest(storage.WriteRequest{
Labels: labels,
Timestamp: time.Now(),
})
}
var metricFamilies map[string]*dto.MetricFamily
ctMediatype, ctParams, ctErr := mime.ParseMediaType(r.Header.Get("Content-Type"))
if ctErr == nil && ctMediatype == "application/vnd.google.protobuf" &&
ctParams["encoding"] == "delimited" &&
ctParams["proto"] == "io.prometheus.client.MetricFamily" {
metricFamilies = map[string]*dto.MetricFamily{}
for {
mf := &dto.MetricFamily{}
if _, err = pbutil.ReadDelimited(r.Body, mf); err != nil {
if err == io.EOF {
err = nil
}
break
}
metricFamilies[mf.GetName()] = mf
}
} else {
// We could do further content-type checks here, but the
// fallback for now will anyway be the text format
// version 0.0.4, so just go for it and see if it works.
var parser expfmt.TextParser
metricFamilies, err = parser.TextToMetricFamilies(r.Body)
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
sanitizeLabels(metricFamilies, labels)
ms.SubmitWriteRequest(storage.WriteRequest{
Labels: labels,
Timestamp: time.Now(),
MetricFamilies: metricFamilies,
})
w.WriteHeader(http.StatusAccepted)
},
)
return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
mtx.Lock()
ps = params
instrumentedHandlerFunc(w, r)
}
}
func ExampleGatherers() {
reg := prometheus.NewRegistry()
temp := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "temperature_kelvin",
Help: "Temperature in Kelvin.",
},
[]string{"location"},
)
reg.MustRegister(temp)
temp.WithLabelValues("outside").Set(273.14)
temp.WithLabelValues("inside").Set(298.44)
var parser expfmt.TextParser
text := `
# TYPE humidity_percent gauge
# HELP humidity_percent Humidity in %.
humidity_percent{location="outside"} 45.4
humidity_percent{location="inside"} 33.2
# TYPE temperature_kelvin gauge
# HELP temperature_kelvin Temperature in Kelvin.
temperature_kelvin{location="somewhere else"} 4.5
`
parseText := func() ([]*dto.MetricFamily, error) {
parsed, err := parser.TextToMetricFamilies(strings.NewReader(text))
if err != nil {
return nil, err
}
var result []*dto.MetricFamily
for _, mf := range parsed {
result = append(result, mf)
}
return result, nil
}
gatherers := prometheus.Gatherers{
reg,
prometheus.GathererFunc(parseText),
}
gathering, err := gatherers.Gather()
if err != nil {
fmt.Println(err)
}
out := &bytes.Buffer{}
for _, mf := range gathering {
if _, err := expfmt.MetricFamilyToText(out, mf); err != nil {
panic(err)
}
}
fmt.Print(out.String())
fmt.Println("----------")
// Note how the temperature_kelvin metric family has been merged from
// different sources. Now try
text = `
# TYPE humidity_percent gauge
# HELP humidity_percent Humidity in %.
humidity_percent{location="outside"} 45.4
humidity_percent{location="inside"} 33.2
# TYPE temperature_kelvin gauge
# HELP temperature_kelvin Temperature in Kelvin.
# Duplicate metric:
temperature_kelvin{location="outside"} 265.3
# Wrong labels:
temperature_kelvin 4.5
`
gathering, err = gatherers.Gather()
if err != nil {
fmt.Println(err)
}
// Note that still as many metrics as possible are returned:
out.Reset()
for _, mf := range gathering {
if _, err := expfmt.MetricFamilyToText(out, mf); err != nil {
panic(err)
}
}
fmt.Print(out.String())
// Output:
// # HELP humidity_percent Humidity in %.
// # TYPE humidity_percent gauge
// humidity_percent{location="inside"} 33.2
// humidity_percent{location="outside"} 45.4
// # HELP temperature_kelvin Temperature in Kelvin.
// # TYPE temperature_kelvin gauge
// temperature_kelvin{location="inside"} 298.44
// temperature_kelvin{location="outside"} 273.14
// temperature_kelvin{location="somewhere else"} 4.5
// ----------
// 2 error(s) occurred:
// * collected metric temperature_kelvin label:<name:"location" value:"outside" > gauge:<value:265.3 > was collected before with the same name and label values
// * collected metric temperature_kelvin gauge:<value:4.5 > has label dimensions inconsistent with previously collected metrics in the same metric family
// # HELP humidity_percent Humidity in %.
// # TYPE humidity_percent gauge
// humidity_percent{location="inside"} 33.2
// humidity_percent{location="outside"} 45.4
// # HELP temperature_kelvin Temperature in Kelvin.
// # TYPE temperature_kelvin gauge
// temperature_kelvin{location="inside"} 298.44
// temperature_kelvin{location="outside"} 273.14
}
