func fetchMetricFamilies(url string, ch chan<- *dto.MetricFamily) {
defer close(ch)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalf("creating GET request for URL %q failed: %s", url, err)
}
req.Header.Add("Accept", acceptHeader)
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatalf("executing GET request for URL %q failed: %s", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Fatalf("GET request for URL %q returned HTTP status %s", url, resp.Status)
}
mediatype, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
if err == nil && mediatype == "application/vnd.google.protobuf" &&
params["encoding"] == "delimited" &&
params["proto"] == "io.prometheus.client.MetricFamily" {
for {
mf := &dto.MetricFamily{}
if _, err = pbutil.ReadDelimited(resp.Body, mf); err != nil {
if err == io.EOF {
break
}
log.Fatalln("reading metric family protocol buffer failed:", err)
}
ch <- mf
}
} else {
// We could do further content-type checks here, but the
// fallback for now will anyway be the text format
// version 0.0.4, so just go for it and see if it works.
var parser expfmt.TextParser
metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
if err != nil {
log.Fatalln("reading text format failed:", err)
}
for _, mf := range metricFamilies {
ch <- mf
}
}
}
// BenchmarkParseProtoGzip is like BenchmarkParseProto above, but parses gzipped
// protobuf format.
func BenchmarkParseProtoGzip(b *testing.B) {
b.StopTimer()
data, err := ioutil.ReadFile("testdata/protobuf.gz")
if err != nil {
b.Fatal(err)
}
b.StartTimer()
for i := 0; i < b.N; i++ {
family := &dto.MetricFamily{}
in, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
b.Fatal(err)
}
for {
family.Reset()
if _, err := pbutil.ReadDelimited(in, family); err != nil {
if err == io.EOF {
break
}
b.Fatal(err)
}
}
}
}
// loadSnapshot loads a snapshot generated by Snapshot() into the state.
// Any previous state is wiped.
func (s *Silences) loadSnapshot(r io.Reader) error {
st := gossipData{}
s.mtx.Lock()
defer s.mtx.Unlock()
for {
var sil pb.MeshSilence
if _, err := pbutil.ReadDelimited(r, &sil); err != nil {
if err == io.EOF {
break
}
return err
}
st[sil.Silence.Id] = &sil
_, err := s.mc.Get(sil.Silence)
if err != nil {
return err
}
}
s.st = st
return nil
}
// Decode implements the Decoder interface.
func (d *protoDecoder) Decode(v *dto.MetricFamily) error {
_, err := pbutil.ReadDelimited(d.r, v)
if err != nil {
return err
}
if !model.IsValidMetricName(model.LabelValue(v.GetName())) {
return fmt.Errorf("invalid metric name %q", v.GetName())
}
for _, m := range v.GetMetric() {
if m == nil {
continue
}
for _, l := range m.GetLabel() {
if l == nil {
continue
}
if !model.LabelValue(l.GetValue()).IsValid() {
return fmt.Errorf("invalid label value %q", l.GetValue())
}
if !model.LabelName(l.GetName()).IsValid() {
return fmt.Errorf("invalid label name %q", l.GetName())
}
}
}
return nil
}
func (u *delimitedUnmarshaller) Unmarshal(reader io.Reader, entry *Entry) error {
pbEntry := &protologpb.Entry{}
if _, err := pbutil.ReadDelimited(reader, pbEntry); err != nil {
return err
}
iEntry, err := pbEntryToEntry(pbEntry)
if err != nil {
return err
}
*entry = *iEntry
return nil
}
func decodeGossipData(msg []byte) (gossipData, error) {
gd := gossipData{}
rd := bytes.NewReader(msg)
for {
var s pb.MeshSilence
if _, err := pbutil.ReadDelimited(rd, &s); err != nil {
if err == io.EOF {
break
}
return gd, err
}
gd[s.Silence.Id] = &s
}
return gd, nil
}
func decodeGossipData(msg []byte) (gossipData, error) {
gd := gossipData{}
rd := bytes.NewReader(msg)
for {
var e pb.MeshEntry
if _, err := pbutil.ReadDelimited(rd, &e); err != nil {
if err == io.EOF {
break
}
return gd, err
}
gd[stateKey(e.Entry.GroupKey, e.Entry.Receiver)] = &e
}
return gd, nil
}
func (m *metricFamilyProcessor) ProcessSingle(i io.Reader, out Ingester, o *ProcessOptions) error {
family := &dto.MetricFamily{}
for {
family.Reset()
if _, err := pbutil.ReadDelimited(i, family); err != nil {
if err == io.EOF {
return nil
}
return err
}
if err := extractMetricFamily(out, o, family); err != nil {
return err
}
}
}
// loadSnapshot loads a snapshot generated by Snapshot() into the state.
func (l *nlog) loadSnapshot(r io.Reader) error {
l.mtx.Lock()
defer l.mtx.Unlock()
st := gossipData{}
for {
var e pb.MeshEntry
if _, err := pbutil.ReadDelimited(r, &e); err != nil {
if err == io.EOF {
break
}
return err
}
st[stateKey(e.Entry.GroupKey, e.Entry.Receiver)] = &e
}
l.st = st
return nil
}
// BenchmarkParseProtoMap is like BenchmarkParseProto but DOES put the parsed
// metric family DTOs into a map. This is not happening during Prometheus
// ingestion. It is just here to measure the overhead of that map creation and
// separate it from the overhead of the text format parsing.
func BenchmarkParseProtoMap(b *testing.B) {
b.StopTimer()
data, err := ioutil.ReadFile("testdata/protobuf")
if err != nil {
b.Fatal(err)
}
b.StartTimer()
for i := 0; i < b.N; i++ {
families := map[string]*dto.MetricFamily{}
in := bytes.NewReader(data)
for {
family := &dto.MetricFamily{}
if _, err := pbutil.ReadDelimited(in, family); err != nil {
if err == io.EOF {
break
}
b.Fatal(err)
}
families[family.GetName()] = family
}
}
}
// Decode implements the Decoder interface.
func (d *protoDecoder) Decode(v *dto.MetricFamily) error {
_, err := pbutil.ReadDelimited(d.r, v)
return err
}
// Push returns an http.Handler which accepts samples over HTTP and stores them
// in the MetricStore. If replace is true, all metrics for the job and instance
// given by the request are deleted before new ones are stored.
//
// The returned handler is already instrumented for Prometheus.
func Push(
ms storage.MetricStore, replace bool,
) func(http.ResponseWriter, *http.Request, httprouter.Params) {
var ps httprouter.Params
var mtx sync.Mutex // Protects ps.
instrumentedHandlerFunc := prometheus.InstrumentHandlerFunc(
"push",
func(w http.ResponseWriter, r *http.Request) {
job := ps.ByName("job")
labelsString := ps.ByName("labels")
mtx.Unlock()
labels, err := splitLabels(labelsString)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if job == "" {
http.Error(w, "job name is required", http.StatusBadRequest)
return
}
labels["job"] = job
if replace {
ms.SubmitWriteRequest(storage.WriteRequest{
Labels: labels,
Timestamp: time.Now(),
})
}
var metricFamilies map[string]*dto.MetricFamily
ctMediatype, ctParams, ctErr := mime.ParseMediaType(r.Header.Get("Content-Type"))
if ctErr == nil && ctMediatype == "application/vnd.google.protobuf" &&
ctParams["encoding"] == "delimited" &&
ctParams["proto"] == "io.prometheus.client.MetricFamily" {
metricFamilies = map[string]*dto.MetricFamily{}
for {
mf := &dto.MetricFamily{}
if _, err = pbutil.ReadDelimited(r.Body, mf); err != nil {
if err == io.EOF {
err = nil
}
break
}
metricFamilies[mf.GetName()] = mf
}
} else {
// We could do further content-type checks here, but the
// fallback for now will anyway be the text format
// version 0.0.4, so just go for it and see if it works.
var parser text.Parser
metricFamilies, err = parser.TextToMetricFamilies(r.Body)
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
sanitizeLabels(metricFamilies, labels)
ms.SubmitWriteRequest(storage.WriteRequest{
Labels: labels,
Timestamp: time.Now(),
MetricFamilies: metricFamilies,
})
w.WriteHeader(http.StatusAccepted)
},
)
return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
mtx.Lock()
ps = params
instrumentedHandlerFunc(w, r)
}
}
// Parse returns a slice of Metrics from a text representation of a
// metrics
func (p *PrometheusParser) Parse(buf []byte) ([]telegraf.Metric, error) {
var metrics []telegraf.Metric
var parser expfmt.TextParser
// parse even if the buffer begins with a newline
buf = bytes.TrimPrefix(buf, []byte("\n"))
// Read raw data
buffer := bytes.NewBuffer(buf)
reader := bufio.NewReader(buffer)
// Get format
mediatype, params, err := mime.ParseMediaType(p.PromFormat["Content-Type"])
// Prepare output
metricFamilies := make(map[string]*dto.MetricFamily)
if err == nil && mediatype == "application/vnd.google.protobuf" &&
params["encoding"] == "delimited" &&
params["proto"] == "io.prometheus.client.MetricFamily" {
for {
metricFamily := &dto.MetricFamily{}
if _, err = pbutil.ReadDelimited(reader, metricFamily); err != nil {
if err == io.EOF {
break
}
return nil, fmt.Errorf("reading metric family protocol buffer failed: %s", err)
}
metricFamilies[metricFamily.GetName()] = metricFamily
}
} else {
metricFamilies, err = parser.TextToMetricFamilies(reader)
if err != nil {
return nil, fmt.Errorf("reading text format failed: %s", err)
}
// read metrics
for metricName, mf := range metricFamilies {
for _, m := range mf.Metric {
// reading tags
tags := makeLabels(m)
/*
for key, value := range p.DefaultTags {
tags[key] = value
}
*/
// reading fields
fields := make(map[string]interface{})
if mf.GetType() == dto.MetricType_SUMMARY {
// summary metric
fields = makeQuantiles(m)
fields["count"] = float64(m.GetSummary().GetSampleCount())
fields["sum"] = float64(m.GetSummary().GetSampleSum())
} else if mf.GetType() == dto.MetricType_HISTOGRAM {
// historgram metric
fields = makeBuckets(m)
fields["count"] = float64(m.GetHistogram().GetSampleCount())
fields["sum"] = float64(m.GetHistogram().GetSampleSum())
} else {
// standard metric
fields = getNameAndValue(m)
}
// converting to telegraf metric
if len(fields) > 0 {
var t time.Time
if m.TimestampMs != nil && *m.TimestampMs > 0 {
t = time.Unix(0, *m.TimestampMs*1000000)
} else {
t = time.Now()
}
metric, err := telegraf.NewMetric(metricName, tags, fields, t)
if err == nil {
metrics = append(metrics, metric)
}
}
}
}
}
return metrics, err
}
func (u *unmarshaller) Unmarshal(reader io.Reader, entry *Entry) error {
_, err := pbutil.ReadDelimited(reader, entry)
return err
}