func fetchMetricFamilies(url string, ch chan<- *dto.MetricFamily) { defer close(ch) req, err := http.NewRequest("GET", url, nil) if err != nil { log.Fatalf("creating GET request for URL %q failed: %s", url, err) } req.Header.Add("Accept", acceptHeader) resp, err := http.DefaultClient.Do(req) if err != nil { log.Fatalf("executing GET request for URL %q failed: %s", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { log.Fatalf("GET request for URL %q returned HTTP status %s", url, resp.Status) } mediatype, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) if err == nil && mediatype == "application/vnd.google.protobuf" && params["encoding"] == "delimited" && params["proto"] == "io.prometheus.client.MetricFamily" { for { mf := &dto.MetricFamily{} if _, err = pbutil.ReadDelimited(resp.Body, mf); err != nil { if err == io.EOF { break } log.Fatalln("reading metric family protocol buffer failed:", err) } ch <- mf } } else { // We could do further content-type checks here, but the // fallback for now will anyway be the text format // version 0.0.4, so just go for it and see if it works. var parser expfmt.TextParser metricFamilies, err := parser.TextToMetricFamilies(resp.Body) if err != nil { log.Fatalln("reading text format failed:", err) } for _, mf := range metricFamilies { ch <- mf } } }
// BenchmarkParseProtoGzip is like BenchmarkParseProto above, but parses gzipped // protobuf format. func BenchmarkParseProtoGzip(b *testing.B) { b.StopTimer() data, err := ioutil.ReadFile("testdata/protobuf.gz") if err != nil { b.Fatal(err) } b.StartTimer() for i := 0; i < b.N; i++ { family := &dto.MetricFamily{} in, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { b.Fatal(err) } for { family.Reset() if _, err := pbutil.ReadDelimited(in, family); err != nil { if err == io.EOF { break } b.Fatal(err) } } } }
// loadSnapshot loads a snapshot generated by Snapshot() into the state. // Any previous state is wiped. func (s *Silences) loadSnapshot(r io.Reader) error { st := gossipData{} s.mtx.Lock() defer s.mtx.Unlock() for { var sil pb.MeshSilence if _, err := pbutil.ReadDelimited(r, &sil); err != nil { if err == io.EOF { break } return err } st[sil.Silence.Id] = &sil _, err := s.mc.Get(sil.Silence) if err != nil { return err } } s.st = st return nil }
// Decode implements the Decoder interface. func (d *protoDecoder) Decode(v *dto.MetricFamily) error { _, err := pbutil.ReadDelimited(d.r, v) if err != nil { return err } if !model.IsValidMetricName(model.LabelValue(v.GetName())) { return fmt.Errorf("invalid metric name %q", v.GetName()) } for _, m := range v.GetMetric() { if m == nil { continue } for _, l := range m.GetLabel() { if l == nil { continue } if !model.LabelValue(l.GetValue()).IsValid() { return fmt.Errorf("invalid label value %q", l.GetValue()) } if !model.LabelName(l.GetName()).IsValid() { return fmt.Errorf("invalid label name %q", l.GetName()) } } } return nil }
func (u *delimitedUnmarshaller) Unmarshal(reader io.Reader, entry *Entry) error { pbEntry := &protologpb.Entry{} if _, err := pbutil.ReadDelimited(reader, pbEntry); err != nil { return err } iEntry, err := pbEntryToEntry(pbEntry) if err != nil { return err } *entry = *iEntry return nil }
func decodeGossipData(msg []byte) (gossipData, error) { gd := gossipData{} rd := bytes.NewReader(msg) for { var s pb.MeshSilence if _, err := pbutil.ReadDelimited(rd, &s); err != nil { if err == io.EOF { break } return gd, err } gd[s.Silence.Id] = &s } return gd, nil }
func decodeGossipData(msg []byte) (gossipData, error) { gd := gossipData{} rd := bytes.NewReader(msg) for { var e pb.MeshEntry if _, err := pbutil.ReadDelimited(rd, &e); err != nil { if err == io.EOF { break } return gd, err } gd[stateKey(e.Entry.GroupKey, e.Entry.Receiver)] = &e } return gd, nil }
func (m *metricFamilyProcessor) ProcessSingle(i io.Reader, out Ingester, o *ProcessOptions) error { family := &dto.MetricFamily{} for { family.Reset() if _, err := pbutil.ReadDelimited(i, family); err != nil { if err == io.EOF { return nil } return err } if err := extractMetricFamily(out, o, family); err != nil { return err } } }
// loadSnapshot loads a snapshot generated by Snapshot() into the state. func (l *nlog) loadSnapshot(r io.Reader) error { l.mtx.Lock() defer l.mtx.Unlock() st := gossipData{} for { var e pb.MeshEntry if _, err := pbutil.ReadDelimited(r, &e); err != nil { if err == io.EOF { break } return err } st[stateKey(e.Entry.GroupKey, e.Entry.Receiver)] = &e } l.st = st return nil }
// BenchmarkParseProtoMap is like BenchmarkParseProto but DOES put the parsed // metric family DTOs into a map. This is not happening during Prometheus // ingestion. It is just here to measure the overhead of that map creation and // separate it from the overhead of the text format parsing. func BenchmarkParseProtoMap(b *testing.B) { b.StopTimer() data, err := ioutil.ReadFile("testdata/protobuf") if err != nil { b.Fatal(err) } b.StartTimer() for i := 0; i < b.N; i++ { families := map[string]*dto.MetricFamily{} in := bytes.NewReader(data) for { family := &dto.MetricFamily{} if _, err := pbutil.ReadDelimited(in, family); err != nil { if err == io.EOF { break } b.Fatal(err) } families[family.GetName()] = family } } }
// Decode implements the Decoder interface. func (d *protoDecoder) Decode(v *dto.MetricFamily) error { _, err := pbutil.ReadDelimited(d.r, v) return err }
// Push returns an http.Handler which accepts samples over HTTP and stores them // in the MetricStore. If replace is true, all metrics for the job and instance // given by the request are deleted before new ones are stored. // // The returned handler is already instrumented for Prometheus. func Push( ms storage.MetricStore, replace bool, ) func(http.ResponseWriter, *http.Request, httprouter.Params) { var ps httprouter.Params var mtx sync.Mutex // Protects ps. instrumentedHandlerFunc := prometheus.InstrumentHandlerFunc( "push", func(w http.ResponseWriter, r *http.Request) { job := ps.ByName("job") labelsString := ps.ByName("labels") mtx.Unlock() labels, err := splitLabels(labelsString) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } if job == "" { http.Error(w, "job name is required", http.StatusBadRequest) return } labels["job"] = job if replace { ms.SubmitWriteRequest(storage.WriteRequest{ Labels: labels, Timestamp: time.Now(), }) } var metricFamilies map[string]*dto.MetricFamily ctMediatype, ctParams, ctErr := mime.ParseMediaType(r.Header.Get("Content-Type")) if ctErr == nil && ctMediatype == "application/vnd.google.protobuf" && ctParams["encoding"] == "delimited" && ctParams["proto"] == "io.prometheus.client.MetricFamily" { metricFamilies = map[string]*dto.MetricFamily{} for { mf := &dto.MetricFamily{} if _, err = pbutil.ReadDelimited(r.Body, mf); err != nil { if err == io.EOF { err = nil } break } metricFamilies[mf.GetName()] = mf } } else { // We could do further content-type checks here, but the // fallback for now will anyway be the text format // version 0.0.4, so just go for it and see if it works. var parser text.Parser metricFamilies, err = parser.TextToMetricFamilies(r.Body) } if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } sanitizeLabels(metricFamilies, labels) ms.SubmitWriteRequest(storage.WriteRequest{ Labels: labels, Timestamp: time.Now(), MetricFamilies: metricFamilies, }) w.WriteHeader(http.StatusAccepted) }, ) return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) { mtx.Lock() ps = params instrumentedHandlerFunc(w, r) } }
// Parse returns a slice of Metrics from a text representation of a // metrics func (p *PrometheusParser) Parse(buf []byte) ([]telegraf.Metric, error) { var metrics []telegraf.Metric var parser expfmt.TextParser // parse even if the buffer begins with a newline buf = bytes.TrimPrefix(buf, []byte("\n")) // Read raw data buffer := bytes.NewBuffer(buf) reader := bufio.NewReader(buffer) // Get format mediatype, params, err := mime.ParseMediaType(p.PromFormat["Content-Type"]) // Prepare output metricFamilies := make(map[string]*dto.MetricFamily) if err == nil && mediatype == "application/vnd.google.protobuf" && params["encoding"] == "delimited" && params["proto"] == "io.prometheus.client.MetricFamily" { for { metricFamily := &dto.MetricFamily{} if _, err = pbutil.ReadDelimited(reader, metricFamily); err != nil { if err == io.EOF { break } return nil, fmt.Errorf("reading metric family protocol buffer failed: %s", err) } metricFamilies[metricFamily.GetName()] = metricFamily } } else { metricFamilies, err = parser.TextToMetricFamilies(reader) if err != nil { return nil, fmt.Errorf("reading text format failed: %s", err) } // read metrics for metricName, mf := range metricFamilies { for _, m := range mf.Metric { // reading tags tags := makeLabels(m) /* for key, value := range p.DefaultTags { tags[key] = value } */ // reading fields fields := make(map[string]interface{}) if mf.GetType() == dto.MetricType_SUMMARY { // summary metric fields = makeQuantiles(m) fields["count"] = float64(m.GetSummary().GetSampleCount()) fields["sum"] = float64(m.GetSummary().GetSampleSum()) } else if mf.GetType() == dto.MetricType_HISTOGRAM { // historgram metric fields = makeBuckets(m) fields["count"] = float64(m.GetHistogram().GetSampleCount()) fields["sum"] = float64(m.GetHistogram().GetSampleSum()) } else { // standard metric fields = getNameAndValue(m) } // converting to telegraf metric if len(fields) > 0 { var t time.Time if m.TimestampMs != nil && *m.TimestampMs > 0 { t = time.Unix(0, *m.TimestampMs*1000000) } else { t = time.Now() } metric, err := telegraf.NewMetric(metricName, tags, fields, t) if err == nil { metrics = append(metrics, metric) } } } } } return metrics, err }
func (u *unmarshaller) Unmarshal(reader io.Reader, entry *Entry) error { _, err := pbutil.ReadDelimited(reader, entry) return err }