func extractSummary(o *DecodeOptions, f *dto.MetricFamily) model.Vector { samples := make(model.Vector, 0, len(f.Metric)) for _, m := range f.Metric { if m.Summary == nil { continue } timestamp := o.Timestamp if m.TimestampMs != nil { timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000) } for _, q := range m.Summary.Quantile { lset := make(model.LabelSet, len(m.Label)+2) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } // BUG(matt): Update other names to "quantile". lset[model.LabelName(model.QuantileLabel)] = model.LabelValue(fmt.Sprint(q.GetQuantile())) lset[model.MetricNameLabel] = model.LabelValue(f.GetName()) samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(q.GetValue()), Timestamp: timestamp, }) } if m.Summary.SampleSum != nil { lset := make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_sum") samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Summary.GetSampleSum()), Timestamp: timestamp, }) } if m.Summary.SampleCount != nil { lset := make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_count") samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Summary.GetSampleCount()), Timestamp: timestamp, }) } } return samples }
func (p *persistence) rebuildLabelIndexes( fpToSeries map[model.Fingerprint]*memorySeries, ) error { count := 0 log.Info("Rebuilding label indexes.") log.Info("Indexing metrics in memory.") for fp, s := range fpToSeries { p.indexMetric(fp, s.metric) count++ if count%10000 == 0 { log.Infof("%d metrics queued for indexing.", count) } } log.Info("Indexing archived metrics.") var fp codable.Fingerprint var m codable.Metric if err := p.archivedFingerprintToMetrics.ForEach(func(kv index.KeyValueAccessor) error { if err := kv.Key(&fp); err != nil { return err } if err := kv.Value(&m); err != nil { return err } p.indexMetric(model.Fingerprint(fp), model.Metric(m)) count++ if count%10000 == 0 { log.Infof("%d metrics queued for indexing.", count) } return nil }); err != nil { return err } log.Info("All requests for rebuilding the label indexes queued. (Actual processing may lag behind.)") return nil }
func extractUntyped(o *DecodeOptions, f *dto.MetricFamily) model.Vector { samples := make(model.Vector, 0, len(f.Metric)) for _, m := range f.Metric { if m.Untyped == nil { continue } lset := make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName()) smpl := &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Untyped.GetValue()), } if m.TimestampMs != nil { smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000) } else { smpl.Timestamp = o.Timestamp } samples = append(samples, smpl) } return samples }
// Append implements storage.SampleAppender. Always returns nil. func (s *ReloadableStorage) Append(smpl *model.Sample) error { s.mtx.RLock() defer s.mtx.RUnlock() if s.queue == nil { return nil } var snew model.Sample snew = *smpl snew.Metric = smpl.Metric.Clone() for ln, lv := range s.externalLabels { if _, ok := smpl.Metric[ln]; !ok { snew.Metric[ln] = lv } } snew.Metric = model.Metric( relabel.Process(model.LabelSet(snew.Metric), s.conf.WriteRelabelConfigs...)) if snew.Metric == nil { return nil } s.queue.Append(&snew) return nil }
// Append implements storage.SampleAppender. Always returns nil. func (s *Storage) Append(smpl *model.Sample) error { s.mtx.RLock() var snew model.Sample snew = *smpl snew.Metric = smpl.Metric.Clone() for ln, lv := range s.externalLabels { if _, ok := smpl.Metric[ln]; !ok { snew.Metric[ln] = lv } } snew.Metric = model.Metric( relabel.Process(model.LabelSet(snew.Metric), s.relabelConfigs...)) s.mtx.RUnlock() if snew.Metric == nil { return nil } for _, q := range s.queues { q.Append(&snew) } return nil }
func (app relabelAppender) Append(s *model.Sample) error { labels := relabel.Process(model.LabelSet(s.Metric), app.relabelings...) // Check if the timeseries was dropped. if labels == nil { return nil } s.Metric = model.Metric(labels) return app.SampleAppender.Append(s) }
func (app relabelAppender) Append(s *model.Sample) error { labels, err := Relabel(model.LabelSet(s.Metric), app.relabelings...) if err != nil { return fmt.Errorf("metric relabeling error %s: %s", s.Metric, err) } // Check if the timeseries was dropped. if labels == nil { return nil } s.Metric = model.Metric(labels) return app.SampleAppender.Append(s) }
func (app relabelAppender) Append(s *model.Sample) { labels, err := Relabel(model.LabelSet(s.Metric), app.relabelings...) if err != nil { log.Errorf("Error while relabeling metric %s: %s", s.Metric, err) return } // Check if the timeseries was dropped. if labels == nil { return } s.Metric = model.Metric(labels) app.app.Append(s) }
// metric parses a metric. // // <label_set> // <metric_identifier> [<label_set>] // func (p *parser) metric() model.Metric { name := "" m := model.Metric{} t := p.peek().typ if t == itemIdentifier || t == itemMetricIdentifier { name = p.next().val t = p.peek().typ } if t != itemLeftBrace && name == "" { p.errorf("missing metric name or metric selector") } if t == itemLeftBrace { m = model.Metric(p.labelSet()) } if name != "" { m[model.MetricNameLabel] = model.LabelValue(name) } return m }
func (r *AlertingRule) sample(alert *Alert, ts model.Time, set bool) *model.Sample { metric := model.Metric(r.labels.Clone()) for ln, lv := range alert.Labels { metric[ln] = lv } metric[model.MetricNameLabel] = alertMetricName metric[model.AlertNameLabel] = model.LabelValue(r.name) metric[alertStateLabel] = model.LabelValue(alert.State.String()) s := &model.Sample{ Metric: metric, Timestamp: ts, Value: 0, } if set { s.Value = 1 } return s }
// scan works like bufio.Scanner.Scan. func (hs *headsScanner) scan() bool { if hs.seriesCurrent == hs.seriesTotal || hs.err != nil { return false } var ( seriesFlags byte fpAsInt uint64 metric codable.Metric persistWatermark int64 modTimeNano int64 modTime time.Time chunkDescsOffset int64 savedFirstTime int64 numChunkDescs int64 firstTime int64 lastTime int64 encoding byte ch chunk.Chunk lastTimeHead model.Time ) if seriesFlags, hs.err = hs.r.ReadByte(); hs.err != nil { return false } headChunkPersisted := seriesFlags&flagHeadChunkPersisted != 0 if fpAsInt, hs.err = codable.DecodeUint64(hs.r); hs.err != nil { return false } hs.fp = model.Fingerprint(fpAsInt) if hs.err = metric.UnmarshalFromReader(hs.r); hs.err != nil { return false } if hs.version != headsFormatLegacyVersion { // persistWatermark only present in v2. persistWatermark, hs.err = binary.ReadVarint(hs.r) if hs.err != nil { return false } modTimeNano, hs.err = binary.ReadVarint(hs.r) if hs.err != nil { return false } if modTimeNano != -1 { modTime = time.Unix(0, modTimeNano) } } if chunkDescsOffset, hs.err = binary.ReadVarint(hs.r); hs.err != nil { return false } if savedFirstTime, hs.err = binary.ReadVarint(hs.r); hs.err != nil { return false } if numChunkDescs, hs.err = binary.ReadVarint(hs.r); hs.err != nil { return false } chunkDescs := make([]*chunk.Desc, numChunkDescs) if hs.version == headsFormatLegacyVersion { if headChunkPersisted { persistWatermark = numChunkDescs } else { persistWatermark = numChunkDescs - 1 } } headChunkClosed := true // Initial assumption. for i := int64(0); i < numChunkDescs; i++ { if i < persistWatermark { if firstTime, hs.err = binary.ReadVarint(hs.r); hs.err != nil { return false } if lastTime, hs.err = binary.ReadVarint(hs.r); hs.err != nil { return false } chunkDescs[i] = &chunk.Desc{ ChunkFirstTime: model.Time(firstTime), ChunkLastTime: model.Time(lastTime), } chunk.NumMemDescs.Inc() } else { // Non-persisted chunk. // If there are non-persisted chunks at all, we consider // the head chunk not to be closed yet. headChunkClosed = false if encoding, hs.err = hs.r.ReadByte(); hs.err != nil { return false } if ch, hs.err = chunk.NewForEncoding(chunk.Encoding(encoding)); hs.err != nil { return false } if hs.err = ch.Unmarshal(hs.r); hs.err != nil { return false } cd := chunk.NewDesc(ch, ch.FirstTime()) if i < numChunkDescs-1 { // This is NOT the head chunk. So it's a chunk // to be persisted, and we need to populate lastTime. hs.chunksToPersistTotal++ cd.MaybePopulateLastTime() } chunkDescs[i] = cd } } if lastTimeHead, hs.err = chunkDescs[len(chunkDescs)-1].LastTime(); hs.err != nil { return false } hs.series = &memorySeries{ metric: model.Metric(metric), chunkDescs: chunkDescs, persistWatermark: int(persistWatermark), modTime: modTime, chunkDescsOffset: int(chunkDescsOffset), savedFirstTime: model.Time(savedFirstTime), lastTime: lastTimeHead, headChunkClosed: headChunkClosed, } hs.seriesCurrent++ return true }
func extractHistogram(o *DecodeOptions, f *dto.MetricFamily) model.Vector { samples := make(model.Vector, 0, len(f.Metric)) for _, m := range f.Metric { if m.Histogram == nil { continue } timestamp := o.Timestamp if m.TimestampMs != nil { timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000) } infSeen := false for _, q := range m.Histogram.Bucket { lset := make(model.LabelSet, len(m.Label)+2) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.LabelName(model.BucketLabel)] = model.LabelValue(fmt.Sprint(q.GetUpperBound())) lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_bucket") if math.IsInf(q.GetUpperBound(), +1) { infSeen = true } samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(q.GetCumulativeCount()), Timestamp: timestamp, }) } if m.Histogram.SampleSum != nil { lset := make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_sum") samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Histogram.GetSampleSum()), Timestamp: timestamp, }) } if m.Histogram.SampleCount != nil { lset := make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_count") count := &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Histogram.GetSampleCount()), Timestamp: timestamp, } samples = append(samples, count) if !infSeen { // Append a infinity bucket sample. lset := make(model.LabelSet, len(m.Label)+2) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.LabelName(model.BucketLabel)] = model.LabelValue("+Inf") lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_bucket") samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: count.Value, Timestamp: timestamp, }) } } } return samples }
func (p *persistence) cleanUpArchiveIndexes( fpToSeries map[model.Fingerprint]*memorySeries, fpsSeen map[model.Fingerprint]struct{}, fpm fpMappings, ) error { log.Info("Cleaning up archive indexes.") var fp codable.Fingerprint var m codable.Metric count := 0 if err := p.archivedFingerprintToMetrics.ForEach(func(kv index.KeyValueAccessor) error { count++ if count%10000 == 0 { log.Infof("%d archived metrics checked.", count) } if err := kv.Key(&fp); err != nil { return err } _, fpSeen := fpsSeen[model.Fingerprint(fp)] inMemory := false if fpSeen { _, inMemory = fpToSeries[model.Fingerprint(fp)] } if !fpSeen || inMemory { if inMemory { log.Warnf("Archive clean-up: Fingerprint %v is not archived. Purging from archive indexes.", model.Fingerprint(fp)) } if !fpSeen { log.Warnf("Archive clean-up: Fingerprint %v is unknown. Purging from archive indexes.", model.Fingerprint(fp)) } // It's fine if the fp is not in the archive indexes. if _, err := p.archivedFingerprintToMetrics.Delete(fp); err != nil { return err } // Delete from timerange index, too. _, err := p.archivedFingerprintToTimeRange.Delete(fp) return err } // fp is legitimately archived. Now we need the metric to check for a mapped fingerprint. if err := kv.Value(&m); err != nil { return err } maybeAddMapping(model.Fingerprint(fp), model.Metric(m), fpm) // Make sure it is in timerange index, too. has, err := p.archivedFingerprintToTimeRange.Has(fp) if err != nil { return err } if has { return nil // All good. } log.Warnf("Archive clean-up: Fingerprint %v is not in time-range index. Unarchiving it for recovery.") // Again, it's fine if fp is not in the archive index. if _, err := p.archivedFingerprintToMetrics.Delete(fp); err != nil { return err } cds, err := p.loadChunkDescs(model.Fingerprint(fp), 0) if err != nil { return err } series, err := newMemorySeries(model.Metric(m), cds, p.seriesFileModTime(model.Fingerprint(fp))) if err != nil { return err } fpToSeries[model.Fingerprint(fp)] = series return nil }); err != nil { return err } count = 0 if err := p.archivedFingerprintToTimeRange.ForEach(func(kv index.KeyValueAccessor) error { count++ if count%10000 == 0 { log.Infof("%d archived time ranges checked.", count) } if err := kv.Key(&fp); err != nil { return err } has, err := p.archivedFingerprintToMetrics.Has(fp) if err != nil { return err } if has { return nil // All good. } log.Warnf("Archive clean-up: Purging unknown fingerprint %v in time-range index.", fp) deleted, err := p.archivedFingerprintToTimeRange.Delete(fp) if err != nil { return err } if !deleted { log.Errorf("Fingerprint %v to be deleted from archivedFingerprintToTimeRange not found. This should never happen.", fp) } return nil }); err != nil { return err } log.Info("Clean-up of archive indexes complete.") return nil }
// loadSeriesMapAndHeads loads the fingerprint to memory-series mapping and all // the chunks contained in the checkpoint (and thus not yet persisted to series // files). The method is capable of loading the checkpoint format v1 and v2. If // recoverable corruption is detected, or if the dirty flag was set from the // beginning, crash recovery is run, which might take a while. If an // unrecoverable error is encountered, it is returned. Call this method during // start-up while nothing else is running in storage land. This method is // utterly goroutine-unsafe. func (p *persistence) loadSeriesMapAndHeads() (sm *seriesMap, chunksToPersist int64, err error) { var chunkDescsTotal int64 fingerprintToSeries := make(map[model.Fingerprint]*memorySeries) sm = &seriesMap{m: fingerprintToSeries} defer func() { if sm != nil && p.dirty { log.Warn("Persistence layer appears dirty.") err = p.recoverFromCrash(fingerprintToSeries) if err != nil { sm = nil } } if err == nil { numMemChunkDescs.Add(float64(chunkDescsTotal)) } }() f, err := os.Open(p.headsFileName()) if os.IsNotExist(err) { return sm, 0, nil } if err != nil { log.Warn("Could not open heads file:", err) p.dirty = true return } defer f.Close() r := bufio.NewReaderSize(f, fileBufSize) buf := make([]byte, len(headsMagicString)) if _, err := io.ReadFull(r, buf); err != nil { log.Warn("Could not read from heads file:", err) p.dirty = true return sm, 0, nil } magic := string(buf) if magic != headsMagicString { log.Warnf( "unexpected magic string, want %q, got %q", headsMagicString, magic, ) p.dirty = true return } version, err := binary.ReadVarint(r) if (version != headsFormatVersion && version != headsFormatLegacyVersion) || err != nil { log.Warnf("unknown heads format version, want %d", headsFormatVersion) p.dirty = true return sm, 0, nil } numSeries, err := codable.DecodeUint64(r) if err != nil { log.Warn("Could not decode number of series:", err) p.dirty = true return sm, 0, nil } for ; numSeries > 0; numSeries-- { seriesFlags, err := r.ReadByte() if err != nil { log.Warn("Could not read series flags:", err) p.dirty = true return sm, chunksToPersist, nil } headChunkPersisted := seriesFlags&flagHeadChunkPersisted != 0 fp, err := codable.DecodeUint64(r) if err != nil { log.Warn("Could not decode fingerprint:", err) p.dirty = true return sm, chunksToPersist, nil } var metric codable.Metric if err := metric.UnmarshalFromReader(r); err != nil { log.Warn("Could not decode metric:", err) p.dirty = true return sm, chunksToPersist, nil } var persistWatermark int64 var modTime time.Time if version != headsFormatLegacyVersion { // persistWatermark only present in v2. persistWatermark, err = binary.ReadVarint(r) if err != nil { log.Warn("Could not decode persist watermark:", err) p.dirty = true return sm, chunksToPersist, nil } modTimeNano, err := binary.ReadVarint(r) if err != nil { log.Warn("Could not decode modification time:", err) p.dirty = true return sm, chunksToPersist, nil } if modTimeNano != -1 { modTime = time.Unix(0, modTimeNano) } } chunkDescsOffset, err := binary.ReadVarint(r) if err != nil { log.Warn("Could not decode chunk descriptor offset:", err) p.dirty = true return sm, chunksToPersist, nil } savedFirstTime, err := binary.ReadVarint(r) if err != nil { log.Warn("Could not decode saved first time:", err) p.dirty = true return sm, chunksToPersist, nil } numChunkDescs, err := binary.ReadVarint(r) if err != nil { log.Warn("Could not decode number of chunk descriptors:", err) p.dirty = true return sm, chunksToPersist, nil } chunkDescs := make([]*chunkDesc, numChunkDescs) if version == headsFormatLegacyVersion { if headChunkPersisted { persistWatermark = numChunkDescs } else { persistWatermark = numChunkDescs - 1 } } for i := int64(0); i < numChunkDescs; i++ { if i < persistWatermark { firstTime, err := binary.ReadVarint(r) if err != nil { log.Warn("Could not decode first time:", err) p.dirty = true return sm, chunksToPersist, nil } lastTime, err := binary.ReadVarint(r) if err != nil { log.Warn("Could not decode last time:", err) p.dirty = true return sm, chunksToPersist, nil } chunkDescs[i] = &chunkDesc{ chunkFirstTime: model.Time(firstTime), chunkLastTime: model.Time(lastTime), } chunkDescsTotal++ } else { // Non-persisted chunk. encoding, err := r.ReadByte() if err != nil { log.Warn("Could not decode chunk type:", err) p.dirty = true return sm, chunksToPersist, nil } chunk := newChunkForEncoding(chunkEncoding(encoding)) if err := chunk.unmarshal(r); err != nil { log.Warn("Could not decode chunk:", err) p.dirty = true return sm, chunksToPersist, nil } chunkDescs[i] = newChunkDesc(chunk) chunksToPersist++ } } fingerprintToSeries[model.Fingerprint(fp)] = &memorySeries{ metric: model.Metric(metric), chunkDescs: chunkDescs, persistWatermark: int(persistWatermark), modTime: modTime, chunkDescsOffset: int(chunkDescsOffset), savedFirstTime: model.Time(savedFirstTime), lastTime: chunkDescs[len(chunkDescs)-1].lastTime(), headChunkClosed: persistWatermark >= numChunkDescs, } } return sm, chunksToPersist, nil }
func (t *Target) scrape(sampleAppender storage.SampleAppender) (err error) { start := time.Now() baseLabels := t.BaseLabels() t.RLock() var ( honorLabels = t.honorLabels httpClient = t.httpClient metricRelabelConfigs = t.metricRelabelConfigs ) t.RUnlock() defer func() { t.status.setLastError(err) recordScrapeHealth(sampleAppender, start, baseLabels, t.status.Health(), time.Since(start)) }() req, err := http.NewRequest("GET", t.URL().String(), nil) if err != nil { panic(err) } req.Header.Add("Accept", acceptHeader) resp, err := httpClient.Do(req) if err != nil { return err } if resp.StatusCode != http.StatusOK { return fmt.Errorf("server returned HTTP status %s", resp.Status) } dec, err := expfmt.NewDecoder(resp.Body, resp.Header) if err != nil { return err } defer resp.Body.Close() sdec := expfmt.SampleDecoder{ Dec: dec, Opts: &expfmt.DecodeOptions{ Timestamp: model.TimeFromUnixNano(start.UnixNano()), }, } t.ingestedSamples = make(chan model.Vector, ingestedSamplesCap) go func() { for { // TODO(fabxc): Changex the SampleAppender interface to return an error // so we can proceed based on the status and don't leak goroutines trying // to append a single sample after dropping all the other ones. // // This will also allow use to reuse this vector and save allocations. var samples model.Vector if err = sdec.Decode(&samples); err != nil { break } if err = t.ingest(samples); err != nil { break } } close(t.ingestedSamples) }() for samples := range t.ingestedSamples { for _, s := range samples { if honorLabels { // Merge the metric with the baseLabels for labels not already set in the // metric. This also considers labels explicitly set to the empty string. for ln, lv := range baseLabels { if _, ok := s.Metric[ln]; !ok { s.Metric[ln] = lv } } } else { // Merge the ingested metric with the base label set. On a collision the // value of the label is stored in a label prefixed with the exported prefix. for ln, lv := range baseLabels { if v, ok := s.Metric[ln]; ok && v != "" { s.Metric[model.ExportedLabelPrefix+ln] = v } s.Metric[ln] = lv } } // Avoid the copy in Relabel if there are no configs. if len(metricRelabelConfigs) > 0 { labels, err := Relabel(model.LabelSet(s.Metric), metricRelabelConfigs...) if err != nil { log.Errorf("Error while relabeling metric %s of instance %s: %s", s.Metric, req.URL, err) continue } // Check if the timeseries was dropped. if labels == nil { continue } s.Metric = model.Metric(labels) } sampleAppender.Append(s) } } if err == io.EOF { return nil } return err }