// preloadChunksForRange loads chunks for the given range from the persistence. // The caller must have locked the fingerprint of the series. func (s *memorySeries) preloadChunksForRange( fp model.Fingerprint, from model.Time, through model.Time, mss *MemorySeriesStorage, ) (SeriesIterator, error) { firstChunkDescTime := model.Latest if len(s.chunkDescs) > 0 { firstChunkDescTime = s.chunkDescs[0].FirstTime() } if s.chunkDescsOffset != 0 && from.Before(firstChunkDescTime) { cds, err := mss.loadChunkDescs(fp, s.persistWatermark) if err != nil { return nopIter, err } s.chunkDescs = append(cds, s.chunkDescs...) s.chunkDescsOffset = 0 s.persistWatermark += len(cds) firstChunkDescTime = s.chunkDescs[0].FirstTime() } if len(s.chunkDescs) == 0 || through.Before(firstChunkDescTime) { return nopIter, nil } // Find first chunk with start time after "from". fromIdx := sort.Search(len(s.chunkDescs), func(i int) bool { return s.chunkDescs[i].FirstTime().After(from) }) // Find first chunk with start time after "through". throughIdx := sort.Search(len(s.chunkDescs), func(i int) bool { return s.chunkDescs[i].FirstTime().After(through) }) if fromIdx == len(s.chunkDescs) { // Even the last chunk starts before "from". Find out if the // series ends before "from" and we don't need to do anything. lt, err := s.chunkDescs[len(s.chunkDescs)-1].LastTime() if err != nil { return nopIter, err } if lt.Before(from) { return nopIter, nil } } if fromIdx > 0 { fromIdx-- } if throughIdx == len(s.chunkDescs) { throughIdx-- } if fromIdx > throughIdx { // Guard against nonsensical result. The caller will quarantine the series with a meaningful log entry. return nopIter, fmt.Errorf("fromIdx=%d is greater than throughIdx=%d, likely caused by data corruption", fromIdx, throughIdx) } pinIndexes := make([]int, 0, throughIdx-fromIdx+1) for i := fromIdx; i <= throughIdx; i++ { pinIndexes = append(pinIndexes, i) } return s.preloadChunks(pinIndexes, fp, mss) }
// preloadChunksForInstant preloads chunks for the latest value in the given // range. If the last sample saved in the memorySeries itself is the latest // value in the given range, it will in fact preload zero chunks and just take // that value. func (s *memorySeries) preloadChunksForInstant( fp model.Fingerprint, from model.Time, through model.Time, mss *MemorySeriesStorage, ) (SeriesIterator, error) { // If we have a lastSamplePair in the series, and thas last samplePair // is in the interval, just take it in a singleSampleSeriesIterator. No // need to pin or load anything. lastSample := s.lastSamplePair() if !through.Before(lastSample.Timestamp) && !from.After(lastSample.Timestamp) && lastSample != model.ZeroSamplePair { iter := &boundedIterator{ it: &singleSampleSeriesIterator{ samplePair: lastSample, metric: s.metric, }, start: model.Now().Add(-mss.dropAfter), } return iter, nil } // If we are here, we are out of luck and have to delegate to the more // expensive method. return s.preloadChunksForRange(fp, from, through, mss) }
func (s *memorySeriesStorage) preloadChunksForRange( fp model.Fingerprint, from model.Time, through model.Time, stalenessDelta time.Duration, ) ([]*chunkDesc, error) { s.fpLocker.Lock(fp) defer s.fpLocker.Unlock(fp) series, ok := s.fpToSeries.get(fp) if !ok { has, first, last, err := s.persistence.hasArchivedMetric(fp) if err != nil { return nil, err } if !has { s.invalidPreloadRequestsCount.Inc() return nil, nil } if from.Add(-stalenessDelta).Before(last) && through.Add(stalenessDelta).After(first) { metric, err := s.persistence.archivedMetric(fp) if err != nil { return nil, err } series = s.getOrCreateSeries(fp, metric) } else { return nil, nil } } return series.preloadChunksForRange(from, through, fp, s) }
// PreloadInstant implements Preloader func (p *memorySeriesPreloader) PreloadInstant( fp model.Fingerprint, timestamp model.Time, stalenessDelta time.Duration, ) SeriesIterator { cds, iter := p.storage.preloadChunksForInstant(fp, timestamp.Add(-stalenessDelta), timestamp) p.pinnedChunkDescs = append(p.pinnedChunkDescs, cds...) return iter }
// eval evaluates the rule expression and then creates pending alerts and fires // or removes previously pending alerts accordingly. func (rule *AlertingRule) eval(timestamp model.Time, engine *promql.Engine) (model.Vector, error) { query, err := engine.NewInstantQuery(rule.vector.String(), timestamp) if err != nil { return nil, err } exprResult, err := query.Exec().Vector() if err != nil { return nil, err } rule.mutex.Lock() defer rule.mutex.Unlock() // Create pending alerts for any new vector elements in the alert expression // or update the expression value for existing elements. resultFPs := map[model.Fingerprint]struct{}{} for _, sample := range exprResult { fp := sample.Metric.Fingerprint() resultFPs[fp] = struct{}{} if alert, ok := rule.activeAlerts[fp]; !ok { labels := model.LabelSet(sample.Metric.Clone()) labels = labels.Merge(rule.labels) if _, ok := labels[model.MetricNameLabel]; ok { delete(labels, model.MetricNameLabel) } rule.activeAlerts[fp] = &Alert{ Name: rule.name, Labels: labels, State: StatePending, ActiveSince: timestamp, Value: sample.Value, } } else { alert.Value = sample.Value } } var vector model.Vector // Check if any pending alerts should be removed or fire now. Write out alert timeseries. for fp, activeAlert := range rule.activeAlerts { if _, ok := resultFPs[fp]; !ok { vector = append(vector, activeAlert.sample(timestamp, 0)) delete(rule.activeAlerts, fp) continue } if activeAlert.State == StatePending && timestamp.Sub(activeAlert.ActiveSince) >= rule.holdDuration { vector = append(vector, activeAlert.sample(timestamp, 0)) activeAlert.State = StateFiring } vector = append(vector, activeAlert.sample(timestamp, 1)) } return vector, nil }
// contains implements Iterator. func (it *varbitChunkIterator) Contains(t model.Time) (bool, error) { last, err := it.LastTimestamp() if err != nil { it.lastError = err return false, err } return !t.Before(it.c.FirstTime()) && !t.After(last), it.lastError }
// preloadChunksForRange loads chunks for the given range from the persistence. // The caller must have locked the fingerprint of the series. func (s *memorySeries) preloadChunksForRange( fp model.Fingerprint, from model.Time, through model.Time, mss *MemorySeriesStorage, ) (SeriesIterator, error) { firstChunkDescTime := model.Latest if len(s.chunkDescs) > 0 { firstChunkDescTime = s.chunkDescs[0].FirstTime() } if s.chunkDescsOffset != 0 && from.Before(firstChunkDescTime) { cds, err := mss.loadChunkDescs(fp, s.persistWatermark) if err != nil { return nopIter, err } s.chunkDescs = append(cds, s.chunkDescs...) s.chunkDescsOffset = 0 s.persistWatermark += len(cds) firstChunkDescTime = s.chunkDescs[0].FirstTime() } if len(s.chunkDescs) == 0 || through.Before(firstChunkDescTime) { return nopIter, nil } // Find first chunk with start time after "from". fromIdx := sort.Search(len(s.chunkDescs), func(i int) bool { return s.chunkDescs[i].FirstTime().After(from) }) // Find first chunk with start time after "through". throughIdx := sort.Search(len(s.chunkDescs), func(i int) bool { return s.chunkDescs[i].FirstTime().After(through) }) if fromIdx == len(s.chunkDescs) { // Even the last chunk starts before "from". Find out if the // series ends before "from" and we don't need to do anything. lt, err := s.chunkDescs[len(s.chunkDescs)-1].LastTime() if err != nil { return nopIter, err } if lt.Before(from) { return nopIter, nil } } if fromIdx > 0 { fromIdx-- } if throughIdx == len(s.chunkDescs) { throughIdx-- } pinIndexes := make([]int, 0, throughIdx-fromIdx+1) for i := fromIdx; i <= throughIdx; i++ { pinIndexes = append(pinIndexes, i) } return s.preloadChunks(pinIndexes, fp, mss) }
// interpolateSamples interpolates a value at a target time between two // provided sample pairs. func interpolateSamples(first, second *model.SamplePair, timestamp model.Time) *model.SamplePair { dv := second.Value - first.Value dt := second.Timestamp.Sub(first.Timestamp) dDt := dv / model.SampleValue(dt) offset := model.SampleValue(timestamp.Sub(first.Timestamp)) return &model.SamplePair{ Value: first.Value + (offset * dDt), Timestamp: timestamp, } }
// findAtOrBefore implements Iterator. func (it *varbitChunkIterator) FindAtOrBefore(t model.Time) bool { if it.len == 0 || t.Before(it.c.FirstTime()) { return false } last := it.c.lastTime() if !t.Before(last) { it.t = last it.v = it.c.lastValue() it.pos = it.len + 1 return true } if t == it.t { return it.lastError == nil } if t.Before(it.t) || it.rewound { it.reset() } var ( prevT = model.Earliest prevV model.SampleValue ) for it.Scan() && t.After(it.t) { prevT = it.t prevV = it.v // TODO(beorn7): If we are in a repeat, we could iterate forward // much faster. } if t == it.t { return it.lastError == nil } it.rewind(prevT, prevV) return it.lastError == nil }
// preloadChunksForRange loads chunks for the given range from the persistence. // The caller must have locked the fingerprint of the series. func (s *memorySeries) preloadChunksForRange( from model.Time, through model.Time, fp model.Fingerprint, mss *memorySeriesStorage, ) ([]*chunkDesc, error) { firstChunkDescTime := model.Latest if len(s.chunkDescs) > 0 { firstChunkDescTime = s.chunkDescs[0].firstTime() } if s.chunkDescsOffset != 0 && from.Before(firstChunkDescTime) { cds, err := mss.loadChunkDescs(fp, s.persistWatermark) if err != nil { return nil, err } s.chunkDescs = append(cds, s.chunkDescs...) s.chunkDescsOffset = 0 s.persistWatermark += len(cds) } if len(s.chunkDescs) == 0 { return nil, nil } // Find first chunk with start time after "from". fromIdx := sort.Search(len(s.chunkDescs), func(i int) bool { return s.chunkDescs[i].firstTime().After(from) }) // Find first chunk with start time after "through". throughIdx := sort.Search(len(s.chunkDescs), func(i int) bool { return s.chunkDescs[i].firstTime().After(through) }) if fromIdx > 0 { fromIdx-- } if throughIdx == len(s.chunkDescs) { throughIdx-- } pinIndexes := make([]int, 0, throughIdx-fromIdx+1) for i := fromIdx; i <= throughIdx; i++ { pinIndexes = append(pinIndexes, i) } return s.preloadChunks(pinIndexes, fp, mss) }
// ValueAtTime implements SeriesIterator. func (it *memorySeriesIterator) ValueAtTime(t model.Time) []model.SamplePair { // The most common case. We are iterating through a chunk. if it.chunkIt != nil && it.chunkIt.contains(t) { return it.chunkIt.valueAtTime(t) } if len(it.chunks) == 0 { return nil } // Before or exactly on the first sample of the series. it.chunkIt = it.chunkIterator(0) ts := it.chunkIt.timestampAtIndex(0) if !t.After(ts) { // return first value of first chunk return []model.SamplePair{{ Timestamp: ts, Value: it.chunkIt.sampleValueAtIndex(0), }} } // After or exactly on the last sample of the series. it.chunkIt = it.chunkIterator(len(it.chunks) - 1) ts = it.chunkIt.lastTimestamp() if !t.Before(ts) { // return last value of last chunk return []model.SamplePair{{ Timestamp: ts, Value: it.chunkIt.sampleValueAtIndex(it.chunkIt.length() - 1), }} } // Find last chunk where firstTime() is before or equal to t. l := len(it.chunks) - 1 i := sort.Search(len(it.chunks), func(i int) bool { return !it.chunks[l-i].firstTime().After(t) }) if i == len(it.chunks) { panic("out of bounds") } it.chunkIt = it.chunkIterator(l - i) ts = it.chunkIt.lastTimestamp() if t.After(ts) { // We ended up between two chunks. sp1 := model.SamplePair{ Timestamp: ts, Value: it.chunkIt.sampleValueAtIndex(it.chunkIt.length() - 1), } it.chunkIt = it.chunkIterator(l - i + 1) return []model.SamplePair{ sp1, { Timestamp: it.chunkIt.timestampAtIndex(0), Value: it.chunkIt.sampleValueAtIndex(0), }, } } return it.chunkIt.valueAtTime(t) }
// metricForRange returns the metric for the given fingerprint if the // corresponding time series has samples between 'from' and 'through', together // with a pointer to the series if it is in memory already. For a series that // does not have samples between 'from' and 'through', the returned bool is // false. For an archived series that does contain samples between 'from' and // 'through', it returns (metric, nil, true). // // The caller must have locked the fp. func (s *memorySeriesStorage) metricForRange( fp model.Fingerprint, from, through model.Time, ) (model.Metric, *memorySeries, bool) { series, ok := s.fpToSeries.get(fp) if ok { if series.lastTime.Before(from) || series.firstTime().After(through) { return nil, nil, false } return series.metric, series, true } // From here on, we are only concerned with archived metrics. // If the high watermark of archived series is before 'from', we are done. watermark := model.Time(atomic.LoadInt64((*int64)(&s.archiveHighWatermark))) if watermark < from { return nil, nil, false } if from.After(model.Earliest) || through.Before(model.Latest) { // The range lookup is relatively cheap, so let's do it first if // we have a chance the archived metric is not in the range. has, first, last := s.persistence.hasArchivedMetric(fp) if !has { s.nonExistentSeriesMatchesCount.Inc() return nil, nil, false } if first.After(through) || last.Before(from) { return nil, nil, false } } metric, err := s.persistence.archivedMetric(fp) if err != nil { // archivedMetric has already flagged the storage as dirty in this case. return nil, nil, false } return metric, nil, true }
// findAtOrAfter implements Iterator. func (it *varbitChunkIterator) FindAtOrAfter(t model.Time) bool { if it.len == 0 || t.After(it.c.lastTime()) { return false } first := it.c.FirstTime() if !t.After(first) { it.reset() return it.Scan() } if t == it.t { return it.lastError == nil } if t.Before(it.t) { it.reset() } for it.Scan() && t.After(it.t) { // TODO(beorn7): If we are in a repeat, we could iterate forward // much faster. } return it.lastError == nil }
// eval evaluates the rule expression and then creates pending alerts and fires // or removes previously pending alerts accordingly. func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, error) { query, err := engine.NewInstantQuery(r.vector.String(), ts) if err != nil { return nil, err } res, err := query.Exec().Vector() if err != nil { return nil, err } r.mtx.Lock() defer r.mtx.Unlock() // Create pending alerts for any new vector elements in the alert expression // or update the expression value for existing elements. resultFPs := map[model.Fingerprint]struct{}{} for _, smpl := range res { fp := smpl.Metric.Fingerprint() resultFPs[fp] = struct{}{} if alert, ok := r.active[fp]; ok { alert.Value = smpl.Value continue } delete(smpl.Metric, model.MetricNameLabel) r.active[fp] = &Alert{ Labels: model.LabelSet(smpl.Metric), ActiveAt: ts, State: StatePending, Value: smpl.Value, } } var vec model.Vector // Check if any pending alerts should be removed or fire now. Write out alert timeseries. for fp, a := range r.active { if _, ok := resultFPs[fp]; !ok { if a.State != StateInactive { vec = append(vec, r.sample(a, ts, false)) } // If the alert was previously firing, keep it around for a given // retention time so it is reported as resolved to the AlertManager. if a.State == StatePending || (a.ResolvedAt != 0 && ts.Sub(a.ResolvedAt) > resolvedRetention) { delete(r.active, fp) } if a.State != StateInactive { a.State = StateInactive a.ResolvedAt = ts } continue } if a.State == StatePending && ts.Sub(a.ActiveAt) >= r.holdDuration { vec = append(vec, r.sample(a, ts, false)) a.State = StateFiring } vec = append(vec, r.sample(a, ts, true)) } return vec, nil }
// contains implements chunkIterator. func (it *doubleDeltaEncodedChunkIterator) contains(t model.Time) bool { return !t.Before(it.baseT) && !t.After(it.timestampAtIndex(it.len-1)) }
// contains implements Iterator. func (it *indexAccessingChunkIterator) Contains(t model.Time) (bool, error) { return !t.Before(it.acc.timestampAtIndex(0)) && !t.After(it.acc.timestampAtIndex(it.len-1)), it.acc.err() }
// dropAndPersistChunks deletes all chunks from a series file whose last sample // time is before beforeTime, and then appends the provided chunks, leaving out // those whose last sample time is before beforeTime. It returns the timestamp // of the first sample in the oldest chunk _not_ dropped, the offset within the // series file of the first chunk persisted (out of the provided chunks), the // number of deleted chunks, and true if all chunks of the series have been // deleted (in which case the returned timestamp will be 0 and must be ignored). // It is the caller's responsibility to make sure nothing is persisted or loaded // for the same fingerprint concurrently. // // Returning an error signals problems with the series file. In this case, the // caller should quarantine the series. func (p *persistence) dropAndPersistChunks( fp model.Fingerprint, beforeTime model.Time, chunks []chunk, ) ( firstTimeNotDropped model.Time, offset int, numDropped int, allDropped bool, err error, ) { // Style note: With the many return values, it was decided to use naked // returns in this method. They make the method more readable, but // please handle with care! if len(chunks) > 0 { // We have chunks to persist. First check if those are already // too old. If that's the case, the chunks in the series file // are all too old, too. i := 0 for ; i < len(chunks); i++ { var lt model.Time lt, err = chunks[i].newIterator().lastTimestamp() if err != nil { return } if !lt.Before(beforeTime) { break } } if i < len(chunks) { firstTimeNotDropped = chunks[i].firstTime() } if i > 0 || firstTimeNotDropped.Before(beforeTime) { // Series file has to go. if numDropped, err = p.deleteSeriesFile(fp); err != nil { return } numDropped += i if i == len(chunks) { allDropped = true return } // Now simply persist what has to be persisted to a new file. _, err = p.persistChunks(fp, chunks[i:]) return } } // If we are here, we have to check the series file itself. f, err := p.openChunkFileForReading(fp) if os.IsNotExist(err) { // No series file. Only need to create new file with chunks to // persist, if there are any. if len(chunks) == 0 { allDropped = true err = nil // Do not report not-exist err. return } offset, err = p.persistChunks(fp, chunks) return } if err != nil { return } defer f.Close() headerBuf := make([]byte, chunkHeaderLen) var firstTimeInFile model.Time // Find the first chunk in the file that should be kept. for ; ; numDropped++ { _, err = f.Seek(offsetForChunkIndex(numDropped), os.SEEK_SET) if err != nil { return } _, err = io.ReadFull(f, headerBuf) if err == io.EOF { // Close the file before trying to delete it. This is necessary on Windows // (this will cause the defer f.Close to fail, but the error is silently ignored) f.Close() // We ran into the end of the file without finding any chunks that should // be kept. Remove the whole file. if numDropped, err = p.deleteSeriesFile(fp); err != nil { return } if len(chunks) == 0 { allDropped = true return } offset, err = p.persistChunks(fp, chunks) return } if err != nil { return } if numDropped == 0 { firstTimeInFile = model.Time( binary.LittleEndian.Uint64(headerBuf[chunkHeaderFirstTimeOffset:]), ) } lastTime := model.Time( binary.LittleEndian.Uint64(headerBuf[chunkHeaderLastTimeOffset:]), ) if !lastTime.Before(beforeTime) { break } } // We've found the first chunk that should be kept. // First check if the shrink ratio is good enough to perform the the // actual drop or leave it for next time if it is not worth the effort. fi, err := f.Stat() if err != nil { return } totalChunks := int(fi.Size())/chunkLenWithHeader + len(chunks) if numDropped == 0 || float64(numDropped)/float64(totalChunks) < p.minShrinkRatio { // Nothing to drop. Just adjust the return values and append the chunks (if any). numDropped = 0 firstTimeNotDropped = firstTimeInFile if len(chunks) > 0 { offset, err = p.persistChunks(fp, chunks) } return } // If we are here, we have to drop some chunks for real. So we need to // record firstTimeNotDropped from the last read header, seek backwards // to the beginning of its header, and start copying everything from // there into a new file. Then append the chunks to the new file. firstTimeNotDropped = model.Time( binary.LittleEndian.Uint64(headerBuf[chunkHeaderFirstTimeOffset:]), ) chunkOps.WithLabelValues(drop).Add(float64(numDropped)) _, err = f.Seek(-chunkHeaderLen, os.SEEK_CUR) if err != nil { return } temp, err := os.OpenFile(p.tempFileNameForFingerprint(fp), os.O_WRONLY|os.O_CREATE, 0640) if err != nil { return } defer func() { // Close the file before trying to rename to it. This is necessary on Windows // (this will cause the defer f.Close to fail, but the error is silently ignored) f.Close() p.closeChunkFile(temp) if err == nil { err = os.Rename(p.tempFileNameForFingerprint(fp), p.fileNameForFingerprint(fp)) } }() written, err := io.Copy(temp, f) if err != nil { return } offset = int(written / chunkLenWithHeader) if len(chunks) > 0 { if err = p.writeChunks(temp, chunks); err != nil { return } } return }
func (cd *chunkDesc) contains(t model.Time) bool { return !t.Before(cd.firstTime()) && !t.After(cd.lastTime()) }
// eval evaluates the rule expression and then creates pending alerts and fires // or removes previously pending alerts accordingly. func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine, externalURLPath string) (model.Vector, error) { query, err := engine.NewInstantQuery(r.vector.String(), ts) if err != nil { return nil, err } res, err := query.Exec().Vector() if err != nil { return nil, err } r.mtx.Lock() defer r.mtx.Unlock() // Create pending alerts for any new vector elements in the alert expression // or update the expression value for existing elements. resultFPs := map[model.Fingerprint]struct{}{} for _, smpl := range res { // Provide the alert information to the template. l := make(map[string]string, len(smpl.Metric)) for k, v := range smpl.Metric { l[string(k)] = string(v) } tmplData := struct { Labels map[string]string Value float64 }{ Labels: l, Value: float64(smpl.Value), } // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. defs := "{{$labels := .Labels}}{{$value := .Value}}" expand := func(text model.LabelValue) model.LabelValue { tmpl := template.NewTemplateExpander( defs+string(text), "__alert_"+r.Name(), tmplData, ts, engine, externalURLPath, ) result, err := tmpl.Expand() if err != nil { result = fmt.Sprintf("<error expanding template: %s>", err) log.Warnf("Error expanding alert template %v with data '%v': %s", r.Name(), tmplData, err) } return model.LabelValue(result) } labels := make(model.LabelSet, len(smpl.Metric)+len(r.labels)+1) for ln, lv := range smpl.Metric { labels[ln] = lv } for ln, lv := range r.labels { labels[ln] = expand(lv) } labels[model.AlertNameLabel] = model.LabelValue(r.Name()) annotations := make(model.LabelSet, len(r.annotations)) for an, av := range r.annotations { annotations[an] = expand(av) } fp := smpl.Metric.Fingerprint() resultFPs[fp] = struct{}{} if alert, ok := r.active[fp]; ok && alert.State != StateInactive { alert.Value = smpl.Value continue } delete(smpl.Metric, model.MetricNameLabel) r.active[fp] = &Alert{ Labels: labels, Annotations: annotations, ActiveAt: ts, State: StatePending, Value: smpl.Value, } } var vec model.Vector // Check if any pending alerts should be removed or fire now. Write out alert timeseries. for fp, a := range r.active { if _, ok := resultFPs[fp]; !ok { if a.State != StateInactive { vec = append(vec, r.sample(a, ts, false)) } // If the alert was previously firing, keep it around for a given // retention time so it is reported as resolved to the AlertManager. if a.State == StatePending || (a.ResolvedAt != 0 && ts.Sub(a.ResolvedAt) > resolvedRetention) { delete(r.active, fp) } if a.State != StateInactive { a.State = StateInactive a.ResolvedAt = ts } continue } if a.State == StatePending && ts.Sub(a.ActiveAt) >= r.holdDuration { vec = append(vec, r.sample(a, ts, false)) a.State = StateFiring } vec = append(vec, r.sample(a, ts, true)) } return vec, nil }