func (c varbitChunk) firstTimeDelta() model.Time { // Only the first 3 bytes are actually the timestamp, so get rid of the // last one by bitshifting. return model.Time(c[varbitFirstTimeDeltaOffset+2]) | model.Time(c[varbitFirstTimeDeltaOffset+1])<<8 | model.Time(c[varbitFirstTimeDeltaOffset])<<16 }
// UnmarshalBinary implements encoding.BinaryUnmarshaler. func (tr *TimeRange) UnmarshalBinary(buf []byte) error { r := bytes.NewReader(buf) first, err := binary.ReadVarint(r) if err != nil { return err } last, err := binary.ReadVarint(r) if err != nil { return err } tr.First = model.Time(first) tr.Last = model.Time(last) return nil }
func (c doubleDeltaEncodedChunk) baseTime() model.Time { return model.Time( binary.LittleEndian.Uint64( c[doubleDeltaHeaderBaseTimeOffset:], ), ) }
func benchmarkRangeValues(b *testing.B, encoding chunkEncoding) { samples := make(model.Samples, 10000) for i := range samples { samples[i] = &model.Sample{ Timestamp: model.Time(2 * i), Value: model.SampleValue(float64(i) * 0.2), } } s, closer := NewTestStorage(b, encoding) defer closer.Close() for _, sample := range samples { s.Append(sample) } s.WaitForIndexing() fp := model.Metric{}.FastFingerprint() _, it := s.preloadChunksForRange(fp, model.Earliest, model.Latest) b.ResetTimer() for i := 0; i < b.N; i++ { for _, sample := range samples { actual := it.RangeValues(metric.Interval{ OldestInclusive: sample.Timestamp - 20, NewestInclusive: sample.Timestamp + 20, }) if len(actual) < 10 { b.Fatalf("not enough samples found") } } } }
func buildTestChunks(t *testing.T, encoding chunk.Encoding) map[model.Fingerprint][]chunk.Chunk { fps := model.Fingerprints{ m1.FastFingerprint(), m2.FastFingerprint(), m3.FastFingerprint(), } fpToChunks := map[model.Fingerprint][]chunk.Chunk{} for _, fp := range fps { fpToChunks[fp] = make([]chunk.Chunk, 0, 10) for i := 0; i < 10; i++ { ch, err := chunk.NewForEncoding(encoding) if err != nil { t.Fatal(err) } chs, err := ch.Add(model.SamplePair{ Timestamp: model.Time(i), Value: model.SampleValue(fp), }) if err != nil { t.Fatal(err) } fpToChunks[fp] = append(fpToChunks[fp], chs[0]) } } return fpToChunks }
func TestToReader(t *testing.T) { cntVec := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "name", Help: "docstring", ConstLabels: prometheus.Labels{"constname": "constvalue"}, }, []string{"labelname"}, ) cntVec.WithLabelValues("val1").Inc() cntVec.WithLabelValues("val2").Inc() reg := prometheus.NewRegistry() reg.MustRegister(cntVec) want := `prefix.name.constname.constvalue.labelname.val1 1 1477043 prefix.name.constname.constvalue.labelname.val2 1 1477043 ` mfs, err := reg.Gather() if err != nil { t.Fatalf("error: %v", err) } now := model.Time(1477043083) var buf bytes.Buffer err = writeMetrics(&buf, mfs, "prefix", now) if err != nil { t.Fatalf("error: %v", err) } if got := buf.String(); want != got { t.Fatalf("wanted \n%s\n, got \n%s\n", want, got) } }
func (c varbitChunk) lastTime() model.Time { return model.Time( binary.BigEndian.Uint64( c[varbitLastTimeOffset:], ), ) }
func TestSampleDeliveryOrder(t *testing.T) { cfg := defaultConfig ts := 10 n := cfg.MaxSamplesPerSend * ts // Ensure we don't drop samples in this test. cfg.QueueCapacity = n samples := make(model.Samples, 0, n) for i := 0; i < n; i++ { name := model.LabelValue(fmt.Sprintf("test_metric_%d", i%ts)) samples = append(samples, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: name, }, Value: model.SampleValue(i), Timestamp: model.Time(i), }) } c := NewTestStorageClient() c.expectSamples(samples) m := NewStorageQueueManager(c, &cfg) // These should be received by the client. for _, s := range samples { m.Append(s) } go m.Run() defer m.Stop() c.waitForExpectedSamples(t) }
// timestampAtIndex implements chunkIterator. func (it *deltaEncodedChunkIterator) timestampAtIndex(idx int) model.Time { offset := deltaHeaderBytes + idx*int(it.tBytes+it.vBytes) switch it.tBytes { case d1: return it.baseT + model.Time(uint8(it.c[offset])) case d2: return it.baseT + model.Time(binary.LittleEndian.Uint16(it.c[offset:])) case d4: return it.baseT + model.Time(binary.LittleEndian.Uint32(it.c[offset:])) case d8: // Take absolute value for d8. return model.Time(binary.LittleEndian.Uint64(it.c[offset:])) default: panic("invalid number of bytes for time delta") } }
// timestampAtIndex implements chunkIterator. func (it *doubleDeltaEncodedChunkIterator) timestampAtIndex(idx int) model.Time { if idx == 0 { return it.baseT } if idx == 1 { // If time bytes are at d8, the time is saved directly rather // than as a difference. if it.tBytes == d8 { return it.baseΔT } return it.baseT + it.baseΔT } offset := doubleDeltaHeaderBytes + (idx-2)*int(it.tBytes+it.vBytes) switch it.tBytes { case d1: return it.baseT + model.Time(idx)*it.baseΔT + model.Time(int8(it.c[offset])) case d2: return it.baseT + model.Time(idx)*it.baseΔT + model.Time(int16(binary.LittleEndian.Uint16(it.c[offset:]))) case d4: return it.baseT + model.Time(idx)*it.baseΔT + model.Time(int32(binary.LittleEndian.Uint32(it.c[offset:]))) case d8: // Take absolute value for d8. return model.Time(binary.LittleEndian.Uint64(it.c[offset:])) default: panic("invalid number of bytes for time delta") } }
func (acc *deltaEncodedIndexAccessor) timestampAtIndex(idx int) model.Time { offset := deltaHeaderBytes + idx*int(acc.tBytes+acc.vBytes) switch acc.tBytes { case d1: return acc.baseT + model.Time(uint8(acc.c[offset])) case d2: return acc.baseT + model.Time(binary.LittleEndian.Uint16(acc.c[offset:])) case d4: return acc.baseT + model.Time(binary.LittleEndian.Uint32(acc.c[offset:])) case d8: // Take absolute value for d8. return model.Time(binary.LittleEndian.Uint64(acc.c[offset:])) default: acc.lastErr = fmt.Errorf("invalid number of bytes for time delta: %d", acc.tBytes) return model.Earliest } }
// loadChunkDescs loads the chunkDescs for a series from disk. offsetFromEnd is // the number of chunkDescs to skip from the end of the series file. It is the // caller's responsibility to not persist or drop anything for the same // fingerprint concurrently. func (p *persistence) loadChunkDescs(fp model.Fingerprint, offsetFromEnd int) ([]*chunkDesc, error) { f, err := p.openChunkFileForReading(fp) if os.IsNotExist(err) { return nil, nil } if err != nil { return nil, err } defer f.Close() fi, err := f.Stat() if err != nil { return nil, err } if fi.Size()%int64(chunkLenWithHeader) != 0 { // The returned error will bubble up and lead to quarantining of the whole series. return nil, fmt.Errorf( "size of series file for fingerprint %v is %d, which is not a multiple of the chunk length %d", fp, fi.Size(), chunkLenWithHeader, ) } numChunks := int(fi.Size())/chunkLenWithHeader - offsetFromEnd cds := make([]*chunkDesc, numChunks) chunkTimesBuf := make([]byte, 16) for i := 0; i < numChunks; i++ { _, err := f.Seek(offsetForChunkIndex(i)+chunkHeaderFirstTimeOffset, os.SEEK_SET) if err != nil { return nil, err } _, err = io.ReadAtLeast(f, chunkTimesBuf, 16) if err != nil { return nil, err } cds[i] = &chunkDesc{ chunkFirstTime: model.Time(binary.LittleEndian.Uint64(chunkTimesBuf)), chunkLastTime: model.Time(binary.LittleEndian.Uint64(chunkTimesBuf[8:])), } } chunkDescOps.WithLabelValues(load).Add(float64(len(cds))) numMemChunkDescs.Add(float64(len(cds))) return cds, nil }
func (c doubleDeltaEncodedChunk) baseTimeDelta() model.Time { if len(c) < doubleDeltaHeaderBaseTimeDeltaOffset+8 { return 0 } return model.Time( binary.LittleEndian.Uint64( c[doubleDeltaHeaderBaseTimeDeltaOffset:], ), ) }
func (it *varbitChunkIterator) readDDT() { if it.repeats > 0 { it.repeats-- } else { switch it.readControlBits(3) { case 0: it.repeats = byte(it.readBitPattern(7)) case 1: it.dT += model.Time(it.readSignedInt(6)) case 2: it.dT += model.Time(it.readSignedInt(17)) case 3: it.dT += model.Time(it.readSignedInt(23)) default: panic("unexpected number of control bits") } } it.t += it.dT }
func TestWriteHistogram(t *testing.T) { histVec := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "name", Help: "docstring", ConstLabels: prometheus.Labels{"constname": "constvalue"}, Buckets: []float64{0.01, 0.02, 0.05, 0.1}, }, []string{"labelname"}, ) histVec.WithLabelValues("val1").Observe(float64(10)) histVec.WithLabelValues("val1").Observe(float64(20)) histVec.WithLabelValues("val1").Observe(float64(30)) histVec.WithLabelValues("val2").Observe(float64(20)) histVec.WithLabelValues("val2").Observe(float64(30)) histVec.WithLabelValues("val2").Observe(float64(40)) reg := prometheus.NewRegistry() reg.MustRegister(histVec) mfs, err := reg.Gather() if err != nil { t.Fatalf("error: %v", err) } now := model.Time(1477043083) var buf bytes.Buffer err = writeMetrics(&buf, mfs, "prefix", now) if err != nil { t.Fatalf("error: %v", err) } want := `prefix.name_bucket.constname.constvalue.labelname.val1.le.0_01 0 1477043 prefix.name_bucket.constname.constvalue.labelname.val1.le.0_02 0 1477043 prefix.name_bucket.constname.constvalue.labelname.val1.le.0_05 0 1477043 prefix.name_bucket.constname.constvalue.labelname.val1.le.0_1 0 1477043 prefix.name_sum.constname.constvalue.labelname.val1 60 1477043 prefix.name_count.constname.constvalue.labelname.val1 3 1477043 prefix.name_bucket.constname.constvalue.labelname.val1.le._Inf 3 1477043 prefix.name_bucket.constname.constvalue.labelname.val2.le.0_01 0 1477043 prefix.name_bucket.constname.constvalue.labelname.val2.le.0_02 0 1477043 prefix.name_bucket.constname.constvalue.labelname.val2.le.0_05 0 1477043 prefix.name_bucket.constname.constvalue.labelname.val2.le.0_1 0 1477043 prefix.name_sum.constname.constvalue.labelname.val2 90 1477043 prefix.name_count.constname.constvalue.labelname.val2 3 1477043 prefix.name_bucket.constname.constvalue.labelname.val2.le._Inf 3 1477043 ` if got := buf.String(); want != got { t.Fatalf("wanted \n%s\n, got \n%s\n", want, got) } }
func TestWriteSummary(t *testing.T) { sumVec := prometheus.NewSummaryVec( prometheus.SummaryOpts{ Name: "name", Help: "docstring", ConstLabels: prometheus.Labels{"constname": "constvalue"}, Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, []string{"labelname"}, ) sumVec.WithLabelValues("val1").Observe(float64(10)) sumVec.WithLabelValues("val1").Observe(float64(20)) sumVec.WithLabelValues("val1").Observe(float64(30)) sumVec.WithLabelValues("val2").Observe(float64(20)) sumVec.WithLabelValues("val2").Observe(float64(30)) sumVec.WithLabelValues("val2").Observe(float64(40)) reg := prometheus.NewRegistry() reg.MustRegister(sumVec) mfs, err := reg.Gather() if err != nil { t.Fatalf("error: %v", err) } now := model.Time(1477043083) var buf bytes.Buffer err = writeMetrics(&buf, mfs, "prefix", now) if err != nil { t.Fatalf("error: %v", err) } want := `prefix.name.constname.constvalue.labelname.val1.quantile.0_5 20 1477043 prefix.name.constname.constvalue.labelname.val1.quantile.0_9 30 1477043 prefix.name.constname.constvalue.labelname.val1.quantile.0_99 30 1477043 prefix.name_sum.constname.constvalue.labelname.val1 60 1477043 prefix.name_count.constname.constvalue.labelname.val1 3 1477043 prefix.name.constname.constvalue.labelname.val2.quantile.0_5 30 1477043 prefix.name.constname.constvalue.labelname.val2.quantile.0_9 40 1477043 prefix.name.constname.constvalue.labelname.val2.quantile.0_99 40 1477043 prefix.name_sum.constname.constvalue.labelname.val2 90 1477043 prefix.name_count.constname.constvalue.labelname.val2 3 1477043 ` if got := buf.String(); want != got { t.Fatalf("wanted \n%s\n, got \n%s\n", want, got) } }
func TestAppendOutOfOrder(t *testing.T) { s, closer := NewTestStorage(t, 1) defer closer.Close() m := model.Metric{ model.MetricNameLabel: "out_of_order", } for i, t := range []int{0, 2, 2, 1} { s.Append(&model.Sample{ Metric: m, Timestamp: model.Time(t), Value: model.SampleValue(i), }) } fp, err := s.mapper.mapFP(m.FastFingerprint(), m) if err != nil { t.Fatal(err) } pl := s.NewPreloader() defer pl.Close() err = pl.PreloadRange(fp, 0, 2, 5*time.Minute) if err != nil { t.Fatalf("Error preloading chunks: %s", err) } it := s.NewIterator(fp) want := []model.SamplePair{ { Timestamp: 0, Value: 0, }, { Timestamp: 2, Value: 1, }, } got := it.RangeValues(metric.Interval{OldestInclusive: 0, NewestInclusive: 2}) if !reflect.DeepEqual(want, got) { t.Fatalf("want %v, got %v", want, got) } }
func testChunk(t *testing.T, encoding chunkEncoding) { samples := make(model.Samples, 500000) for i := range samples { samples[i] = &model.Sample{ Timestamp: model.Time(i), Value: model.SampleValue(float64(i) * 0.2), } } s, closer := NewTestStorage(t, encoding) defer closer.Close() for _, sample := range samples { s.Append(sample) } s.WaitForIndexing() for m := range s.fpToSeries.iter() { s.fpLocker.Lock(m.fp) defer s.fpLocker.Unlock(m.fp) // TODO remove, see below var values []model.SamplePair for _, cd := range m.series.chunkDescs { if cd.isEvicted() { continue } it := cd.c.newIterator() for it.scan() { values = append(values, it.value()) } if it.err() != nil { t.Error(it.err()) } } for i, v := range values { if samples[i].Timestamp != v.Timestamp { t.Errorf("%d. Got %v; want %v", i, v.Timestamp, samples[i].Timestamp) } if samples[i].Value != v.Value { t.Errorf("%d. Got %v; want %v", i, v.Value, samples[i].Value) } } //s.fpLocker.Unlock(m.fp) } log.Info("test done, closing") }
func buildTestChunks(encoding chunkEncoding) map[model.Fingerprint][]chunk { fps := model.Fingerprints{ m1.FastFingerprint(), m2.FastFingerprint(), m3.FastFingerprint(), } fpToChunks := map[model.Fingerprint][]chunk{} for _, fp := range fps { fpToChunks[fp] = make([]chunk, 0, 10) for i := 0; i < 10; i++ { fpToChunks[fp] = append(fpToChunks[fp], newChunkForEncoding(encoding).add(&model.SamplePair{ Timestamp: model.Time(i), Value: model.SampleValue(fp), })[0]) } } return fpToChunks }
// TestLoop is just a smoke test for the loop method, if we can switch it on and // off without disaster. func TestLoop(t *testing.T) { if testing.Short() { t.Skip("Skipping test in short mode.") } samples := make(model.Samples, 1000) for i := range samples { samples[i] = &model.Sample{ Timestamp: model.Time(2 * i), Value: model.SampleValue(float64(i) * 0.2), } } directory := testutil.NewTemporaryDirectory("test_storage", t) defer directory.Close() o := &MemorySeriesStorageOptions{ MemoryChunks: 50, MaxChunksToPersist: 1000000, PersistenceRetentionPeriod: 24 * 7 * time.Hour, PersistenceStoragePath: directory.Path(), CheckpointInterval: 250 * time.Millisecond, SyncStrategy: Adaptive, MinShrinkRatio: 0.1, } storage := NewMemorySeriesStorage(o) if err := storage.Start(); err != nil { t.Errorf("Error starting storage: %s", err) } for _, s := range samples { storage.Append(s) } storage.WaitForIndexing() series, _ := storage.(*memorySeriesStorage).fpToSeries.get(model.Metric{}.FastFingerprint()) cdsBefore := len(series.chunkDescs) time.Sleep(fpMaxWaitDuration + time.Second) // TODO(beorn7): Ugh, need to wait for maintenance to kick in. cdsAfter := len(series.chunkDescs) storage.Stop() if cdsBefore <= cdsAfter { t.Errorf( "Number of chunk descriptors should have gone down by now. Got before %d, after %d.", cdsBefore, cdsAfter, ) } }
func benchmarkAppend(b *testing.B, encoding chunkEncoding) { samples := make(model.Samples, b.N) for i := range samples { samples[i] = &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: model.LabelValue(fmt.Sprintf("test_metric_%d", i%10)), "label1": model.LabelValue(fmt.Sprintf("test_metric_%d", i%10)), "label2": model.LabelValue(fmt.Sprintf("test_metric_%d", i%10)), }, Timestamp: model.Time(i), Value: model.SampleValue(i), } } b.ResetTimer() s, closer := NewTestStorage(b, encoding) defer closer.Close() for _, sample := range samples { s.Append(sample) } }
// linearRegression performs a least-square linear regression analysis on the // provided SamplePairs. It returns the slope, and the intercept value at the // provided time. func linearRegression(samples []model.SamplePair, interceptTime model.Time) (slope, intercept model.SampleValue) { var ( n model.SampleValue sumX, sumY model.SampleValue sumXY, sumX2 model.SampleValue ) for _, sample := range samples { x := model.SampleValue( model.Time(sample.Timestamp-interceptTime).UnixNano(), ) / 1e9 n += 1.0 sumY += sample.Value sumX += x sumXY += x * sample.Value sumX2 += x * x } covXY := sumXY - sumX*sumY/n varX := sumX2 - sumX*sumX/n slope = covXY / varX intercept = sumY/n - slope*sumX/n return slope, intercept }
// metricForRange returns the metric for the given fingerprint if the // corresponding time series has samples between 'from' and 'through', together // with a pointer to the series if it is in memory already. For a series that // does not have samples between 'from' and 'through', the returned bool is // false. For an archived series that does contain samples between 'from' and // 'through', it returns (metric, nil, true). // // The caller must have locked the fp. func (s *memorySeriesStorage) metricForRange( fp model.Fingerprint, from, through model.Time, ) (model.Metric, *memorySeries, bool) { series, ok := s.fpToSeries.get(fp) if ok { if series.lastTime.Before(from) || series.firstTime().After(through) { return nil, nil, false } return series.metric, series, true } // From here on, we are only concerned with archived metrics. // If the high watermark of archived series is before 'from', we are done. watermark := model.Time(atomic.LoadInt64((*int64)(&s.archiveHighWatermark))) if watermark < from { return nil, nil, false } if from.After(model.Earliest) || through.Before(model.Latest) { // The range lookup is relatively cheap, so let's do it first if // we have a chance the archived metric is not in the range. has, first, last := s.persistence.hasArchivedMetric(fp) if !has { s.nonExistentSeriesMatchesCount.Inc() return nil, nil, false } if first.After(through) || last.Before(from) { return nil, nil, false } } metric, err := s.persistence.archivedMetric(fp) if err != nil { // archivedMetric has already flagged the storage as dirty in this case. return nil, nil, false } return metric, nil, true }
func TestLen(t *testing.T) { chunks := []Chunk{} for _, encoding := range []Encoding{Delta, DoubleDelta, Varbit} { c, err := NewForEncoding(encoding) if err != nil { t.Fatal(err) } chunks = append(chunks, c) } for _, c := range chunks { for i := 0; i <= 10; i++ { if c.Len() != i { t.Errorf("chunk type %s should have %d samples, had %d", c.Encoding(), i, c.Len()) } cs, _ := c.Add(model.SamplePair{ Timestamp: model.Time(i), Value: model.SampleValue(i), }) c = cs[0] } } }
func (acc *doubleDeltaEncodedIndexAccessor) timestampAtIndex(idx int) model.Time { if idx == 0 { return acc.baseT } if idx == 1 { // If time bytes are at d8, the time is saved directly rather // than as a difference. if acc.tBytes == d8 { return acc.baseΔT } return acc.baseT + acc.baseΔT } offset := doubleDeltaHeaderBytes + (idx-2)*int(acc.tBytes+acc.vBytes) switch acc.tBytes { case d1: return acc.baseT + model.Time(idx)*acc.baseΔT + model.Time(int8(acc.c[offset])) case d2: return acc.baseT + model.Time(idx)*acc.baseΔT + model.Time(int16(binary.LittleEndian.Uint16(acc.c[offset:]))) case d4: return acc.baseT + model.Time(idx)*acc.baseΔT + model.Time(int32(binary.LittleEndian.Uint32(acc.c[offset:]))) case d8: // Take absolute value for d8. return model.Time(binary.LittleEndian.Uint64(acc.c[offset:])) default: acc.lastErr = fmt.Errorf("invalid number of bytes for time delta: %d", acc.tBytes) return model.Earliest } }
// Add implements chunk. func (c doubleDeltaEncodedChunk) Add(s model.SamplePair) ([]Chunk, error) { // TODO(beorn7): Since we return &c, this method might cause an unnecessary allocation. if c.len() == 0 { return c.addFirstSample(s), nil } tb := c.timeBytes() vb := c.valueBytes() if c.len() == 1 { return c.addSecondSample(s, tb, vb) } remainingBytes := cap(c) - len(c) sampleSize := c.sampleSize() // Do we generally have space for another sample in this chunk? If not, // overflow into a new one. if remainingBytes < sampleSize { return addToOverflowChunk(&c, s) } projectedTime := c.baseTime() + model.Time(c.len())*c.baseTimeDelta() ddt := s.Timestamp - projectedTime projectedValue := c.baseValue() + model.SampleValue(c.len())*c.baseValueDelta() ddv := s.Value - projectedValue ntb, nvb, nInt := tb, vb, c.isInt() // If the new sample is incompatible with the current encoding, reencode the // existing chunk data into new chunk(s). if c.isInt() && !isInt64(ddv) { // int->float. nvb = d4 nInt = false } else if !c.isInt() && vb == d4 && projectedValue+model.SampleValue(float32(ddv)) != s.Value { // float32->float64. nvb = d8 } else { if tb < d8 { // Maybe more bytes for timestamp. ntb = max(tb, bytesNeededForSignedTimestampDelta(ddt)) } if c.isInt() && vb < d8 { // Maybe more bytes for sample value. nvb = max(vb, bytesNeededForIntegerSampleValueDelta(ddv)) } } if tb != ntb || vb != nvb || c.isInt() != nInt { if len(c)*2 < cap(c) { return transcodeAndAdd(newDoubleDeltaEncodedChunk(ntb, nvb, nInt, cap(c)), &c, s) } // Chunk is already half full. Better create a new one and save the transcoding efforts. return addToOverflowChunk(&c, s) } offset := len(c) c = c[:offset+sampleSize] switch tb { case d1: c[offset] = byte(ddt) case d2: binary.LittleEndian.PutUint16(c[offset:], uint16(ddt)) case d4: binary.LittleEndian.PutUint32(c[offset:], uint32(ddt)) case d8: // Store the absolute value (no delta) in case of d8. binary.LittleEndian.PutUint64(c[offset:], uint64(s.Timestamp)) default: return nil, fmt.Errorf("invalid number of bytes for time delta: %d", tb) } offset += int(tb) if c.isInt() { switch vb { case d0: // No-op. Constant delta is stored as base value. case d1: c[offset] = byte(int8(ddv)) case d2: binary.LittleEndian.PutUint16(c[offset:], uint16(int16(ddv))) case d4: binary.LittleEndian.PutUint32(c[offset:], uint32(int32(ddv))) // d8 must not happen. Those samples are encoded as float64. default: return nil, fmt.Errorf("invalid number of bytes for integer delta: %d", vb) } } else { switch vb { case d4: binary.LittleEndian.PutUint32(c[offset:], math.Float32bits(float32(ddv))) case d8: // Store the absolute value (no delta) in case of d8. binary.LittleEndian.PutUint64(c[offset:], math.Float64bits(float64(s.Value))) default: return nil, fmt.Errorf("invalid number of bytes for floating point delta: %d", vb) } } return []Chunk{&c}, nil }
func testCheckpointAndLoadSeriesMapAndHeads(t *testing.T, encoding chunk.Encoding) { p, closer := newTestPersistence(t, encoding) defer closer.Close() fpLocker := newFingerprintLocker(10) sm := newSeriesMap() s1, _ := newMemorySeries(m1, nil, time.Time{}) s2, _ := newMemorySeries(m2, nil, time.Time{}) s3, _ := newMemorySeries(m3, nil, time.Time{}) s4, _ := newMemorySeries(m4, nil, time.Time{}) s5, _ := newMemorySeries(m5, nil, time.Time{}) s1.add(model.SamplePair{Timestamp: 1, Value: 3.14}) s3.add(model.SamplePair{Timestamp: 2, Value: 2.7}) s3.headChunkClosed = true s3.persistWatermark = 1 for i := 0; i < 10000; i++ { s4.add(model.SamplePair{ Timestamp: model.Time(i), Value: model.SampleValue(i) / 2, }) s5.add(model.SamplePair{ Timestamp: model.Time(i), Value: model.SampleValue(i * i), }) } s5.persistWatermark = 3 chunkCountS4 := len(s4.chunkDescs) chunkCountS5 := len(s5.chunkDescs) sm.put(m1.FastFingerprint(), s1) sm.put(m2.FastFingerprint(), s2) sm.put(m3.FastFingerprint(), s3) sm.put(m4.FastFingerprint(), s4) sm.put(m5.FastFingerprint(), s5) if err := p.checkpointSeriesMapAndHeads(sm, fpLocker); err != nil { t.Fatal(err) } loadedSM, _, err := p.loadSeriesMapAndHeads() if err != nil { t.Fatal(err) } if loadedSM.length() != 4 { t.Errorf("want 4 series in map, got %d", loadedSM.length()) } if loadedS1, ok := loadedSM.get(m1.FastFingerprint()); ok { if !reflect.DeepEqual(loadedS1.metric, m1) { t.Errorf("want metric %v, got %v", m1, loadedS1.metric) } if !reflect.DeepEqual(loadedS1.head().C, s1.head().C) { t.Error("head chunks differ") } if loadedS1.chunkDescsOffset != 0 { t.Errorf("want chunkDescsOffset 0, got %d", loadedS1.chunkDescsOffset) } if loadedS1.headChunkClosed { t.Error("headChunkClosed is true") } if loadedS1.head().ChunkFirstTime != 1 { t.Errorf("want ChunkFirstTime in head chunk to be 1, got %d", loadedS1.head().ChunkFirstTime) } if loadedS1.head().ChunkLastTime != model.Earliest { t.Error("want ChunkLastTime in head chunk to be unset") } } else { t.Errorf("couldn't find %v in loaded map", m1) } if loadedS3, ok := loadedSM.get(m3.FastFingerprint()); ok { if !reflect.DeepEqual(loadedS3.metric, m3) { t.Errorf("want metric %v, got %v", m3, loadedS3.metric) } if loadedS3.head().C != nil { t.Error("head chunk not evicted") } if loadedS3.chunkDescsOffset != 0 { t.Errorf("want chunkDescsOffset 0, got %d", loadedS3.chunkDescsOffset) } if !loadedS3.headChunkClosed { t.Error("headChunkClosed is false") } if loadedS3.head().ChunkFirstTime != 2 { t.Errorf("want ChunkFirstTime in head chunk to be 2, got %d", loadedS3.head().ChunkFirstTime) } if loadedS3.head().ChunkLastTime != 2 { t.Errorf("want ChunkLastTime in head chunk to be 2, got %d", loadedS3.head().ChunkLastTime) } } else { t.Errorf("couldn't find %v in loaded map", m3) } if loadedS4, ok := loadedSM.get(m4.FastFingerprint()); ok { if !reflect.DeepEqual(loadedS4.metric, m4) { t.Errorf("want metric %v, got %v", m4, loadedS4.metric) } if got, want := len(loadedS4.chunkDescs), chunkCountS4; got != want { t.Errorf("got %d chunkDescs, want %d", got, want) } if got, want := loadedS4.persistWatermark, 0; got != want { t.Errorf("got persistWatermark %d, want %d", got, want) } if loadedS4.chunkDescs[2].IsEvicted() { t.Error("3rd chunk evicted") } if loadedS4.chunkDescs[3].IsEvicted() { t.Error("4th chunk evicted") } if loadedS4.chunkDescsOffset != 0 { t.Errorf("want chunkDescsOffset 0, got %d", loadedS4.chunkDescsOffset) } if loadedS4.headChunkClosed { t.Error("headChunkClosed is true") } for i, cd := range loadedS4.chunkDescs { if cd.ChunkFirstTime != cd.C.FirstTime() { t.Errorf( "chunk.Desc[%d]: ChunkFirstTime not consistent with chunk, want %d, got %d", i, cd.C.FirstTime(), cd.ChunkFirstTime, ) } if i == len(loadedS4.chunkDescs)-1 { // Head chunk. if cd.ChunkLastTime != model.Earliest { t.Error("want ChunkLastTime in head chunk to be unset") } continue } lastTime, err := cd.C.NewIterator().LastTimestamp() if err != nil { t.Fatal(err) } if cd.ChunkLastTime != lastTime { t.Errorf( "chunk.Desc[%d]: ChunkLastTime not consistent with chunk, want %d, got %d", i, lastTime, cd.ChunkLastTime, ) } } } else { t.Errorf("couldn't find %v in loaded map", m4) } if loadedS5, ok := loadedSM.get(m5.FastFingerprint()); ok { if !reflect.DeepEqual(loadedS5.metric, m5) { t.Errorf("want metric %v, got %v", m5, loadedS5.metric) } if got, want := len(loadedS5.chunkDescs), chunkCountS5; got != want { t.Errorf("got %d chunkDescs, want %d", got, want) } if got, want := loadedS5.persistWatermark, 3; got != want { t.Errorf("got persistWatermark %d, want %d", got, want) } if !loadedS5.chunkDescs[2].IsEvicted() { t.Error("3rd chunk not evicted") } if loadedS5.chunkDescs[3].IsEvicted() { t.Error("4th chunk evicted") } if loadedS5.chunkDescsOffset != 0 { t.Errorf("want chunkDescsOffset 0, got %d", loadedS5.chunkDescsOffset) } if loadedS5.headChunkClosed { t.Error("headChunkClosed is true") } for i, cd := range loadedS5.chunkDescs { if i < 3 { // Evicted chunks. if cd.ChunkFirstTime == model.Earliest { t.Errorf("chunk.Desc[%d]: ChunkLastTime not set", i) } continue } if cd.ChunkFirstTime != cd.C.FirstTime() { t.Errorf( "chunk.Desc[%d]: ChunkFirstTime not consistent with chunk, want %d, got %d", i, cd.C.FirstTime(), cd.ChunkFirstTime, ) } if i == len(loadedS5.chunkDescs)-1 { // Head chunk. if cd.ChunkLastTime != model.Earliest { t.Error("want ChunkLastTime in head chunk to be unset") } continue } lastTime, err := cd.C.NewIterator().LastTimestamp() if err != nil { t.Fatal(err) } if cd.ChunkLastTime != lastTime { t.Errorf( "chunk.Desc[%d]: ChunkLastTime not consistent with chunk, want %d, got %d", i, cd.ChunkLastTime, lastTime, ) } } } else { t.Errorf("couldn't find %v in loaded map", m5) } }
func benchmarkValueAtTime(b *testing.B, encoding chunkEncoding) { samples := make(model.Samples, 10000) for i := range samples { samples[i] = &model.Sample{ Timestamp: model.Time(2 * i), Value: model.SampleValue(float64(i) * 0.2), } } s, closer := NewTestStorage(b, encoding) defer closer.Close() for _, sample := range samples { s.Append(sample) } s.WaitForIndexing() fp := model.Metric{}.FastFingerprint() b.ResetTimer() for i := 0; i < b.N; i++ { it := s.NewIterator(fp) // #1 Exactly on a sample. for i, expected := range samples { actual := it.ValueAtTime(expected.Timestamp) if len(actual) != 1 { b.Fatalf("1.%d. Expected exactly one result, got %d.", i, len(actual)) } if expected.Timestamp != actual[0].Timestamp { b.Errorf("1.%d. Got %v; want %v", i, actual[0].Timestamp, expected.Timestamp) } if expected.Value != actual[0].Value { b.Errorf("1.%d. Got %v; want %v", i, actual[0].Value, expected.Value) } } // #2 Between samples. for i, expected1 := range samples { if i == len(samples)-1 { continue } expected2 := samples[i+1] actual := it.ValueAtTime(expected1.Timestamp + 1) if len(actual) != 2 { b.Fatalf("2.%d. Expected exactly 2 results, got %d.", i, len(actual)) } if expected1.Timestamp != actual[0].Timestamp { b.Errorf("2.%d. Got %v; want %v", i, actual[0].Timestamp, expected1.Timestamp) } if expected1.Value != actual[0].Value { b.Errorf("2.%d. Got %v; want %v", i, actual[0].Value, expected1.Value) } if expected2.Timestamp != actual[1].Timestamp { b.Errorf("2.%d. Got %v; want %v", i, actual[1].Timestamp, expected1.Timestamp) } if expected2.Value != actual[1].Value { b.Errorf("2.%d. Got %v; want %v", i, actual[1].Value, expected1.Value) } } } }
func testPersistLoadDropChunks(t *testing.T, encoding chunk.Encoding) { p, closer := newTestPersistence(t, encoding) defer closer.Close() fpToChunks := buildTestChunks(t, encoding) for fp, chunks := range fpToChunks { firstTimeNotDropped, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, model.Earliest, chunks) if err != nil { t.Fatal(err) } if got, want := firstTimeNotDropped, model.Time(0); got != want { t.Errorf("Want firstTimeNotDropped %v, got %v.", got, want) } if got, want := offset, 0; got != want { t.Errorf("Want offset %v, got %v.", got, want) } if got, want := numDropped, 0; got != want { t.Errorf("Want numDropped %v, got %v.", got, want) } if allDropped { t.Error("All dropped.") } } for fp, expectedChunks := range fpToChunks { indexes := make([]int, 0, len(expectedChunks)) for i := range expectedChunks { indexes = append(indexes, i) } actualChunks, err := p.loadChunks(fp, indexes, 0) if err != nil { t.Fatal(err) } for _, i := range indexes { if !chunksEqual(expectedChunks[i], actualChunks[i]) { t.Errorf("%d. Chunks not equal.", i) } } // Load all chunk descs. actualChunkDescs, err := p.loadChunkDescs(fp, 0) if len(actualChunkDescs) != 10 { t.Errorf("Got %d chunkDescs, want %d.", len(actualChunkDescs), 10) } for i, cd := range actualChunkDescs { lastTime, err := cd.LastTime() if err != nil { t.Fatal(err) } if cd.FirstTime() != model.Time(i) || lastTime != model.Time(i) { t.Errorf( "Want ts=%v, got firstTime=%v, lastTime=%v.", i, cd.FirstTime(), lastTime, ) } } // Load chunk descs partially. actualChunkDescs, err = p.loadChunkDescs(fp, 5) if err != nil { t.Fatal(err) } if len(actualChunkDescs) != 5 { t.Errorf("Got %d chunkDescs, want %d.", len(actualChunkDescs), 5) } for i, cd := range actualChunkDescs { lastTime, err := cd.LastTime() if err != nil { t.Fatal(err) } if cd.FirstTime() != model.Time(i) || lastTime != model.Time(i) { t.Errorf( "Want ts=%v, got firstTime=%v, lastTime=%v.", i, cd.FirstTime(), lastTime, ) } } } // Drop half of the chunks. for fp, expectedChunks := range fpToChunks { firstTime, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, 5, nil) if err != nil { t.Fatal(err) } if offset != 5 { t.Errorf("want offset 5, got %d", offset) } if firstTime != 5 { t.Errorf("want first time 5, got %d", firstTime) } if numDropped != 5 { t.Errorf("want 5 dropped chunks, got %v", numDropped) } if allDropped { t.Error("all chunks dropped") } indexes := make([]int, 5) for i := range indexes { indexes[i] = i } actualChunks, err := p.loadChunks(fp, indexes, 0) if err != nil { t.Fatal(err) } for _, i := range indexes { if !chunksEqual(expectedChunks[i+5], actualChunks[i]) { t.Errorf("%d. Chunks not equal.", i) } } } // Drop all the chunks. for fp := range fpToChunks { firstTime, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, 100, nil) if firstTime != 0 { t.Errorf("want first time 0, got %d", firstTime) } if err != nil { t.Fatal(err) } if offset != 0 { t.Errorf("want offset 0, got %d", offset) } if numDropped != 5 { t.Errorf("want 5 dropped chunks, got %v", numDropped) } if !allDropped { t.Error("not all chunks dropped") } } // Re-add first two of the chunks. for fp, chunks := range fpToChunks { firstTimeNotDropped, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, model.Earliest, chunks[:2]) if err != nil { t.Fatal(err) } if got, want := firstTimeNotDropped, model.Time(0); got != want { t.Errorf("Want firstTimeNotDropped %v, got %v.", got, want) } if got, want := offset, 0; got != want { t.Errorf("Want offset %v, got %v.", got, want) } if got, want := numDropped, 0; got != want { t.Errorf("Want numDropped %v, got %v.", got, want) } if allDropped { t.Error("All dropped.") } } // Drop the first of the chunks while adding two more. for fp, chunks := range fpToChunks { firstTime, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, 1, chunks[2:4]) if err != nil { t.Fatal(err) } if offset != 1 { t.Errorf("want offset 1, got %d", offset) } if firstTime != 1 { t.Errorf("want first time 1, got %d", firstTime) } if numDropped != 1 { t.Errorf("want 1 dropped chunk, got %v", numDropped) } if allDropped { t.Error("all chunks dropped") } wantChunks := chunks[1:4] indexes := make([]int, len(wantChunks)) for i := range indexes { indexes[i] = i } gotChunks, err := p.loadChunks(fp, indexes, 0) if err != nil { t.Fatal(err) } for i, wantChunk := range wantChunks { if !chunksEqual(wantChunk, gotChunks[i]) { t.Errorf("%d. Chunks not equal.", i) } } } // Drop all the chunks while adding two more. for fp, chunks := range fpToChunks { firstTime, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, 4, chunks[4:6]) if err != nil { t.Fatal(err) } if offset != 0 { t.Errorf("want offset 0, got %d", offset) } if firstTime != 4 { t.Errorf("want first time 4, got %d", firstTime) } if numDropped != 3 { t.Errorf("want 3 dropped chunks, got %v", numDropped) } if allDropped { t.Error("all chunks dropped") } wantChunks := chunks[4:6] indexes := make([]int, len(wantChunks)) for i := range indexes { indexes[i] = i } gotChunks, err := p.loadChunks(fp, indexes, 0) if err != nil { t.Fatal(err) } for i, wantChunk := range wantChunks { if !chunksEqual(wantChunk, gotChunks[i]) { t.Errorf("%d. Chunks not equal.", i) } } } // While adding two more, drop all but one of the added ones. for fp, chunks := range fpToChunks { firstTime, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, 7, chunks[6:8]) if err != nil { t.Fatal(err) } if offset != 0 { t.Errorf("want offset 0, got %d", offset) } if firstTime != 7 { t.Errorf("want first time 7, got %d", firstTime) } if numDropped != 3 { t.Errorf("want 3 dropped chunks, got %v", numDropped) } if allDropped { t.Error("all chunks dropped") } wantChunks := chunks[7:8] indexes := make([]int, len(wantChunks)) for i := range indexes { indexes[i] = i } gotChunks, err := p.loadChunks(fp, indexes, 0) if err != nil { t.Fatal(err) } for i, wantChunk := range wantChunks { if !chunksEqual(wantChunk, gotChunks[i]) { t.Errorf("%d. Chunks not equal.", i) } } } // While adding two more, drop all chunks including the added ones. for fp, chunks := range fpToChunks { firstTime, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, 10, chunks[8:]) if err != nil { t.Fatal(err) } if offset != 0 { t.Errorf("want offset 0, got %d", offset) } if firstTime != 0 { t.Errorf("want first time 0, got %d", firstTime) } if numDropped != 3 { t.Errorf("want 3 dropped chunks, got %v", numDropped) } if !allDropped { t.Error("not all chunks dropped") } } // Now set minShrinkRatio to 0.25 and play with it. p.minShrinkRatio = 0.25 // Re-add 8 chunks. for fp, chunks := range fpToChunks { firstTimeNotDropped, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, model.Earliest, chunks[:8]) if err != nil { t.Fatal(err) } if got, want := firstTimeNotDropped, model.Time(0); got != want { t.Errorf("Want firstTimeNotDropped %v, got %v.", got, want) } if got, want := offset, 0; got != want { t.Errorf("Want offset %v, got %v.", got, want) } if got, want := numDropped, 0; got != want { t.Errorf("Want numDropped %v, got %v.", got, want) } if allDropped { t.Error("All dropped.") } } // Drop only the first chunk should not happen, but persistence should still work. for fp, chunks := range fpToChunks { firstTime, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, 1, chunks[8:9]) if err != nil { t.Fatal(err) } if offset != 8 { t.Errorf("want offset 8, got %d", offset) } if firstTime != 0 { t.Errorf("want first time 0, got %d", firstTime) } if numDropped != 0 { t.Errorf("want 0 dropped chunk, got %v", numDropped) } if allDropped { t.Error("all chunks dropped") } } // Drop only the first two chunks should not happen, either. for fp := range fpToChunks { firstTime, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, 2, nil) if err != nil { t.Fatal(err) } if offset != 0 { t.Errorf("want offset 0, got %d", offset) } if firstTime != 0 { t.Errorf("want first time 0, got %d", firstTime) } if numDropped != 0 { t.Errorf("want 0 dropped chunk, got %v", numDropped) } if allDropped { t.Error("all chunks dropped") } } // Drop the first three chunks should finally work. for fp, chunks := range fpToChunks { firstTime, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, 3, chunks[9:]) if err != nil { t.Fatal(err) } if offset != 6 { t.Errorf("want offset 6, got %d", offset) } if firstTime != 3 { t.Errorf("want first time 3, got %d", firstTime) } if numDropped != 3 { t.Errorf("want 3 dropped chunk, got %v", numDropped) } if allDropped { t.Error("all chunks dropped") } } }
func testRangeValues(t *testing.T, encoding chunkEncoding) { samples := make(model.Samples, 10000) for i := range samples { samples[i] = &model.Sample{ Timestamp: model.Time(2 * i), Value: model.SampleValue(float64(i) * 0.2), } } s, closer := NewTestStorage(t, encoding) defer closer.Close() for _, sample := range samples { s.Append(sample) } s.WaitForIndexing() fp := model.Metric{}.FastFingerprint() it := s.NewIterator(fp) // #1 Zero length interval at sample. for i, expected := range samples { actual := it.RangeValues(metric.Interval{ OldestInclusive: expected.Timestamp, NewestInclusive: expected.Timestamp, }) if len(actual) != 1 { t.Fatalf("1.%d. Expected exactly one result, got %d.", i, len(actual)) } if expected.Timestamp != actual[0].Timestamp { t.Errorf("1.%d. Got %v; want %v.", i, actual[0].Timestamp, expected.Timestamp) } if expected.Value != actual[0].Value { t.Errorf("1.%d. Got %v; want %v.", i, actual[0].Value, expected.Value) } } // #2 Zero length interval off sample. for i, expected := range samples { actual := it.RangeValues(metric.Interval{ OldestInclusive: expected.Timestamp + 1, NewestInclusive: expected.Timestamp + 1, }) if len(actual) != 0 { t.Fatalf("2.%d. Expected no result, got %d.", i, len(actual)) } } // #3 2sec interval around sample. for i, expected := range samples { actual := it.RangeValues(metric.Interval{ OldestInclusive: expected.Timestamp - 1, NewestInclusive: expected.Timestamp + 1, }) if len(actual) != 1 { t.Fatalf("3.%d. Expected exactly one result, got %d.", i, len(actual)) } if expected.Timestamp != actual[0].Timestamp { t.Errorf("3.%d. Got %v; want %v.", i, actual[0].Timestamp, expected.Timestamp) } if expected.Value != actual[0].Value { t.Errorf("3.%d. Got %v; want %v.", i, actual[0].Value, expected.Value) } } // #4 2sec interval sample to sample. for i, expected1 := range samples { if i == len(samples)-1 { continue } expected2 := samples[i+1] actual := it.RangeValues(metric.Interval{ OldestInclusive: expected1.Timestamp, NewestInclusive: expected1.Timestamp + 2, }) if len(actual) != 2 { t.Fatalf("4.%d. Expected exactly 2 results, got %d.", i, len(actual)) } if expected1.Timestamp != actual[0].Timestamp { t.Errorf("4.%d. Got %v for 1st result; want %v.", i, actual[0].Timestamp, expected1.Timestamp) } if expected1.Value != actual[0].Value { t.Errorf("4.%d. Got %v for 1st result; want %v.", i, actual[0].Value, expected1.Value) } if expected2.Timestamp != actual[1].Timestamp { t.Errorf("4.%d. Got %v for 2nd result; want %v.", i, actual[1].Timestamp, expected2.Timestamp) } if expected2.Value != actual[1].Value { t.Errorf("4.%d. Got %v for 2nd result; want %v.", i, actual[1].Value, expected2.Value) } } // #5 corner cases: Interval ends at first sample, interval starts // at last sample, interval entirely before/after samples. expected := samples[0] actual := it.RangeValues(metric.Interval{ OldestInclusive: expected.Timestamp - 2, NewestInclusive: expected.Timestamp, }) if len(actual) != 1 { t.Fatalf("5.1. Expected exactly one result, got %d.", len(actual)) } if expected.Timestamp != actual[0].Timestamp { t.Errorf("5.1. Got %v; want %v.", actual[0].Timestamp, expected.Timestamp) } if expected.Value != actual[0].Value { t.Errorf("5.1. Got %v; want %v.", actual[0].Value, expected.Value) } expected = samples[len(samples)-1] actual = it.RangeValues(metric.Interval{ OldestInclusive: expected.Timestamp, NewestInclusive: expected.Timestamp + 2, }) if len(actual) != 1 { t.Fatalf("5.2. Expected exactly one result, got %d.", len(actual)) } if expected.Timestamp != actual[0].Timestamp { t.Errorf("5.2. Got %v; want %v.", actual[0].Timestamp, expected.Timestamp) } if expected.Value != actual[0].Value { t.Errorf("5.2. Got %v; want %v.", actual[0].Value, expected.Value) } firstSample := samples[0] actual = it.RangeValues(metric.Interval{ OldestInclusive: firstSample.Timestamp - 4, NewestInclusive: firstSample.Timestamp - 2, }) if len(actual) != 0 { t.Fatalf("5.3. Expected no results, got %d.", len(actual)) } lastSample := samples[len(samples)-1] actual = it.RangeValues(metric.Interval{ OldestInclusive: lastSample.Timestamp + 2, NewestInclusive: lastSample.Timestamp + 4, }) if len(actual) != 0 { t.Fatalf("5.3. Expected no results, got %d.", len(actual)) } }