func newDiskFrontier(i leveldb.Iterator) (d *diskFrontier, err error) { if !i.SeekToLast() || i.Key() == nil { return } lastKey, err := extractSampleKey(i) if err != nil { panic(err) } if !i.SeekToFirst() || i.Key() == nil { return } firstKey, err := extractSampleKey(i) if i.Key() == nil { return } if err != nil { panic(err) } d = &diskFrontier{} d.firstFingerprint = model.NewFingerprintFromRowKey(*firstKey.Fingerprint.Signature) d.firstSupertime = indexable.DecodeTime(firstKey.Timestamp) d.lastFingerprint = model.NewFingerprintFromRowKey(*lastKey.Fingerprint.Signature) d.lastSupertime = indexable.DecodeTime(lastKey.Timestamp) return }
func (l *LevelDBMetricPersistence) GetFingerprintsForLabelName(labelName model.LabelName) (fps model.Fingerprints, err error) { begin := time.Now() defer func() { duration := time.Since(begin) recordOutcome(duration, err, map[string]string{operation: getFingerprintsForLabelName, result: success}, map[string]string{operation: getFingerprintsForLabelName, result: failure}) }() raw, err := l.labelNameToFingerprints.Get(coding.NewProtocolBuffer(model.LabelNameToDTO(&labelName))) if err != nil { return } unmarshaled := &dto.FingerprintCollection{} err = proto.Unmarshal(raw, unmarshaled) if err != nil { return } for _, m := range unmarshaled.Member { fp := model.NewFingerprintFromRowKey(*m.Signature) fps = append(fps, fp) } return }
func (c curationState) Get() (key, value coding.Encoder) { key = coding.NewProtocolBuffer(&dto.CurationKey{ Fingerprint: model.NewFingerprintFromRowKey(c.fingerprint).ToDTO(), MinimumGroupSize: proto.Uint32(uint32(c.groupSize)), OlderThan: proto.Int64(int64(c.recencyThreshold)), }) value = coding.NewProtocolBuffer(&dto.CurationValue{ LastCompletionTimestamp: proto.Int64(c.lastCurated.Unix()), }) return }
func (l *LevelDBMetricPersistence) GetFingerprintsForLabelSet(labelSet model.LabelSet) (fps model.Fingerprints, err error) { begin := time.Now() defer func() { duration := time.Since(begin) recordOutcome(duration, err, map[string]string{operation: getFingerprintsForLabelSet, result: success}, map[string]string{operation: getFingerprintsForLabelSet, result: failure}) }() sets := []utility.Set{} for _, labelSetDTO := range model.LabelSetToDTOs(&labelSet) { f, err := l.labelSetToFingerprints.Get(coding.NewProtocolBuffer(labelSetDTO)) if err != nil { return fps, err } unmarshaled := &dto.FingerprintCollection{} err = proto.Unmarshal(f, unmarshaled) if err != nil { return fps, err } set := utility.Set{} for _, m := range unmarshaled.Member { fp := model.NewFingerprintFromRowKey(*m.Signature) set.Add(fp) } sets = append(sets, set) } numberOfSets := len(sets) if numberOfSets == 0 { return } base := sets[0] for i := 1; i < numberOfSets; i++ { base = base.Intersection(sets[i]) } for _, e := range base.Elements() { fingerprint := e.(model.Fingerprint) fps = append(fps, fingerprint) } return }
func (w watermarkDecoder) DecodeKey(in interface{}) (out interface{}, err error) { var ( key = &dto.Fingerprint{} bytes = in.([]byte) ) err = proto.Unmarshal(bytes, key) if err != nil { panic(err) } out = model.NewFingerprintFromRowKey(*key.Signature) return }
func (s sampleGroup) Get() (key, value coding.Encoder) { key = coding.NewProtocolBuffer(&dto.SampleKey{ Fingerprint: model.NewFingerprintFromRowKey(s.fingerprint).ToDTO(), Timestamp: indexable.EncodeTime(s.values[0].time), LastTimestamp: proto.Int64(s.values[len(s.values)-1].time.Unix()), SampleCount: proto.Uint32(uint32(len(s.values))), }) series := &dto.SampleValueSeries{} for _, value := range s.values { series.Value = append(series.Value, &dto.SampleValueSeries_Value{ Timestamp: proto.Int64(value.time.Unix()), Value: proto.Float32(float32(value.value)), }) } value = coding.NewProtocolBuffer(series) return }
func testBuilder(t test.Tester) { type atTime struct { fingerprint string time time.Time } type atInterval struct { fingerprint string from time.Time through time.Time interval time.Duration } type atRange struct { fingerprint string from time.Time through time.Time } type in struct { atTimes []atTime atIntervals []atInterval atRanges []atRange } type out []struct { fingerprint string operations ops } var scenarios = []struct { in in out out }{ // // Ensure that the fingerprint is sorted in proper order. { in: in{ atTimes: []atTime{ { fingerprint: "0000000000000001111-a-4-a", time: time.Unix(100, 0), }, { fingerprint: "0000000000000000000-a-4-a", time: time.Unix(100, 0), }, }, }, out: out{ { fingerprint: "00000000000000000000-a-4-a", }, { fingerprint: "00000000000000001111-a-4-a", }, }, }, // // Ensure that the fingerprint-timestamp pairs are sorted in proper order. { in: in{ atTimes: []atTime{ { fingerprint: "1111-a-4-a", time: time.Unix(100, 0), }, { fingerprint: "1111-a-4-a", time: time.Unix(200, 0), }, { fingerprint: "0-a-4-a", time: time.Unix(100, 0), }, { fingerprint: "0-a-4-a", time: time.Unix(0, 0), }, }, }, out: out{ { fingerprint: "00000000000000000000-a-4-a", }, { fingerprint: "00000000000000001111-a-4-a", }, }, }, // Ensure grouping of operations { in: in{ atTimes: []atTime{ { fingerprint: "1111-a-4-a", time: time.Unix(100, 0), }, }, atRanges: []atRange{ { fingerprint: "1111-a-4-a", from: time.Unix(100, 0), through: time.Unix(1000, 0), }, { fingerprint: "1111-a-4-a", from: time.Unix(100, 0), through: time.Unix(9000, 0), }, }, }, out: out{ { fingerprint: "00000000000000001111-a-4-a", }, }, }, } for i, scenario := range scenarios { builder := viewRequestBuilder{ operations: map[model.Fingerprint]ops{}, } for _, atTime := range scenario.in.atTimes { fingerprint := model.NewFingerprintFromRowKey(atTime.fingerprint) builder.GetMetricAtTime(fingerprint, atTime.time) } for _, atInterval := range scenario.in.atIntervals { fingerprint := model.NewFingerprintFromRowKey(atInterval.fingerprint) builder.GetMetricAtInterval(fingerprint, atInterval.from, atInterval.through, atInterval.interval) } for _, atRange := range scenario.in.atRanges { fingerprint := model.NewFingerprintFromRowKey(atRange.fingerprint) builder.GetMetricRange(fingerprint, atRange.from, atRange.through) } jobs := builder.ScanJobs() if len(scenario.out) != len(jobs) { t.Fatalf("%d. expected job length of %d, got %d\n", i, len(scenario.out), len(jobs)) } for j, job := range scenario.out { if jobs[j].fingerprint.ToRowKey() != job.fingerprint { t.Fatalf("%d.%d. expected fingerprint %s, got %s\n", i, j, job.fingerprint, jobs[j].fingerprint.ToRowKey()) } } } }
// newSeriesFrontier furnishes a populated diskFrontier for a given // fingerprint. A nil diskFrontier will be returned if the series cannot // be found in the store. func newSeriesFrontier(f model.Fingerprint, d diskFrontier, i leveldb.Iterator) (s *seriesFrontier, err error) { var ( lowerSeek = firstSupertime upperSeek = lastSupertime ) // If the diskFrontier for this iterator says that the candidate fingerprint // is outside of its seeking domain, there is no way that a seriesFrontier // could be materialized. Simply bail. if !d.ContainsFingerprint(f) { return } // If we are either the first or the last key in the database, we need to use // pessimistic boundary frontiers. if f.Equal(d.firstFingerprint) { lowerSeek = indexable.EncodeTime(d.firstSupertime) } if f.Equal(d.lastFingerprint) { upperSeek = indexable.EncodeTime(d.lastSupertime) } key := &dto.SampleKey{ Fingerprint: f.ToDTO(), Timestamp: upperSeek, } raw, err := coding.NewProtocolBuffer(key).Encode() if err != nil { panic(err) } i.Seek(raw) if i.Key() == nil { return } retrievedKey, err := extractSampleKey(i) if err != nil { panic(err) } retrievedFingerprint := model.NewFingerprintFromRowKey(*retrievedKey.Fingerprint.Signature) // The returned fingerprint may not match if the original seek key lives // outside of a metric's frontier. This is probable, for we are seeking to // to the maximum allowed time, which could advance us to the next // fingerprint. // // if !retrievedFingerprint.Equal(f) { i.Previous() retrievedKey, err = extractSampleKey(i) if err != nil { panic(err) } retrievedFingerprint := model.NewFingerprintFromRowKey(*retrievedKey.Fingerprint.Signature) // If the previous key does not match, we know that the requested // fingerprint does not live in the database. if !retrievedFingerprint.Equal(f) { return } } s = &seriesFrontier{ lastSupertime: indexable.DecodeTime(retrievedKey.Timestamp), lastTime: time.Unix(*retrievedKey.LastTimestamp, 0), } key.Timestamp = lowerSeek raw, err = coding.NewProtocolBuffer(key).Encode() if err != nil { panic(err) } i.Seek(raw) retrievedKey, err = extractSampleKey(i) if err != nil { panic(err) } retrievedFingerprint = model.NewFingerprintFromRowKey(*retrievedKey.Fingerprint.Signature) s.firstSupertime = indexable.DecodeTime(retrievedKey.Timestamp) return }
func (w watermarkState) Get() (key, value coding.Encoder) { key = coding.NewProtocolBuffer(model.NewFingerprintFromRowKey(w.fingerprint).ToDTO()) value = coding.NewProtocolBuffer(model.NewWatermarkFromTime(w.lastAppended).ToMetricHighWatermarkDTO()) return }