func newDiskFrontier(i leveldb.Iterator) (d *diskFrontier, err error) {
if !i.SeekToLast() || i.Key() == nil {
return
}
lastKey, err := extractSampleKey(i)
if err != nil {
panic(err)
}
if !i.SeekToFirst() || i.Key() == nil {
return
}
firstKey, err := extractSampleKey(i)
if i.Key() == nil {
return
}
if err != nil {
panic(err)
}
d = &diskFrontier{}
d.firstFingerprint = model.NewFingerprintFromRowKey(*firstKey.Fingerprint.Signature)
d.firstSupertime = indexable.DecodeTime(firstKey.Timestamp)
d.lastFingerprint = model.NewFingerprintFromRowKey(*lastKey.Fingerprint.Signature)
d.lastSupertime = indexable.DecodeTime(lastKey.Timestamp)
return
}
func (l *LevelDBMetricPersistence) GetFingerprintsForLabelName(labelName model.LabelName) (fps model.Fingerprints, err error) {
begin := time.Now()
defer func() {
duration := time.Since(begin)
recordOutcome(duration, err, map[string]string{operation: getFingerprintsForLabelName, result: success}, map[string]string{operation: getFingerprintsForLabelName, result: failure})
}()
raw, err := l.labelNameToFingerprints.Get(coding.NewProtocolBuffer(model.LabelNameToDTO(&labelName)))
if err != nil {
return
}
unmarshaled := &dto.FingerprintCollection{}
err = proto.Unmarshal(raw, unmarshaled)
if err != nil {
return
}
for _, m := range unmarshaled.Member {
fp := model.NewFingerprintFromRowKey(*m.Signature)
fps = append(fps, fp)
}
return
}
func (c curationState) Get() (key, value coding.Encoder) {
key = coding.NewProtocolBuffer(&dto.CurationKey{
Fingerprint: model.NewFingerprintFromRowKey(c.fingerprint).ToDTO(),
MinimumGroupSize: proto.Uint32(uint32(c.groupSize)),
OlderThan: proto.Int64(int64(c.recencyThreshold)),
})
value = coding.NewProtocolBuffer(&dto.CurationValue{
LastCompletionTimestamp: proto.Int64(c.lastCurated.Unix()),
})
return
}
func (l *LevelDBMetricPersistence) GetFingerprintsForLabelSet(labelSet model.LabelSet) (fps model.Fingerprints, err error) {
begin := time.Now()
defer func() {
duration := time.Since(begin)
recordOutcome(duration, err, map[string]string{operation: getFingerprintsForLabelSet, result: success}, map[string]string{operation: getFingerprintsForLabelSet, result: failure})
}()
sets := []utility.Set{}
for _, labelSetDTO := range model.LabelSetToDTOs(&labelSet) {
f, err := l.labelSetToFingerprints.Get(coding.NewProtocolBuffer(labelSetDTO))
if err != nil {
return fps, err
}
unmarshaled := &dto.FingerprintCollection{}
err = proto.Unmarshal(f, unmarshaled)
if err != nil {
return fps, err
}
set := utility.Set{}
for _, m := range unmarshaled.Member {
fp := model.NewFingerprintFromRowKey(*m.Signature)
set.Add(fp)
}
sets = append(sets, set)
}
numberOfSets := len(sets)
if numberOfSets == 0 {
return
}
base := sets[0]
for i := 1; i < numberOfSets; i++ {
base = base.Intersection(sets[i])
}
for _, e := range base.Elements() {
fingerprint := e.(model.Fingerprint)
fps = append(fps, fingerprint)
}
return
}
func (w watermarkDecoder) DecodeKey(in interface{}) (out interface{}, err error) {
var (
key = &dto.Fingerprint{}
bytes = in.([]byte)
)
err = proto.Unmarshal(bytes, key)
if err != nil {
panic(err)
}
out = model.NewFingerprintFromRowKey(*key.Signature)
return
}
func (s sampleGroup) Get() (key, value coding.Encoder) {
key = coding.NewProtocolBuffer(&dto.SampleKey{
Fingerprint: model.NewFingerprintFromRowKey(s.fingerprint).ToDTO(),
Timestamp: indexable.EncodeTime(s.values[0].time),
LastTimestamp: proto.Int64(s.values[len(s.values)-1].time.Unix()),
SampleCount: proto.Uint32(uint32(len(s.values))),
})
series := &dto.SampleValueSeries{}
for _, value := range s.values {
series.Value = append(series.Value, &dto.SampleValueSeries_Value{
Timestamp: proto.Int64(value.time.Unix()),
Value: proto.Float32(float32(value.value)),
})
}
value = coding.NewProtocolBuffer(series)
return
}
func testBuilder(t test.Tester) {
type atTime struct {
fingerprint string
time time.Time
}
type atInterval struct {
fingerprint string
from time.Time
through time.Time
interval time.Duration
}
type atRange struct {
fingerprint string
from time.Time
through time.Time
}
type in struct {
atTimes []atTime
atIntervals []atInterval
atRanges []atRange
}
type out []struct {
fingerprint string
operations ops
}
var scenarios = []struct {
in in
out out
}{
// // Ensure that the fingerprint is sorted in proper order.
{
in: in{
atTimes: []atTime{
{
fingerprint: "0000000000000001111-a-4-a",
time: time.Unix(100, 0),
},
{
fingerprint: "0000000000000000000-a-4-a",
time: time.Unix(100, 0),
},
},
},
out: out{
{
fingerprint: "00000000000000000000-a-4-a",
},
{
fingerprint: "00000000000000001111-a-4-a",
},
},
},
// // Ensure that the fingerprint-timestamp pairs are sorted in proper order.
{
in: in{
atTimes: []atTime{
{
fingerprint: "1111-a-4-a",
time: time.Unix(100, 0),
},
{
fingerprint: "1111-a-4-a",
time: time.Unix(200, 0),
},
{
fingerprint: "0-a-4-a",
time: time.Unix(100, 0),
},
{
fingerprint: "0-a-4-a",
time: time.Unix(0, 0),
},
},
},
out: out{
{
fingerprint: "00000000000000000000-a-4-a",
},
{
fingerprint: "00000000000000001111-a-4-a",
},
},
},
// Ensure grouping of operations
{
in: in{
atTimes: []atTime{
{
fingerprint: "1111-a-4-a",
time: time.Unix(100, 0),
},
},
atRanges: []atRange{
{
fingerprint: "1111-a-4-a",
from: time.Unix(100, 0),
through: time.Unix(1000, 0),
},
{
fingerprint: "1111-a-4-a",
from: time.Unix(100, 0),
through: time.Unix(9000, 0),
},
},
},
out: out{
{
fingerprint: "00000000000000001111-a-4-a",
},
},
},
}
for i, scenario := range scenarios {
builder := viewRequestBuilder{
operations: map[model.Fingerprint]ops{},
}
for _, atTime := range scenario.in.atTimes {
fingerprint := model.NewFingerprintFromRowKey(atTime.fingerprint)
builder.GetMetricAtTime(fingerprint, atTime.time)
}
for _, atInterval := range scenario.in.atIntervals {
fingerprint := model.NewFingerprintFromRowKey(atInterval.fingerprint)
builder.GetMetricAtInterval(fingerprint, atInterval.from, atInterval.through, atInterval.interval)
}
for _, atRange := range scenario.in.atRanges {
fingerprint := model.NewFingerprintFromRowKey(atRange.fingerprint)
builder.GetMetricRange(fingerprint, atRange.from, atRange.through)
}
jobs := builder.ScanJobs()
if len(scenario.out) != len(jobs) {
t.Fatalf("%d. expected job length of %d, got %d\n", i, len(scenario.out), len(jobs))
}
for j, job := range scenario.out {
if jobs[j].fingerprint.ToRowKey() != job.fingerprint {
t.Fatalf("%d.%d. expected fingerprint %s, got %s\n", i, j, job.fingerprint, jobs[j].fingerprint.ToRowKey())
}
}
}
}
// newSeriesFrontier furnishes a populated diskFrontier for a given
// fingerprint. A nil diskFrontier will be returned if the series cannot
// be found in the store.
func newSeriesFrontier(f model.Fingerprint, d diskFrontier, i leveldb.Iterator) (s *seriesFrontier, err error) {
var (
lowerSeek = firstSupertime
upperSeek = lastSupertime
)
// If the diskFrontier for this iterator says that the candidate fingerprint
// is outside of its seeking domain, there is no way that a seriesFrontier
// could be materialized. Simply bail.
if !d.ContainsFingerprint(f) {
return
}
// If we are either the first or the last key in the database, we need to use
// pessimistic boundary frontiers.
if f.Equal(d.firstFingerprint) {
lowerSeek = indexable.EncodeTime(d.firstSupertime)
}
if f.Equal(d.lastFingerprint) {
upperSeek = indexable.EncodeTime(d.lastSupertime)
}
key := &dto.SampleKey{
Fingerprint: f.ToDTO(),
Timestamp: upperSeek,
}
raw, err := coding.NewProtocolBuffer(key).Encode()
if err != nil {
panic(err)
}
i.Seek(raw)
if i.Key() == nil {
return
}
retrievedKey, err := extractSampleKey(i)
if err != nil {
panic(err)
}
retrievedFingerprint := model.NewFingerprintFromRowKey(*retrievedKey.Fingerprint.Signature)
// The returned fingerprint may not match if the original seek key lives
// outside of a metric's frontier. This is probable, for we are seeking to
// to the maximum allowed time, which could advance us to the next
// fingerprint.
//
//
if !retrievedFingerprint.Equal(f) {
i.Previous()
retrievedKey, err = extractSampleKey(i)
if err != nil {
panic(err)
}
retrievedFingerprint := model.NewFingerprintFromRowKey(*retrievedKey.Fingerprint.Signature)
// If the previous key does not match, we know that the requested
// fingerprint does not live in the database.
if !retrievedFingerprint.Equal(f) {
return
}
}
s = &seriesFrontier{
lastSupertime: indexable.DecodeTime(retrievedKey.Timestamp),
lastTime: time.Unix(*retrievedKey.LastTimestamp, 0),
}
key.Timestamp = lowerSeek
raw, err = coding.NewProtocolBuffer(key).Encode()
if err != nil {
panic(err)
}
i.Seek(raw)
retrievedKey, err = extractSampleKey(i)
if err != nil {
panic(err)
}
retrievedFingerprint = model.NewFingerprintFromRowKey(*retrievedKey.Fingerprint.Signature)
s.firstSupertime = indexable.DecodeTime(retrievedKey.Timestamp)
return
}
func (w watermarkState) Get() (key, value coding.Encoder) {
key = coding.NewProtocolBuffer(model.NewFingerprintFromRowKey(w.fingerprint).ToDTO())
value = coding.NewProtocolBuffer(model.NewWatermarkFromTime(w.lastAppended).ToMetricHighWatermarkDTO())
return
}
