func (l *LevelDBMetricPersistence) appendFingerprints(m *dto.Metric) (err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: appendFingerprints, result: success}, map[string]string{operation: appendFingerprints, result: failure}) }() fingerprintDTO, err := model.MessageToFingerprintDTO(m) if err != nil { return } fingerprintKey := coding.NewProtocolBufferEncoder(fingerprintDTO) metricDTOEncoder := coding.NewProtocolBufferEncoder(m) err = l.fingerprintToMetrics.Put(fingerprintKey, metricDTOEncoder) if err != nil { return } labelCount := len(m.LabelPair) labelPairErrors := make(chan error, labelCount) labelNameErrors := make(chan error, labelCount) for _, labelPair := range m.LabelPair { go func(labelPair *dto.LabelPair) { labelNameErrors <- l.appendLabelNameFingerprint(labelPair, fingerprintDTO) }(labelPair) go func(labelPair *dto.LabelPair) { labelPairErrors <- l.appendLabelPairFingerprint(labelPair, fingerprintDTO) }(labelPair) } for i := 0; i < cap(labelPairErrors); i++ { err = <-labelPairErrors if err != nil { return } } for i := 0; i < cap(labelNameErrors); i++ { err = <-labelNameErrors if err != nil { return } } return }
func (l *LevelDBMetricPersistence) AppendSample(sample *model.Sample) (err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: appendSample, result: success}, map[string]string{operation: appendSample, result: failure}) }() metricDTO := model.SampleToMetricDTO(sample) indexHas, err := l.hasIndexMetric(metricDTO) if err != nil { return } if !indexHas { err = l.indexMetric(metricDTO) if err != nil { return } err = l.appendFingerprints(metricDTO) if err != nil { return } } fingerprintDTO, err := model.MessageToFingerprintDTO(metricDTO) if err != nil { return } sampleKeyDTO := &dto.SampleKey{ Fingerprint: fingerprintDTO, Timestamp: indexable.EncodeTime(sample.Timestamp), } sampleValueDTO := &dto.SampleValue{ Value: proto.Float32(float32(sample.Value)), } sampleKeyEncoded := coding.NewProtocolBufferEncoder(sampleKeyDTO) sampleValueEncoded := coding.NewProtocolBufferEncoder(sampleValueDTO) err = l.metricSamples.Put(sampleKeyEncoded, sampleValueEncoded) if err != nil { return } return }
func (l *LevelDBMetricPersistence) setLabelPairFingerprints(labelPair *dto.LabelPair, fingerprints *dto.FingerprintCollection) (err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: setLabelPairFingerprints, result: success}, map[string]string{operation: setLabelPairFingerprints, result: failure}) }() labelPairEncoded := coding.NewProtocolBufferEncoder(labelPair) fingerprintsEncoded := coding.NewProtocolBufferEncoder(fingerprints) err = l.labelSetToFingerprints.Put(labelPairEncoded, fingerprintsEncoded) return }
func (l *LevelDBMetricPersistence) GetMetricForFingerprint(f *model.Fingerprint) (m *model.Metric, err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: getMetricForFingerprint, result: success}, map[string]string{operation: getMetricForFingerprint, result: failure}) }() raw, err := l.fingerprintToMetrics.Get(coding.NewProtocolBufferEncoder(model.FingerprintToDTO(f))) if err != nil { return } unmarshaled := &dto.Metric{} err = proto.Unmarshal(raw, unmarshaled) if err != nil { return } m = &model.Metric{} for _, v := range unmarshaled.LabelPair { (*m)[model.LabelName(*v.Name)] = model.LabelValue(*v.Value) } return }
func (l *LevelDBMetricPersistence) GetFingerprintsForLabelName(labelName *model.LabelName) (fps []*model.Fingerprint, err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: getFingerprintsForLabelName, result: success}, map[string]string{operation: getFingerprintsForLabelName, result: failure}) }() fps = make([]*model.Fingerprint, 0, 0) raw, err := l.labelNameToFingerprints.Get(coding.NewProtocolBufferEncoder(model.LabelNameToDTO(labelName))) if err != nil { return } unmarshaled := &dto.FingerprintCollection{} err = proto.Unmarshal(raw, unmarshaled) if err != nil { return } for _, m := range unmarshaled.Member { fp := model.Fingerprint(*m.Signature) fps = append(fps, &fp) } return }
func (l *LevelDBMetricPersistence) indexMetric(dto *dto.Metric) (err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: indexMetric, result: success}, map[string]string{operation: indexMetric, result: failure}) }() dtoKey := coding.NewProtocolBufferEncoder(dto) err = l.metricMembershipIndex.Put(dtoKey) return }
func (l *LevelDBMetricPersistence) GetFingerprintsForLabelSet(labelSet *model.LabelSet) (fps []*model.Fingerprint, err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: getFingerprintsForLabelSet, result: success}, map[string]string{operation: getFingerprintsForLabelSet, result: failure}) }() sets := []utility.Set{} for _, labelSetDTO := range model.LabelSetToDTOs(labelSet) { f, err := l.labelSetToFingerprints.Get(coding.NewProtocolBufferEncoder(labelSetDTO)) if err != nil { return fps, err } unmarshaled := &dto.FingerprintCollection{} err = proto.Unmarshal(f, unmarshaled) if err != nil { return fps, err } set := utility.Set{} for _, m := range unmarshaled.Member { fp := model.Fingerprint(*m.Signature) set.Add(fp) } sets = append(sets, set) } numberOfSets := len(sets) if numberOfSets == 0 { return } base := sets[0] for i := 1; i < numberOfSets; i++ { base = base.Intersection(sets[i]) } fps = []*model.Fingerprint{} for _, e := range base.Elements() { fingerprint := e.(model.Fingerprint) fps = append(fps, &fingerprint) } return }
func (l *LevelDBMetricPersistence) HasLabelName(dto *dto.LabelName) (value bool, err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: hasLabelName, result: success}, map[string]string{operation: hasLabelName, result: failure}) }() dtoKey := coding.NewProtocolBufferEncoder(dto) value, err = l.labelNameToFingerprints.Has(dtoKey) return }
func (l *LevelDBMetricPersistence) GetAllMetrics() ([]model.LabelSet, error) { if getAll, getAllError := l.labelSetToFingerprints.GetAll(); getAllError == nil { result := make([]model.LabelSet, 0) fingerprintCollection := &dto.FingerprintCollection{} fingerprints := make(utility.Set) for _, pair := range getAll { if unmarshalError := proto.Unmarshal(pair.Right, fingerprintCollection); unmarshalError == nil { for _, member := range fingerprintCollection.Member { if !fingerprints.Has(*member.Signature) { fingerprints.Add(*member.Signature) fingerprintEncoded := coding.NewProtocolBufferEncoder(member) if labelPairCollectionRaw, labelPairCollectionRawError := l.fingerprintToMetrics.Get(fingerprintEncoded); labelPairCollectionRawError == nil { labelPairCollectionDTO := &dto.LabelSet{} if labelPairCollectionDTOMarshalError := proto.Unmarshal(labelPairCollectionRaw, labelPairCollectionDTO); labelPairCollectionDTOMarshalError == nil { intermediate := make(model.LabelSet, 0) for _, member := range labelPairCollectionDTO.Member { n := model.LabelName(*member.Name) v := model.LabelValue(*member.Value) intermediate[n] = v } result = append(result, intermediate) } else { return nil, labelPairCollectionDTOMarshalError } } else { return nil, labelPairCollectionRawError } } } } else { return nil, unmarshalError } } return result, nil } else { return nil, getAllError } return nil, errors.New("Unknown error encountered when querying metrics.") }
// XXX: Delete me and replace with GetFingerprintsForLabelName. func (l *LevelDBMetricPersistence) GetLabelNameFingerprints(n *dto.LabelName) (c *dto.FingerprintCollection, err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: getLabelNameFingerprints, result: success}, map[string]string{operation: getLabelNameFingerprints, result: failure}) }() dtoKey := coding.NewProtocolBufferEncoder(n) get, err := l.labelNameToFingerprints.Get(dtoKey) if err != nil { return } c = &dto.FingerprintCollection{} err = proto.Unmarshal(get, c) return }
// Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package leveldb import ( "github.com/matttproud/prometheus/coding" dto "github.com/matttproud/prometheus/model/generated" "github.com/matttproud/prometheus/storage/raw/leveldb" ) var ( existenceValue = coding.NewProtocolBufferEncoder(&dto.MembershipIndexValue{}) ) type LevelDBMembershipIndex struct { persistence *leveldb.LevelDBPersistence } func (l *LevelDBMembershipIndex) Close() error { return l.persistence.Close() } func (l *LevelDBMembershipIndex) Has(key coding.Encoder) (bool, error) { return l.persistence.Has(key) } func (l *LevelDBMembershipIndex) Drop(key coding.Encoder) error {
func (l *LevelDBMetricPersistence) GetSamplesForMetric(metric model.Metric, interval model.Interval) ([]model.Samples, error) { metricDTO := model.MetricToDTO(&metric) if fingerprintDTO, fingerprintDTOErr := model.MessageToFingerprintDTO(metricDTO); fingerprintDTOErr == nil { if iterator, closer, iteratorErr := l.metricSamples.GetIterator(); iteratorErr == nil { defer closer.Close() start := &dto.SampleKey{ Fingerprint: fingerprintDTO, Timestamp: indexable.EncodeTime(interval.OldestInclusive), } emission := make([]model.Samples, 0) if encode, encodeErr := coding.NewProtocolBufferEncoder(start).Encode(); encodeErr == nil { iterator.Seek(encode) predicate := keyIsAtMostOld(interval.NewestInclusive) for iterator = iterator; iterator.Valid(); iterator.Next() { key := &dto.SampleKey{} value := &dto.SampleValue{} if keyUnmarshalErr := proto.Unmarshal(iterator.Key(), key); keyUnmarshalErr == nil { if valueUnmarshalErr := proto.Unmarshal(iterator.Value(), value); valueUnmarshalErr == nil { if fingerprintsEqual(fingerprintDTO, key.Fingerprint) { // Wart if predicate(key) { emission = append(emission, model.Samples{ Value: model.SampleValue(*value.Value), Timestamp: indexable.DecodeTime(key.Timestamp), }) } else { break } } else { break } } else { return nil, valueUnmarshalErr } } else { return nil, keyUnmarshalErr } } return emission, nil } else { log.Printf("Could not encode the start key: %q\n", encodeErr) return nil, encodeErr } } else { log.Printf("Could not acquire iterator: %q\n", iteratorErr) return nil, iteratorErr } } else { log.Printf("Could not create fingerprint for the metric: %q\n", fingerprintDTOErr) return nil, fingerprintDTOErr } panic("unreachable") }
func (l *LevelDBMetricPersistence) GetRangeValues(m *model.Metric, i *model.Interval, s *metric.StalenessPolicy) (v *model.SampleSet, err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: getRangeValues, result: success}, map[string]string{operation: getRangeValues, result: failure}) }() d := model.MetricToDTO(m) f, err := model.MessageToFingerprintDTO(d) if err != nil { return } k := &dto.SampleKey{ Fingerprint: f, Timestamp: indexable.EncodeTime(i.OldestInclusive), } e, err := coding.NewProtocolBufferEncoder(k).Encode() if err != nil { return } iterator, closer, err := l.metricSamples.GetIterator() if err != nil { return } defer closer.Close() iterator.Seek(e) predicate := keyIsOlderThan(i.NewestInclusive) for ; iterator.Valid(); iterator.Next() { retrievedKey := &dto.SampleKey{} retrievedKey, err = extractSampleKey(iterator) if err != nil { return } if predicate(retrievedKey) { break } if !fingerprintsEqual(retrievedKey.Fingerprint, k.Fingerprint) { break } retrievedValue, err := extractSampleValue(iterator) if err != nil { return nil, err } if v == nil { v = &model.SampleSet{} } v.Values = append(v.Values, model.SamplePair{ Value: model.SampleValue(*retrievedValue.Value), Timestamp: indexable.DecodeTime(retrievedKey.Timestamp), }) } return }
func (l *LevelDBMetricPersistence) GetValueAtTime(m *model.Metric, t *time.Time, s *metric.StalenessPolicy) (sample *model.Sample, err error) { begin := time.Now() defer func() { duration := time.Now().Sub(begin) recordOutcome(storageOperations, storageLatency, duration, err, map[string]string{operation: getValueAtTime, result: success}, map[string]string{operation: getValueAtTime, result: failure}) }() d := model.MetricToDTO(m) f, err := model.MessageToFingerprintDTO(d) if err != nil { return } // Candidate for Refactoring k := &dto.SampleKey{ Fingerprint: f, Timestamp: indexable.EncodeTime(*t), } e, err := coding.NewProtocolBufferEncoder(k).Encode() if err != nil { return } iterator, closer, err := l.metricSamples.GetIterator() if err != nil { return } defer closer.Close() iterator.Seek(e) if !iterator.Valid() { /* * Two cases for this: * 1.) Corruption in LevelDB. * 2.) Key seek after AND outside known range. * * Once a LevelDB iterator goes invalid, it cannot be recovered; thusly, * we need to create a new in order to check if the last value in the * database is sufficient for our purposes. This is, in all reality, a * corner case but one that could bring down the system. */ iterator, closer, err = l.metricSamples.GetIterator() if err != nil { return } defer closer.Close() iterator.SeekToLast() if !iterator.Valid() { /* * For whatever reason, the LevelDB cannot be recovered. */ return } } var ( firstKey *dto.SampleKey firstValue *dto.SampleValue ) firstKey, err = extractSampleKey(iterator) if err != nil { return } peekAhead := false if !fingerprintsEqual(firstKey.Fingerprint, k.Fingerprint) { /* * This allows us to grab values for metrics if our request time is after * the last recorded time subject to the staleness policy due to the nuances * of LevelDB storage: * * # Assumptions: * - K0 < K1 in terms of sorting. * - T0 < T1 in terms of sorting. * * # Data * * K0-T0 * K0-T1 * K0-T2 * K1-T0 * K1-T1 * * # Scenario * K0-T3, which does not exist, is requested. LevelDB will thusly seek to * K1-T1, when K0-T2 exists as a perfectly good candidate to check subject * to the provided staleness policy and such. */ peekAhead = true } firstTime := indexable.DecodeTime(firstKey.Timestamp) if t.Before(firstTime) || peekAhead { iterator.Prev() if !iterator.Valid() { /* * Two cases for this: * 1.) Corruption in LevelDB. * 2.) Key seek before AND outside known range. * * This is an explicit validation to ensure that if no previous values for * the series are found, the query aborts. */ return } var ( alternativeKey *dto.SampleKey alternativeValue *dto.SampleValue ) alternativeKey, err = extractSampleKey(iterator) if err != nil { return } if !fingerprintsEqual(alternativeKey.Fingerprint, k.Fingerprint) { return } /* * At this point, we found a previous value in the same series in the * database. LevelDB originally seeked to the subsequent element given * the key, but we need to consider this adjacency instead. */ alternativeTime := indexable.DecodeTime(alternativeKey.Timestamp) firstKey = alternativeKey firstValue = alternativeValue firstTime = alternativeTime } firstDelta := firstTime.Sub(*t) if firstDelta < 0 { firstDelta *= -1 } if firstDelta > s.DeltaAllowance { return } firstValue, err = extractSampleValue(iterator) if err != nil { return } sample = model.SampleFromDTO(m, t, firstValue) if firstDelta == time.Duration(0) { return } iterator.Next() if !iterator.Valid() { /* * Two cases for this: * 1.) Corruption in LevelDB. * 2.) Key seek after AND outside known range. * * This means that there are no more values left in the storage; and if this * point is reached, we know that the one that has been found is within the * allowed staleness limits. */ return } var secondKey *dto.SampleKey secondKey, err = extractSampleKey(iterator) if err != nil { return } if !fingerprintsEqual(secondKey.Fingerprint, k.Fingerprint) { return } else { /* * At this point, current entry in the database has the same key as the * previous. For this reason, the validation logic will expect that the * distance between the two points shall not exceed the staleness policy * allowed limit to reduce interpolation errors. * * For this reason, the sample is reset in case of other subsequent * validation behaviors. */ sample = nil } secondTime := indexable.DecodeTime(secondKey.Timestamp) totalDelta := secondTime.Sub(firstTime) if totalDelta > s.DeltaAllowance { return } var secondValue *dto.SampleValue secondValue, err = extractSampleValue(iterator) if err != nil { return } interpolated := interpolate(firstTime, secondTime, *firstValue.Value, *secondValue.Value, *t) sampleValue := &dto.SampleValue{ Value: &interpolated, } sample = model.SampleFromDTO(m, t, sampleValue) return }