Ejemplo n.º 1
0
func (p *persistence) rebuildLabelIndexes(
	fpToSeries map[clientmodel.Fingerprint]*memorySeries,
) error {
	count := 0
	log.Info("Rebuilding label indexes.")
	log.Info("Indexing metrics in memory.")
	for fp, s := range fpToSeries {
		p.indexMetric(fp, s.metric)
		count++
		if count%10000 == 0 {
			log.Infof("%d metrics queued for indexing.", count)
		}
	}
	log.Info("Indexing archived metrics.")
	var fp codable.Fingerprint
	var m codable.Metric
	if err := p.archivedFingerprintToMetrics.ForEach(func(kv index.KeyValueAccessor) error {
		if err := kv.Key(&fp); err != nil {
			return err
		}
		if err := kv.Value(&m); err != nil {
			return err
		}
		p.indexMetric(clientmodel.Fingerprint(fp), clientmodel.Metric(m))
		count++
		if count%10000 == 0 {
			log.Infof("%d metrics queued for indexing.", count)
		}
		return nil
	}); err != nil {
		return err
	}
	log.Info("All requests for rebuilding the label indexes queued. (Actual processing may lag behind.)")
	return nil
}
Ejemplo n.º 2
0
func (m *fpMapper) nextMappedFP() clientmodel.Fingerprint {
	mappedFP := clientmodel.Fingerprint(atomic.AddUint64((*uint64)(&m.highestMappedFP), 1))
	if mappedFP > maxMappedFP {
		panic(fmt.Errorf("more than %v fingerprints mapped in collision detection", maxMappedFP))
	}
	return mappedFP
}
Ejemplo n.º 3
0
func TestCheckpointAndLoadFPMappings(t *testing.T) {
	p, closer := newTestPersistence(t, 1)
	defer closer.Close()

	in := fpMappings{
		1: map[string]clientmodel.Fingerprint{
			"foo": 1,
			"bar": 2,
		},
		3: map[string]clientmodel.Fingerprint{
			"baz": 4,
		},
	}

	if err := p.checkpointFPMappings(in); err != nil {
		t.Fatal(err)
	}

	out, fp, err := p.loadFPMappings()
	if err != nil {
		t.Fatal(err)
	}
	if got, want := fp, clientmodel.Fingerprint(4); got != want {
		t.Errorf("got highest FP %v, want %v", got, want)
	}
	if !reflect.DeepEqual(in, out) {
		t.Errorf("got collision map %v, want %v", out, in)
	}
}
Ejemplo n.º 4
0
func BenchmarkFingerprintLockerSerial(b *testing.B) {
	numFingerprints := 10
	locker := newFingerprintLocker(100)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		fp := clientmodel.Fingerprint(i % numFingerprints)
		locker.Lock(fp)
		locker.Unlock(fp)
	}
}
Ejemplo n.º 5
0
// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (fps *Fingerprints) UnmarshalBinary(buf []byte) error {
	numFPs, offset := binary.Varint(buf)
	if offset <= 0 {
		return fmt.Errorf("could not decode length of Fingerprints, varint decoding returned %d", offset)
	}
	*fps = make(Fingerprints, numFPs)

	for i := range *fps {
		(*fps)[i] = clientmodel.Fingerprint(binary.BigEndian.Uint64(buf[offset+i*8:]))
	}
	return nil
}
Ejemplo n.º 6
0
// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (fps *FingerprintSet) UnmarshalBinary(buf []byte) error {
	numFPs, offset := binary.Varint(buf)
	if offset <= 0 {
		return fmt.Errorf("could not decode length of Fingerprints, varint decoding returned %d", offset)
	}
	*fps = make(FingerprintSet, numFPs)

	for i := 0; i < int(numFPs); i++ {
		(*fps)[clientmodel.Fingerprint(binary.BigEndian.Uint64(buf[offset+i*8:]))] = struct{}{}
	}
	return nil
}
Ejemplo n.º 7
0
func BenchmarkFingerprintLockerParallel(b *testing.B) {
	numGoroutines := 10
	numFingerprints := 10
	numLockOps := b.N
	locker := newFingerprintLocker(100)

	wg := sync.WaitGroup{}
	b.ResetTimer()
	for i := 0; i < numGoroutines; i++ {
		wg.Add(1)
		go func(i int) {
			for j := 0; j < numLockOps; j++ {
				fp1 := clientmodel.Fingerprint(j % numFingerprints)
				fp2 := clientmodel.Fingerprint(j%numFingerprints + numFingerprints)
				locker.Lock(fp1)
				locker.Lock(fp2)
				locker.Unlock(fp2)
				locker.Unlock(fp1)
			}
			wg.Done()
		}(i)
	}
	wg.Wait()
}
Ejemplo n.º 8
0
// fingerprintsModifiedBefore returns the fingerprints of archived timeseries
// that have live samples before the provided timestamp. This method is
// goroutine-safe.
func (p *persistence) fingerprintsModifiedBefore(beforeTime clientmodel.Timestamp) ([]clientmodel.Fingerprint, error) {
	var fp codable.Fingerprint
	var tr codable.TimeRange
	fps := []clientmodel.Fingerprint{}
	p.archivedFingerprintToTimeRange.ForEach(func(kv index.KeyValueAccessor) error {
		if err := kv.Value(&tr); err != nil {
			return err
		}
		if tr.First.Before(beforeTime) {
			if err := kv.Key(&fp); err != nil {
				return err
			}
			fps = append(fps, clientmodel.Fingerprint(fp))
		}
		return nil
	})
	return fps, nil
}
Ejemplo n.º 9
0
// loadSeriesMapAndHeads loads the fingerprint to memory-series mapping and all
// the chunks contained in the checkpoint (and thus not yet persisted to series
// files). The method is capable of loading the checkpoint format v1 and v2. If
// recoverable corruption is detected, or if the dirty flag was set from the
// beginning, crash recovery is run, which might take a while. If an
// unrecoverable error is encountered, it is returned. Call this method during
// start-up while nothing else is running in storage land. This method is
// utterly goroutine-unsafe.
func (p *persistence) loadSeriesMapAndHeads() (sm *seriesMap, chunksToPersist int64, err error) {
	var chunkDescsTotal int64
	fingerprintToSeries := make(map[clientmodel.Fingerprint]*memorySeries)
	sm = &seriesMap{m: fingerprintToSeries}

	defer func() {
		if sm != nil && p.dirty {
			log.Warn("Persistence layer appears dirty.")
			err = p.recoverFromCrash(fingerprintToSeries)
			if err != nil {
				sm = nil
			}
		}
		if err == nil {
			numMemChunkDescs.Add(float64(chunkDescsTotal))
		}
	}()

	f, err := os.Open(p.headsFileName())
	if os.IsNotExist(err) {
		return sm, 0, nil
	}
	if err != nil {
		log.Warn("Could not open heads file:", err)
		p.dirty = true
		return
	}
	defer f.Close()
	r := bufio.NewReaderSize(f, fileBufSize)

	buf := make([]byte, len(headsMagicString))
	if _, err := io.ReadFull(r, buf); err != nil {
		log.Warn("Could not read from heads file:", err)
		p.dirty = true
		return sm, 0, nil
	}
	magic := string(buf)
	if magic != headsMagicString {
		log.Warnf(
			"unexpected magic string, want %q, got %q",
			headsMagicString, magic,
		)
		p.dirty = true
		return
	}
	version, err := binary.ReadVarint(r)
	if (version != headsFormatVersion && version != headsFormatLegacyVersion) || err != nil {
		log.Warnf("unknown heads format version, want %d", headsFormatVersion)
		p.dirty = true
		return sm, 0, nil
	}
	numSeries, err := codable.DecodeUint64(r)
	if err != nil {
		log.Warn("Could not decode number of series:", err)
		p.dirty = true
		return sm, 0, nil
	}

	for ; numSeries > 0; numSeries-- {
		seriesFlags, err := r.ReadByte()
		if err != nil {
			log.Warn("Could not read series flags:", err)
			p.dirty = true
			return sm, chunksToPersist, nil
		}
		headChunkPersisted := seriesFlags&flagHeadChunkPersisted != 0
		fp, err := codable.DecodeUint64(r)
		if err != nil {
			log.Warn("Could not decode fingerprint:", err)
			p.dirty = true
			return sm, chunksToPersist, nil
		}
		var metric codable.Metric
		if err := metric.UnmarshalFromReader(r); err != nil {
			log.Warn("Could not decode metric:", err)
			p.dirty = true
			return sm, chunksToPersist, nil
		}
		var persistWatermark int64
		var modTime time.Time
		if version != headsFormatLegacyVersion {
			// persistWatermark only present in v2.
			persistWatermark, err = binary.ReadVarint(r)
			if err != nil {
				log.Warn("Could not decode persist watermark:", err)
				p.dirty = true
				return sm, chunksToPersist, nil
			}
			modTimeNano, err := binary.ReadVarint(r)
			if err != nil {
				log.Warn("Could not decode modification time:", err)
				p.dirty = true
				return sm, chunksToPersist, nil
			}
			if modTimeNano != -1 {
				modTime = time.Unix(0, modTimeNano)
			}
		}
		chunkDescsOffset, err := binary.ReadVarint(r)
		if err != nil {
			log.Warn("Could not decode chunk descriptor offset:", err)
			p.dirty = true
			return sm, chunksToPersist, nil
		}
		savedFirstTime, err := binary.ReadVarint(r)
		if err != nil {
			log.Warn("Could not decode saved first time:", err)
			p.dirty = true
			return sm, chunksToPersist, nil
		}
		numChunkDescs, err := binary.ReadVarint(r)
		if err != nil {
			log.Warn("Could not decode number of chunk descriptors:", err)
			p.dirty = true
			return sm, chunksToPersist, nil
		}
		chunkDescs := make([]*chunkDesc, numChunkDescs)
		if version == headsFormatLegacyVersion {
			if headChunkPersisted {
				persistWatermark = numChunkDescs
			} else {
				persistWatermark = numChunkDescs - 1
			}
		}

		for i := int64(0); i < numChunkDescs; i++ {
			if i < persistWatermark {
				firstTime, err := binary.ReadVarint(r)
				if err != nil {
					log.Warn("Could not decode first time:", err)
					p.dirty = true
					return sm, chunksToPersist, nil
				}
				lastTime, err := binary.ReadVarint(r)
				if err != nil {
					log.Warn("Could not decode last time:", err)
					p.dirty = true
					return sm, chunksToPersist, nil
				}
				chunkDescs[i] = &chunkDesc{
					chunkFirstTime: clientmodel.Timestamp(firstTime),
					chunkLastTime:  clientmodel.Timestamp(lastTime),
				}
				chunkDescsTotal++
			} else {
				// Non-persisted chunk.
				encoding, err := r.ReadByte()
				if err != nil {
					log.Warn("Could not decode chunk type:", err)
					p.dirty = true
					return sm, chunksToPersist, nil
				}
				chunk := newChunkForEncoding(chunkEncoding(encoding))
				if err := chunk.unmarshal(r); err != nil {
					log.Warn("Could not decode chunk:", err)
					p.dirty = true
					return sm, chunksToPersist, nil
				}
				chunkDescs[i] = newChunkDesc(chunk)
				chunksToPersist++
			}
		}

		fingerprintToSeries[clientmodel.Fingerprint(fp)] = &memorySeries{
			metric:           clientmodel.Metric(metric),
			chunkDescs:       chunkDescs,
			persistWatermark: int(persistWatermark),
			modTime:          modTime,
			chunkDescsOffset: int(chunkDescsOffset),
			savedFirstTime:   clientmodel.Timestamp(savedFirstTime),
			headChunkClosed:  persistWatermark >= numChunkDescs,
		}
	}
	return sm, chunksToPersist, nil
}
Ejemplo n.º 10
0
// loadFPMappings loads the fingerprint mappings. It also returns the highest
// mapped fingerprint and any error encountered. If p.mappingsFileName is not
// found, the method returns (fpMappings{}, 0, nil). Do not call concurrently
// with checkpointFPMappings.
func (p *persistence) loadFPMappings() (fpMappings, clientmodel.Fingerprint, error) {
	fpm := fpMappings{}
	var highestMappedFP clientmodel.Fingerprint

	f, err := os.Open(p.mappingsFileName())
	if os.IsNotExist(err) {
		return fpm, 0, nil
	}
	if err != nil {
		return nil, 0, err
	}
	defer f.Close()
	r := bufio.NewReaderSize(f, fileBufSize)

	buf := make([]byte, len(mappingsMagicString))
	if _, err := io.ReadFull(r, buf); err != nil {
		return nil, 0, err
	}
	magic := string(buf)
	if magic != mappingsMagicString {
		return nil, 0, fmt.Errorf(
			"unexpected magic string, want %q, got %q",
			mappingsMagicString, magic,
		)
	}
	version, err := binary.ReadUvarint(r)
	if version != mappingsFormatVersion || err != nil {
		return nil, 0, fmt.Errorf("unknown fingerprint mappings format version, want %d", mappingsFormatVersion)
	}
	numRawFPs, err := binary.ReadUvarint(r)
	if err != nil {
		return nil, 0, err
	}
	for ; numRawFPs > 0; numRawFPs-- {
		rawFP, err := codable.DecodeUint64(r)
		if err != nil {
			return nil, 0, err
		}
		numMappings, err := binary.ReadUvarint(r)
		if err != nil {
			return nil, 0, err
		}
		mappings := make(map[string]clientmodel.Fingerprint, numMappings)
		for ; numMappings > 0; numMappings-- {
			lenMS, err := binary.ReadUvarint(r)
			if err != nil {
				return nil, 0, err
			}
			buf := make([]byte, lenMS)
			if _, err := io.ReadFull(r, buf); err != nil {
				return nil, 0, err
			}
			fp, err := codable.DecodeUint64(r)
			if err != nil {
				return nil, 0, err
			}
			mappedFP := clientmodel.Fingerprint(fp)
			if mappedFP > highestMappedFP {
				highestMappedFP = mappedFP
			}
			mappings[string(buf)] = mappedFP
		}
		fpm[clientmodel.Fingerprint(rawFP)] = mappings
	}
	return fpm, highestMappedFP, nil
}
Ejemplo n.º 11
0
	fileBufSize = 1 << 16 // 64kiB.

	chunkHeaderLen             = 17
	chunkHeaderTypeOffset      = 0
	chunkHeaderFirstTimeOffset = 1
	chunkHeaderLastTimeOffset  = 9
	chunkLenWithHeader         = chunkLen + chunkHeaderLen
	chunkMaxBatchSize          = 64 // How many chunks to load at most in one batch.

	indexingMaxBatchSize  = 1024 * 1024
	indexingBatchTimeout  = 500 * time.Millisecond // Commit batch when idle for that long.
	indexingQueueCapacity = 1024 * 16
)

var fpLen = len(clientmodel.Fingerprint(0).String()) // Length of a fingerprint as string.

const (
	flagHeadChunkPersisted byte = 1 << iota
	// Add more flags here like:
	// flagFoo
	// flagBar
)

type indexingOpType byte

const (
	add indexingOpType = iota
	remove
)
Ejemplo n.º 12
0
func (p *persistence) cleanUpArchiveIndexes(
	fpToSeries map[clientmodel.Fingerprint]*memorySeries,
	fpsSeen map[clientmodel.Fingerprint]struct{},
	fpm fpMappings,
) error {
	log.Info("Cleaning up archive indexes.")
	var fp codable.Fingerprint
	var m codable.Metric
	count := 0
	if err := p.archivedFingerprintToMetrics.ForEach(func(kv index.KeyValueAccessor) error {
		count++
		if count%10000 == 0 {
			log.Infof("%d archived metrics checked.", count)
		}
		if err := kv.Key(&fp); err != nil {
			return err
		}
		_, fpSeen := fpsSeen[clientmodel.Fingerprint(fp)]
		inMemory := false
		if fpSeen {
			_, inMemory = fpToSeries[clientmodel.Fingerprint(fp)]
		}
		if !fpSeen || inMemory {
			if inMemory {
				log.Warnf("Archive clean-up: Fingerprint %v is not archived. Purging from archive indexes.", clientmodel.Fingerprint(fp))
			}
			if !fpSeen {
				log.Warnf("Archive clean-up: Fingerprint %v is unknown. Purging from archive indexes.", clientmodel.Fingerprint(fp))
			}
			// It's fine if the fp is not in the archive indexes.
			if _, err := p.archivedFingerprintToMetrics.Delete(fp); err != nil {
				return err
			}
			// Delete from timerange index, too.
			_, err := p.archivedFingerprintToTimeRange.Delete(fp)
			return err
		}
		// fp is legitimately archived. Now we need the metric to check for a mapped fingerprint.
		if err := kv.Value(&m); err != nil {
			return err
		}
		maybeAddMapping(clientmodel.Fingerprint(fp), clientmodel.Metric(m), fpm)
		// Make sure it is in timerange index, too.
		has, err := p.archivedFingerprintToTimeRange.Has(fp)
		if err != nil {
			return err
		}
		if has {
			return nil // All good.
		}
		log.Warnf("Archive clean-up: Fingerprint %v is not in time-range index. Unarchiving it for recovery.")
		// Again, it's fine if fp is not in the archive index.
		if _, err := p.archivedFingerprintToMetrics.Delete(fp); err != nil {
			return err
		}
		cds, err := p.loadChunkDescs(clientmodel.Fingerprint(fp), 0)
		if err != nil {
			return err
		}
		series := newMemorySeries(clientmodel.Metric(m), cds, p.seriesFileModTime(clientmodel.Fingerprint(fp)))
		fpToSeries[clientmodel.Fingerprint(fp)] = series
		return nil
	}); err != nil {
		return err
	}
	count = 0
	if err := p.archivedFingerprintToTimeRange.ForEach(func(kv index.KeyValueAccessor) error {
		count++
		if count%10000 == 0 {
			log.Infof("%d archived time ranges checked.", count)
		}
		if err := kv.Key(&fp); err != nil {
			return err
		}
		has, err := p.archivedFingerprintToMetrics.Has(fp)
		if err != nil {
			return err
		}
		if has {
			return nil // All good.
		}
		log.Warnf("Archive clean-up: Purging unknown fingerprint %v in time-range index.", fp)
		deleted, err := p.archivedFingerprintToTimeRange.Delete(fp)
		if err != nil {
			return err
		}
		if !deleted {
			log.Errorf("Fingerprint %v to be deleted from archivedFingerprintToTimeRange not found. This should never happen.", fp)
		}
		return nil
	}); err != nil {
		return err
	}
	log.Info("Clean-up of archive indexes complete.")
	return nil
}
Ejemplo n.º 13
0
func TestFPMapper(t *testing.T) {
	sm := newSeriesMap()

	p, closer := newTestPersistence(t, 1)
	defer closer.Close()

	mapper, err := newFPMapper(sm, p)
	if err != nil {
		t.Fatal(err)
	}

	// Everything is empty, resolving a FP should do nothing.
	gotFP, err := mapper.mapFP(fp1, cm11)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm12)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// cm11 is in sm. Adding cm11 should do nothing. Mapping cm12 should resolve
	// the collision.
	sm.put(fp1, &memorySeries{metric: cm11})
	gotFP, err = mapper.mapFP(fp1, cm11)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm12)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// The mapped cm12 is added to sm, too. That should not change the outcome.
	sm.put(clientmodel.Fingerprint(1), &memorySeries{metric: cm12})
	gotFP, err = mapper.mapFP(fp1, cm11)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm12)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Now map cm13, should reproducibly result in the next mapped FP.
	gotFP, err = mapper.mapFP(fp1, cm13)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm13)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Add cm13 to sm. Should not change anything.
	sm.put(clientmodel.Fingerprint(2), &memorySeries{metric: cm13})
	gotFP, err = mapper.mapFP(fp1, cm11)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm12)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm13)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Now add cm21 and cm22 in the same way, checking the mapped FPs.
	gotFP, err = mapper.mapFP(fp2, cm21)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	sm.put(fp2, &memorySeries{metric: cm21})
	gotFP, err = mapper.mapFP(fp2, cm21)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp2, cm22)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(3); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	sm.put(clientmodel.Fingerprint(3), &memorySeries{metric: cm22})
	gotFP, err = mapper.mapFP(fp2, cm21)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp2, cm22)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(3); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Map cm31, resulting in a mapping straight away.
	gotFP, err = mapper.mapFP(fp3, cm31)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(4); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	sm.put(clientmodel.Fingerprint(4), &memorySeries{metric: cm31})

	// Map cm32, which is now mapped for two reasons...
	gotFP, err = mapper.mapFP(fp3, cm32)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(5); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	sm.put(clientmodel.Fingerprint(5), &memorySeries{metric: cm32})

	// Now check ALL the mappings, just to be sure.
	gotFP, err = mapper.mapFP(fp1, cm11)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm12)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm13)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp2, cm21)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp2, cm22)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(3); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp3, cm31)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(4); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp3, cm32)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(5); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Remove all the fingerprints from sm, which should change nothing, as
	// the existing mappings stay and should be detected.
	sm.del(fp1)
	sm.del(fp2)
	sm.del(fp3)
	gotFP, err = mapper.mapFP(fp1, cm11)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm12)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm13)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp2, cm21)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp2, cm22)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(3); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp3, cm31)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(4); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp3, cm32)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(5); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Load the mapper anew from disk and then check all the mappings again
	// to make sure all changes have made it to disk.
	mapper, err = newFPMapper(sm, p)
	if err != nil {
		t.Fatal(err)
	}
	gotFP, err = mapper.mapFP(fp1, cm11)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm12)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp1, cm13)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp2, cm21)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp2, cm22)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(3); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp3, cm31)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(4); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp3, cm32)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(5); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// To make sure that the mapping layer is not queried if the FP is found
	// in sm but the mapping layer is queried before going to the archive,
	// now put fp1 with cm12 in sm and fp2 with cm22 into archive (which
	// will never happen in practice as only mapped FPs are put into sm and
	// the archive).
	sm.put(fp1, &memorySeries{metric: cm12})
	p.archiveMetric(fp2, cm22, 0, 0)
	gotFP, err = mapper.mapFP(fp1, cm12)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := fp1; gotFP != wantFP { // No mapping happened.
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP, err = mapper.mapFP(fp2, cm22)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(3); gotFP != wantFP { // Old mapping still applied.
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// If we now map cm21, we should get a mapping as the collision with the
	// archived metric is detected. Again, this is a pathological situation
	// that must never happen in real operations. It's just staged here to
	// test the expected behavior.
	gotFP, err = mapper.mapFP(fp2, cm21)
	if err != nil {
		t.Fatal(err)
	}
	if wantFP := clientmodel.Fingerprint(6); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

}
Ejemplo n.º 14
0
package local

import (
	"testing"

	clientmodel "github.com/prometheus/client_golang/model"
)

var (
	// cm11, cm12, cm13 are colliding with fp1.
	// cm21, cm22 are colliding with fp2.
	// cm31, cm32 are colliding with fp3, which is below maxMappedFP.
	// Note that fingerprints are set and not actually calculated.
	// The collision detection is independent from the actually used
	// fingerprinting algorithm.
	fp1  = clientmodel.Fingerprint(maxMappedFP + 1)
	fp2  = clientmodel.Fingerprint(maxMappedFP + 2)
	fp3  = clientmodel.Fingerprint(1)
	cm11 = clientmodel.Metric{
		"foo":   "bar",
		"dings": "bumms",
	}
	cm12 = clientmodel.Metric{
		"bar": "foo",
	}
	cm13 = clientmodel.Metric{
		"foo": "bar",
	}
	cm21 = clientmodel.Metric{
		"foo":   "bumms",
		"dings": "bar",
Ejemplo n.º 15
0
// loadSeriesMapAndHeads loads the fingerprint to memory-series mapping and all
// open (non-full) head chunks. If recoverable corruption is detected, or if the
// dirty flag was set from the beginning, crash recovery is run, which might
// take a while. If an unrecoverable error is encountered, it is returned. Call
// this method during start-up while nothing else is running in storage
// land. This method is utterly goroutine-unsafe.
func (p *persistence) loadSeriesMapAndHeads() (sm *seriesMap, err error) {
	var chunksTotal, chunkDescsTotal int64
	fingerprintToSeries := make(map[clientmodel.Fingerprint]*memorySeries)
	sm = &seriesMap{m: fingerprintToSeries}

	defer func() {
		if sm != nil && p.dirty {
			glog.Warning("Persistence layer appears dirty.")
			err = p.recoverFromCrash(fingerprintToSeries)
			if err != nil {
				sm = nil
			}
		}
		if err == nil {
			atomic.AddInt64(&numMemChunks, chunksTotal)
			numMemChunkDescs.Add(float64(chunkDescsTotal))
		}
	}()

	f, err := os.Open(p.headsFileName())
	if os.IsNotExist(err) {
		return sm, nil
	}
	if err != nil {
		glog.Warning("Could not open heads file:", err)
		p.dirty = true
		return
	}
	defer f.Close()
	r := bufio.NewReaderSize(f, fileBufSize)

	buf := make([]byte, len(headsMagicString))
	if _, err := io.ReadFull(r, buf); err != nil {
		glog.Warning("Could not read from heads file:", err)
		p.dirty = true
		return sm, nil
	}
	magic := string(buf)
	if magic != headsMagicString {
		glog.Warningf(
			"unexpected magic string, want %q, got %q",
			headsMagicString, magic,
		)
		p.dirty = true
		return
	}
	if version, err := binary.ReadVarint(r); version != headsFormatVersion || err != nil {
		glog.Warningf("unknown heads format version, want %d", headsFormatVersion)
		p.dirty = true
		return sm, nil
	}
	numSeries, err := codable.DecodeUint64(r)
	if err != nil {
		glog.Warning("Could not decode number of series:", err)
		p.dirty = true
		return sm, nil
	}

	for ; numSeries > 0; numSeries-- {
		seriesFlags, err := r.ReadByte()
		if err != nil {
			glog.Warning("Could not read series flags:", err)
			p.dirty = true
			return sm, nil
		}
		headChunkPersisted := seriesFlags&flagHeadChunkPersisted != 0
		fp, err := codable.DecodeUint64(r)
		if err != nil {
			glog.Warning("Could not decode fingerprint:", err)
			p.dirty = true
			return sm, nil
		}
		var metric codable.Metric
		if err := metric.UnmarshalFromReader(r); err != nil {
			glog.Warning("Could not decode metric:", err)
			p.dirty = true
			return sm, nil
		}
		chunkDescsOffset, err := binary.ReadVarint(r)
		if err != nil {
			glog.Warning("Could not decode chunk descriptor offset:", err)
			p.dirty = true
			return sm, nil
		}
		savedFirstTime, err := binary.ReadVarint(r)
		if err != nil {
			glog.Warning("Could not decode saved first time:", err)
			p.dirty = true
			return sm, nil
		}
		numChunkDescs, err := binary.ReadVarint(r)
		if err != nil {
			glog.Warning("Could not decode number of chunk descriptors:", err)
			p.dirty = true
			return sm, nil
		}
		chunkDescs := make([]*chunkDesc, numChunkDescs)
		chunkDescsTotal += numChunkDescs

		for i := int64(0); i < numChunkDescs; i++ {
			if headChunkPersisted || i < numChunkDescs-1 {
				firstTime, err := binary.ReadVarint(r)
				if err != nil {
					glog.Warning("Could not decode first time:", err)
					p.dirty = true
					return sm, nil
				}
				lastTime, err := binary.ReadVarint(r)
				if err != nil {
					glog.Warning("Could not decode last time:", err)
					p.dirty = true
					return sm, nil
				}
				chunkDescs[i] = &chunkDesc{
					chunkFirstTime: clientmodel.Timestamp(firstTime),
					chunkLastTime:  clientmodel.Timestamp(lastTime),
				}
			} else {
				// Non-persisted head chunk.
				chunksTotal++
				chunkType, err := r.ReadByte()
				if err != nil {
					glog.Warning("Could not decode chunk type:", err)
					p.dirty = true
					return sm, nil
				}
				chunk := chunkForType(chunkType)
				if err := chunk.unmarshal(r); err != nil {
					glog.Warning("Could not decode chunk type:", err)
					p.dirty = true
					return sm, nil
				}
				chunkDescs[i] = newChunkDesc(chunk)
			}
		}

		fingerprintToSeries[clientmodel.Fingerprint(fp)] = &memorySeries{
			metric:             clientmodel.Metric(metric),
			chunkDescs:         chunkDescs,
			chunkDescsOffset:   int(chunkDescsOffset),
			savedFirstTime:     clientmodel.Timestamp(savedFirstTime),
			headChunkPersisted: headChunkPersisted,
		}
	}
	return sm, nil
}
Ejemplo n.º 16
0
func (p *persistence) cleanUpArchiveIndexes(
	fpToSeries map[clientmodel.Fingerprint]*memorySeries,
	fpsSeen map[clientmodel.Fingerprint]struct{},
) error {
	glog.Info("Cleaning up archive indexes.")
	var fp codable.Fingerprint
	var m codable.Metric
	count := 0
	if err := p.archivedFingerprintToMetrics.ForEach(func(kv index.KeyValueAccessor) error {
		count++
		if count%10000 == 0 {
			glog.Infof("%d archived metrics checked.", count)
		}
		if err := kv.Key(&fp); err != nil {
			return err
		}
		_, fpSeen := fpsSeen[clientmodel.Fingerprint(fp)]
		inMemory := false
		if fpSeen {
			_, inMemory = fpToSeries[clientmodel.Fingerprint(fp)]
		}
		if !fpSeen || inMemory {
			if inMemory {
				glog.Warningf("Archive clean-up: Fingerprint %v is not archived. Purging from archive indexes.", clientmodel.Fingerprint(fp))
			}
			if !fpSeen {
				glog.Warningf("Archive clean-up: Fingerprint %v is unknown. Purging from archive indexes.", clientmodel.Fingerprint(fp))
			}
			if err := p.archivedFingerprintToMetrics.Delete(fp); err != nil {
				return err
			}
			// Delete from timerange index, too.
			p.archivedFingerprintToTimeRange.Delete(fp)
			// TODO: Ignoring errors here as fp might not be in
			// timerange index (which is good) but which would
			// return an error. Delete signature could be changed
			// like the Get signature to detect a real error.
			return nil
		}
		// fp is legitimately archived. Make sure it is in timerange index, too.
		has, err := p.archivedFingerprintToTimeRange.Has(fp)
		if err != nil {
			return err
		}
		if has {
			return nil // All good.
		}
		glog.Warningf("Archive clean-up: Fingerprint %v is not in time-range index. Unarchiving it for recovery.")
		if err := p.archivedFingerprintToMetrics.Delete(fp); err != nil {
			return err
		}
		if err := kv.Value(&m); err != nil {
			return err
		}
		series := newMemorySeries(clientmodel.Metric(m), false, math.MinInt64)
		cds, err := p.loadChunkDescs(clientmodel.Fingerprint(fp), clientmodel.Now())
		if err != nil {
			return err
		}
		series.chunkDescs = cds
		series.chunkDescsOffset = 0
		fpToSeries[clientmodel.Fingerprint(fp)] = series
		return nil
	}); err != nil {
		return err
	}
	count = 0
	if err := p.archivedFingerprintToTimeRange.ForEach(func(kv index.KeyValueAccessor) error {
		count++
		if count%10000 == 0 {
			glog.Infof("%d archived time ranges checked.", count)
		}
		if err := kv.Key(&fp); err != nil {
			return err
		}
		has, err := p.archivedFingerprintToMetrics.Has(fp)
		if err != nil {
			return err
		}
		if has {
			return nil // All good.
		}
		glog.Warningf("Archive clean-up: Purging unknown fingerprint %v in time-range index.", fp)
		if err := p.archivedFingerprintToTimeRange.Delete(fp); err != nil {
			return err
		}
		return nil
	}); err != nil {
		return err
	}
	glog.Info("Clean-up of archive indexes complete.")
	return nil
}