Exemplo n.º 1
0
func newDiskFrontier(i leveldb.Iterator) (d *diskFrontier, err error) {

	if !i.SeekToLast() || i.Key() == nil {
		return
	}

	lastKey, err := extractSampleKey(i)
	if err != nil {
		panic(err)
	}

	if !i.SeekToFirst() || i.Key() == nil {
		return
	}
	firstKey, err := extractSampleKey(i)
	if i.Key() == nil {
		return
	}
	if err != nil {
		panic(err)
	}

	d = &diskFrontier{}

	d.firstFingerprint = model.NewFingerprintFromRowKey(*firstKey.Fingerprint.Signature)
	d.firstSupertime = indexable.DecodeTime(firstKey.Timestamp)
	d.lastFingerprint = model.NewFingerprintFromRowKey(*lastKey.Fingerprint.Signature)
	d.lastSupertime = indexable.DecodeTime(lastKey.Timestamp)

	return
}
Exemplo n.º 2
0
func (t *TieredStorage) renderView(viewJob viewJob) {
	// Telemetry.
	var err error
	begin := time.Now()
	defer func() {
		t.memorySemaphore <- true

		duration := time.Since(begin)

		recordOutcome(
			duration,
			err,
			map[string]string{operation: renderView, result: success},
			map[string]string{operation: renderView, result: failure},
		)
	}()

	view := newView()

	var iterator leveldb.Iterator
	diskPresent := true

	firstBlock, _ := t.sampleKeys.Get()
	defer t.sampleKeys.Give(firstBlock)

	lastBlock, _ := t.sampleKeys.Get()
	defer t.sampleKeys.Give(lastBlock)

	sampleKeyDto, _ := t.dtoSampleKeys.Get()
	defer t.dtoSampleKeys.Give(sampleKeyDto)

	defer func() {
		// Give back all ops not yet popped.
		for viewJob.builder.HasOp() {
			giveBackOp(viewJob.builder.PopOp())
		}
	}()

	extractionTimer := viewJob.stats.GetTimer(stats.ViewDataExtractionTime).Start()
	for viewJob.builder.HasOp() {
		op := viewJob.builder.PopOp()
		defer giveBackOp(op)

		fp := op.Fingerprint()
		old, err := t.seriesTooOld(fp, op.CurrentTime())
		if err != nil {
			glog.Errorf("Error getting watermark from cache for %s: %s", fp, err)
			continue
		}
		if old {
			continue
		}

		memValues := t.memoryArena.CloneSamples(fp)

		for !op.Consumed() {
			// Abort the view rendering if the caller (makeView) has timed out.
			if len(viewJob.abort) > 0 {
				return
			}

			// Load data value chunk(s) around the current time.
			targetTime := op.CurrentTime()

			currentChunk := chunk{}
			// If we aimed before the oldest value in memory, load more data from disk.
			if (len(memValues) == 0 || memValues.FirstTimeAfter(targetTime)) && diskPresent {
				if iterator == nil {
					// Get a single iterator that will be used for all data extraction
					// below.
					iterator, _ = t.DiskStorage.MetricSamples.NewIterator(true)
					defer iterator.Close()
					if diskPresent = iterator.SeekToLast(); diskPresent {
						if err := iterator.Key(sampleKeyDto); err != nil {
							panic(err)
						}

						lastBlock.Load(sampleKeyDto)

						if !iterator.SeekToFirst() {
							diskPresent = false
						} else {
							if err := iterator.Key(sampleKeyDto); err != nil {
								panic(err)
							}

							firstBlock.Load(sampleKeyDto)
						}
					}
				}

				if diskPresent {
					diskTimer := viewJob.stats.GetTimer(stats.ViewDiskExtractionTime).Start()
					diskValues, expired := t.loadChunkAroundTime(
						iterator,
						fp,
						targetTime,
						firstBlock,
						lastBlock,
					)
					if expired {
						diskPresent = false
					}
					diskTimer.Stop()

					// If we aimed past the newest value on disk,
					// combine it with the next value from memory.
					if len(diskValues) == 0 {
						currentChunk = chunk(memValues)
					} else {
						if len(memValues) > 0 && diskValues.LastTimeBefore(targetTime) {
							latestDiskValue := diskValues[len(diskValues)-1:]
							currentChunk = append(chunk(latestDiskValue), chunk(memValues)...)
						} else {
							currentChunk = chunk(diskValues)
						}
					}
				} else {
					currentChunk = chunk(memValues)
				}
			} else {
				currentChunk = chunk(memValues)
			}

			// There's no data at all for this fingerprint, so stop processing.
			if len(currentChunk) == 0 {
				break
			}

			currentChunk = currentChunk.TruncateBefore(targetTime)

			lastChunkTime := currentChunk[len(currentChunk)-1].Timestamp
			if lastChunkTime.After(targetTime) {
				targetTime = lastChunkTime
			}

			if op.CurrentTime().After(targetTime) {
				break
			}

			// Extract all needed data from the current chunk and append the
			// extracted samples to the materialized view.
			for !op.Consumed() && !op.CurrentTime().After(targetTime) {
				view.appendSamples(fp, op.ExtractSamples(metric.Values(currentChunk)))
			}
		}
	}
	extractionTimer.Stop()

	viewJob.output <- view
	return
}