Beispiel #1
0
// FindValidDataPoints does a backwards walk through time to examine the
// highest resolution data for each archive / time period.  We collect valid
// data points and return them in a *[]TimeSeriesPoint.  The second value
// return is an int containing the total number of points examined.  This
// allows one to calculate the percentage of used and unused points stored
// in the Whisper database.
func FindValidDataPoints(wsp *whisper.Whisper) ([]*whisper.TimeSeriesPoint, int, error) {
	points := make([]*whisper.TimeSeriesPoint, 0)
	count := 0
	retentions := whisper.RetentionsByPrecision{wsp.Retentions()}
	sort.Sort(retentions)

	start := int(time.Now().Unix())
	from := 0
	for _, r := range retentions.Iterator() {
		from = int(time.Now().Unix()) - r.MaxRetention()

		ts, err := wsp.Fetch(from, start)
		if err != nil {
			return make([]*whisper.TimeSeriesPoint, 0, 0), 0, err
		}
		count = count + len(ts.Values())
		for _, v := range ts.Points() {
			if !math.IsNaN(v.Value) {
				points = append(points, v)
			}
		}

		start = from
	}

	return points, count, nil
}
Beispiel #2
0
// OpenWSP() runs the fill operation on two whisper.Whisper objects that are
// already open.
// * srcWsp - source *whisper.Whisper object
// * dstWsp - destination *whisper,Whisper object
// * startTime - Unix time such as int(time.Now().Unix()).  We fill from
//   this time walking backwards to the beginning.
//
// This code heavily inspired by https://github.com/jssjr/carbonate
// and matches its behavior exactly.
func OpenWSP(srcWsp, dstWsp *whisper.Whisper, startTime int) error {
	// Loop over each archive/retention, highest resolution first
	dstRetentions := whisper.RetentionsByPrecision{dstWsp.Retentions()}
	sort.Sort(dstRetentions)
	for _, v := range dstRetentions.Iterator() {
		// fromTime is the earliest timestamp in this archive
		fromTime := int(time.Now().Unix()) - v.MaxRetention()
		if fromTime >= startTime {
			continue
		}

		// Fetch data from dest for this archive
		ts, err := dstWsp.Fetch(fromTime, startTime)
		if err != nil {
			return err
		}

		// FSM: Find gaps, and fill them from the source
		start := ts.FromTime()
		gapstart := -1
		for _, dp := range ts.Values() {
			if math.IsNaN(dp) && gapstart < 0 {
				gapstart = start
			} else if !math.IsNaN(dp) && gapstart >= 0 {
				// Carbonate ignores single units lost.  Means:
				// XXX: Gap of a single step are ignored as the
				// following if uses > not, =>
				if (start - gapstart) > v.SecondsPerPoint() {
					// XXX: Fence post: This replaces the
					// current DP -- a known good value
					fillArchive(srcWsp, dstWsp, gapstart-ts.Step(), start)
					// We always fill starting at gap-step
					// because the Fetch() command will pull
					// the next valid interval's point even
					// if we give it a valid interval.
				}
				gapstart = -1
			} else if gapstart >= 0 && start == ts.UntilTime()-ts.Step() {
				// The timeSeries doesn't actually include a
				// value for ts.UntilTime(), like len() we need
				// to subtract a step to index the last value
				fillArchive(srcWsp, dstWsp, gapstart-ts.Step(), start)
			}

			start += ts.Step()
		}

		// reset startTime so that we can examine the next highest
		// resolution archive without the first getting in the way
		startTime = fromTime
	}

	return nil
}
Beispiel #3
0
// fillArchive() is a private function that fills data points from srcWSP
// into dstWsp.  Used by FIll()
// * srcWsp and dstWsp are *whisper.Whisper open files
// * start and stop define an inclusive time window to fill
// On error an error value is returned.
//
// This code heavily inspired by https://github.com/jssjr/carbonate
func fillArchive(srcWsp, dstWsp *whisper.Whisper, start, stop int) error {
	// Fetch the range defined by start and stop always taking the values
	// from the highest precision archive, which man require multiple
	// fetch/merge updates.
	srcRetentions := whisper.RetentionsByPrecision{srcWsp.Retentions()}
	sort.Sort(srcRetentions)

	if start < srcWsp.StartTime() && stop < srcWsp.StartTime() {
		// Nothing to fill/merge
		return nil
	}

	// Begin our backwards walk in time
	for _, v := range srcRetentions.Iterator() {
		points := make([]*whisper.TimeSeriesPoint, 0)
		rTime := int(time.Now().Unix()) - v.MaxRetention()
		if stop <= rTime {
			// This archive contains no data points in the window
			continue
		}

		// Start and the start time or the beginning of this archive
		fromTime := start
		if rTime > start {
			fromTime = rTime
		}

		ts, err := srcWsp.Fetch(fromTime, stop)
		if err != nil {
			return err
		}
		// Build a list of points to merge
		tsStart := ts.FromTime()
		for _, dp := range ts.Values() {
			if !math.IsNaN(dp) {
				points = append(points, &whisper.TimeSeriesPoint{tsStart, dp})
			}
			tsStart += ts.Step()
		}
		dstWsp.UpdateMany(points)

		stop = fromTime
		if start >= stop {
			// Nothing more to fetch
			break
		}
	}

	return nil
}