// FindValidDataPoints does a backwards walk through time to examine the
// highest resolution data for each archive / time period. We collect valid
// data points and return them in a *[]TimeSeriesPoint. The second value
// return is an int containing the total number of points examined. This
// allows one to calculate the percentage of used and unused points stored
// in the Whisper database.
func FindValidDataPoints(wsp *whisper.Whisper) ([]*whisper.TimeSeriesPoint, int, error) {
points := make([]*whisper.TimeSeriesPoint, 0)
count := 0
retentions := whisper.RetentionsByPrecision{wsp.Retentions()}
sort.Sort(retentions)
start := int(time.Now().Unix())
from := 0
for _, r := range retentions.Iterator() {
from = int(time.Now().Unix()) - r.MaxRetention()
ts, err := wsp.Fetch(from, start)
if err != nil {
return make([]*whisper.TimeSeriesPoint, 0, 0), 0, err
}
count = count + len(ts.Values())
for _, v := range ts.Points() {
if !math.IsNaN(v.Value) {
points = append(points, v)
}
}
start = from
}
return points, count, nil
}
// OpenWSP() runs the fill operation on two whisper.Whisper objects that are
// already open.
// * srcWsp - source *whisper.Whisper object
// * dstWsp - destination *whisper,Whisper object
// * startTime - Unix time such as int(time.Now().Unix()). We fill from
// this time walking backwards to the beginning.
//
// This code heavily inspired by https://github.com/jssjr/carbonate
// and matches its behavior exactly.
func OpenWSP(srcWsp, dstWsp *whisper.Whisper, startTime int) error {
// Loop over each archive/retention, highest resolution first
dstRetentions := whisper.RetentionsByPrecision{dstWsp.Retentions()}
sort.Sort(dstRetentions)
for _, v := range dstRetentions.Iterator() {
// fromTime is the earliest timestamp in this archive
fromTime := int(time.Now().Unix()) - v.MaxRetention()
if fromTime >= startTime {
continue
}
// Fetch data from dest for this archive
ts, err := dstWsp.Fetch(fromTime, startTime)
if err != nil {
return err
}
// FSM: Find gaps, and fill them from the source
start := ts.FromTime()
gapstart := -1
for _, dp := range ts.Values() {
if math.IsNaN(dp) && gapstart < 0 {
gapstart = start
} else if !math.IsNaN(dp) && gapstart >= 0 {
// Carbonate ignores single units lost. Means:
// XXX: Gap of a single step are ignored as the
// following if uses > not, =>
if (start - gapstart) > v.SecondsPerPoint() {
// XXX: Fence post: This replaces the
// current DP -- a known good value
fillArchive(srcWsp, dstWsp, gapstart-ts.Step(), start)
// We always fill starting at gap-step
// because the Fetch() command will pull
// the next valid interval's point even
// if we give it a valid interval.
}
gapstart = -1
} else if gapstart >= 0 && start == ts.UntilTime()-ts.Step() {
// The timeSeries doesn't actually include a
// value for ts.UntilTime(), like len() we need
// to subtract a step to index the last value
fillArchive(srcWsp, dstWsp, gapstart-ts.Step(), start)
}
start += ts.Step()
}
// reset startTime so that we can examine the next highest
// resolution archive without the first getting in the way
startTime = fromTime
}
return nil
}
// fillArchive() is a private function that fills data points from srcWSP
// into dstWsp. Used by FIll()
// * srcWsp and dstWsp are *whisper.Whisper open files
// * start and stop define an inclusive time window to fill
// On error an error value is returned.
//
// This code heavily inspired by https://github.com/jssjr/carbonate
func fillArchive(srcWsp, dstWsp *whisper.Whisper, start, stop int) error {
// Fetch the range defined by start and stop always taking the values
// from the highest precision archive, which man require multiple
// fetch/merge updates.
srcRetentions := whisper.RetentionsByPrecision{srcWsp.Retentions()}
sort.Sort(srcRetentions)
if start < srcWsp.StartTime() && stop < srcWsp.StartTime() {
// Nothing to fill/merge
return nil
}
// Begin our backwards walk in time
for _, v := range srcRetentions.Iterator() {
points := make([]*whisper.TimeSeriesPoint, 0)
rTime := int(time.Now().Unix()) - v.MaxRetention()
if stop <= rTime {
// This archive contains no data points in the window
continue
}
// Start and the start time or the beginning of this archive
fromTime := start
if rTime > start {
fromTime = rTime
}
ts, err := srcWsp.Fetch(fromTime, stop)
if err != nil {
return err
}
// Build a list of points to merge
tsStart := ts.FromTime()
for _, dp := range ts.Values() {
if !math.IsNaN(dp) {
points = append(points, &whisper.TimeSeriesPoint{tsStart, dp})
}
tsStart += ts.Step()
}
dstWsp.UpdateMany(points)
stop = fromTime
if start >= stop {
// Nothing more to fetch
break
}
}
return nil
}