func whisperWorker() {
for path := range whisperFiles {
fd, err := os.Open(path)
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: Failed to open whisper file '%s': %s\n", path, err.Error())
if !skipWhisperErrors {
return
}
}
w, err := whisper.OpenWhisper(fd)
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: Failed to open whisper file '%s': %s\n", path, err.Error())
if !skipWhisperErrors {
return
}
}
var earliestArchiveTimestamp uint32
numTotalPoints := uint32(0)
for i := range w.Header.Archives {
numTotalPoints += w.Header.Archives[i].Points
}
points := make(map[uint32]whisper.Point)
// iterate over archives from high-res to low-res, remembering
// the earliest timestamp and filtering out those rows from
// subsequent archives so as to never have duplicates.
for i, archive := range w.Header.Archives {
allPoints, err := w.DumpArchive(i)
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: Failed to read archive %d in '%s', skipping: %s\n", i, path, err.Error())
if !skipWhisperErrors {
return
}
}
var earliestTimestamp uint32 = 0
var latestTimestamp uint32 = 0
n := 0
for _, point := range allPoints {
// we have to filter out the "None" records (where we didn't fill in data) explicitly here!
if point.Timestamp != 0 {
if earliestArchiveTimestamp == 0 || point.Timestamp < earliestArchiveTimestamp {
point.Value = point.Value
points[point.Timestamp] = point
n += 1
if earliestTimestamp > point.Timestamp || earliestTimestamp == 0 {
earliestTimestamp = point.Timestamp
}
if latestTimestamp < point.Timestamp || latestTimestamp == 0 {
latestTimestamp = point.Timestamp
}
}
}
}
// And now we need to remove all points that are latest - retention
if earliestTimestamp < latestTimestamp-archive.SecondsPerPoint*archive.Points {
earliestArchiveTimestamp = latestTimestamp - archive.SecondsPerPoint*archive.Points
} else {
earliestArchiveTimestamp = earliestTimestamp
}
n = 0
for ts, _ := range points {
if ts < earliestArchiveTimestamp {
delete(points, ts)
n += 1
}
}
}
w.Close()
serie := &abstractSerie{path, points}
series <- serie
}
whisperWorkersWg.Done()
}
func whisperWorker() {
for path := range whisperFiles {
fd, err := os.Open(path)
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: Failed to open whisper file '%s': %s\n", path, err.Error())
if skipWhisperErrors {
continue
} else {
exit <- 2
}
}
w, err := whisper.OpenWhisper(fd)
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: Failed to open whisper file '%s': %s\n", path, err.Error())
if skipWhisperErrors {
continue
} else {
exit <- 2
}
}
pre := time.Now()
var duration time.Duration
var points []whisper.Point
if all {
numTotalPoints := uint32(0)
for i := range w.Header.Archives {
numTotalPoints += w.Header.Archives[i].Points
}
points = make([]whisper.Point, 0, numTotalPoints)
// iterate in backwards archive order (low res to high res)
// so that if you write points of multiple archives to the same series, the high res ones will overwrite the low res ones
for i := len(w.Header.Archives) - 1; i >= 0; i-- {
allPoints, err := w.DumpArchive(i)
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: Failed to read archive %d in '%s', skipping: %s\n", i, path, err.Error())
if skipWhisperErrors {
continue
} else {
exit <- 2
}
}
for _, point := range allPoints {
// we have to filter out the "None" records (where we didn't fill in data) explicitly here!
if point.Timestamp != 0 {
points = append(points, point)
}
}
}
duration = time.Since(pre)
} else {
// not sure how it works, but i've emperically verified that this ignores null records, which is what we want
// i.e. if whisper has a slot every minute, but you only have data every 3 minutes, we'll only process those records
_, points, err = w.FetchUntil(fromTime, untilTime)
duration = time.Since(pre)
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: Failed to read file '%s' from %d to %d, skipping: %s (operation took %v)\n", path, fromTime, untilTime, err.Error(), duration)
if skipWhisperErrors {
w.Close()
continue
} else {
exit <- 2
}
}
}
w.Close()
whisperReadTimer.Update(duration)
serie := &abstractSerie{path, points}
influxSeries <- serie
}
whisperWorkersWg.Done()
}