func (self *CoordinatorImpl) CommitSeriesData(db string, series *protocol.Series) error {
lastPointIndex := 0
now := common.CurrentTime()
var shardToWrite cluster.Shard
for _, point := range series.Points {
if point.Timestamp == nil {
point.Timestamp = &now
}
}
lastTime := int64(math.MinInt64)
if len(series.Points) > 0 && *series.Points[0].Timestamp == lastTime {
// just a hack to make sure lastTime will never equal the first
// point's timestamp
lastTime = 0
}
// sort the points by timestamp
series.SortPointsTimeDescending()
for i, point := range series.Points {
if *point.Timestamp != lastTime {
shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *point.Timestamp)
if err != nil {
return err
}
if shardToWrite == nil {
shardToWrite = shard
} else if shardToWrite.Id() != shard.Id() {
newIndex := i
newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[lastPointIndex:newIndex]}
if err := self.write(db, newSeries, shardToWrite); err != nil {
return err
}
lastPointIndex = newIndex
shardToWrite = shard
}
lastTime = *point.Timestamp
}
}
series.Points = series.Points[lastPointIndex:]
if len(series.Points) > 0 {
if shardToWrite == nil {
shardToWrite, _ = self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *series.Points[0].Timestamp)
}
err := self.write(db, series, shardToWrite)
if err != nil {
log.Error("COORD error writing: ", err)
return err
}
return err
}
return nil
}
// TODO: refactor this for clarity. This got super ugly...
// Function yields all results that are safe to do so ensuring order. Returns all results that must wait for more from the servers.
func (self *CoordinatorImpl) yieldResultsForSeries(isAscending bool, leftover *protocol.Series, responses []*protocol.Response, yield func(*protocol.Series) error) *protocol.Series {
// results can come from different servers. Some of which won't know about fields that other servers may know about.
// We need to normalize all this so that all fields are represented and the other field values are null.
// Give each unique field name an index. We'll use this map later to construct the results and make sure that
// the response objects have their fields in the result.
fieldIndexes := make(map[string]int)
for _, response := range responses {
for _, name := range response.Series.Fields {
if _, hasField := fieldIndexes[name]; !hasField {
fieldIndexes[name] = len(fieldIndexes)
}
}
}
fields := make([]string, len(fieldIndexes), len(fieldIndexes))
for name, index := range fieldIndexes {
fields[index] = name
}
fieldCount := len(fields)
result := &protocol.Series{Name: responses[0].Series.Name, Fields: fields, Points: make([]*protocol.Point, 0)}
if leftover == nil {
leftover = &protocol.Series{Name: responses[0].Series.Name, Fields: fields, Points: make([]*protocol.Point, 0)}
}
barrierTime := BARRIER_TIME_MIN
if isAscending {
barrierTime = BARRIER_TIME_MAX
}
var shouldYieldComparator func(rawTime *int64) bool
if isAscending {
shouldYieldComparator = func(rawTime *int64) bool {
if rawTime != nil && *rawTime < barrierTime {
return true
} else {
return false
}
}
} else {
shouldYieldComparator = func(rawTime *int64) bool {
if rawTime != nil && *rawTime > barrierTime {
return true
} else {
return false
}
}
}
// find the barrier time
for _, response := range responses {
if shouldYieldComparator(response.NextPointTime) {
barrierTime = *response.NextPointTime
}
}
// yield the points from leftover that are safe
for _, point := range leftover.Points {
if shouldYieldComparator(point.Timestamp) {
result.Points = append(result.Points, point)
} else {
break
}
}
// if they all got added, clear out the leftover
if len(leftover.Points) == len(result.Points) {
leftover.Points = make([]*protocol.Point, 0)
}
if barrierTime == BARRIER_TIME_MIN || barrierTime == BARRIER_TIME_MAX {
// all the nextPointTimes were nil so we're safe to send everything
for _, response := range responses {
// if this is the case we know that all responses contained the same
// fields. So just append the points
if len(response.Series.Fields) == fieldCount {
result.Points = append(result.Points, response.Series.Points...)
} else {
log.Debug("Responses from servers had different numbers of fields.")
for _, p := range response.Series.Points {
self.normalizePointAndAppend(fieldIndexes, result, response.Series.Fields, p)
}
}
}
if len(leftover.Fields) == fieldCount {
result.Points = append(result.Points, leftover.Points...)
leftover.Points = []*protocol.Point{}
} else {
log.Debug("Responses from servers had different numbers of fields.")
for _, p := range leftover.Points {
self.normalizePointAndAppend(fieldIndexes, result, leftover.Fields, p)
}
}
} else {
for _, response := range responses {
if shouldYieldComparator(response.NextPointTime) {
// all points safe to yield
if fieldCount == len(response.Series.Fields) {
result.Points = append(result.Points, response.Series.Points...)
} else {
log.Debug("Responses from servers had different numbers of fields.")
for _, p := range response.Series.Points {
self.normalizePointAndAppend(fieldIndexes, result, response.Series.Fields, p)
}
}
continue
}
if fieldCount == len(response.Series.Fields) {
for i, point := range response.Series.Points {
if shouldYieldComparator(point.Timestamp) {
result.Points = append(result.Points, point)
} else {
// since they're returned in order, we can just append these to
// the leftover and break out.
leftover.Points = append(leftover.Points, response.Series.Points[i:]...)
break
}
}
} else {
for i, point := range response.Series.Points {
if shouldYieldComparator(point.Timestamp) {
self.normalizePointAndAppend(fieldIndexes, result, response.Series.Fields, point)
} else {
// since they're returned in order, we can just append these to
// the leftover and break out.
for _, point := range response.Series.Points[i:] {
self.normalizePointAndAppend(fieldIndexes, leftover, response.Series.Fields, point)
}
break
}
}
}
}
}
if isAscending {
result.SortPointsTimeAscending()
leftover.SortPointsTimeAscending()
} else {
result.SortPointsTimeDescending()
leftover.SortPointsTimeDescending()
}
// Don't yield an empty points array, the engine will think it's the end of the stream.
// streamResultsFromChannels will send the empty ones after all channels have returned.
if len(result.Points) > 0 {
yield(result)
}
if len(leftover.Points) > 0 {
return leftover
}
return nil
}