func (self *CoordinatorImpl) CommitSeriesData(db string, series *protocol.Series) error { lastPointIndex := 0 now := common.CurrentTime() var shardToWrite cluster.Shard for _, point := range series.Points { if point.Timestamp == nil { point.Timestamp = &now } } lastTime := int64(math.MinInt64) if len(series.Points) > 0 && *series.Points[0].Timestamp == lastTime { // just a hack to make sure lastTime will never equal the first // point's timestamp lastTime = 0 } // sort the points by timestamp series.SortPointsTimeDescending() for i, point := range series.Points { if *point.Timestamp != lastTime { shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *point.Timestamp) if err != nil { return err } if shardToWrite == nil { shardToWrite = shard } else if shardToWrite.Id() != shard.Id() { newIndex := i newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[lastPointIndex:newIndex]} if err := self.write(db, newSeries, shardToWrite); err != nil { return err } lastPointIndex = newIndex shardToWrite = shard } lastTime = *point.Timestamp } } series.Points = series.Points[lastPointIndex:] if len(series.Points) > 0 { if shardToWrite == nil { shardToWrite, _ = self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *series.Points[0].Timestamp) } err := self.write(db, series, shardToWrite) if err != nil { log.Error("COORD error writing: ", err) return err } return err } return nil }
// TODO: refactor this for clarity. This got super ugly... // Function yields all results that are safe to do so ensuring order. Returns all results that must wait for more from the servers. func (self *CoordinatorImpl) yieldResultsForSeries(isAscending bool, leftover *protocol.Series, responses []*protocol.Response, yield func(*protocol.Series) error) *protocol.Series { // results can come from different servers. Some of which won't know about fields that other servers may know about. // We need to normalize all this so that all fields are represented and the other field values are null. // Give each unique field name an index. We'll use this map later to construct the results and make sure that // the response objects have their fields in the result. fieldIndexes := make(map[string]int) for _, response := range responses { for _, name := range response.Series.Fields { if _, hasField := fieldIndexes[name]; !hasField { fieldIndexes[name] = len(fieldIndexes) } } } fields := make([]string, len(fieldIndexes), len(fieldIndexes)) for name, index := range fieldIndexes { fields[index] = name } fieldCount := len(fields) result := &protocol.Series{Name: responses[0].Series.Name, Fields: fields, Points: make([]*protocol.Point, 0)} if leftover == nil { leftover = &protocol.Series{Name: responses[0].Series.Name, Fields: fields, Points: make([]*protocol.Point, 0)} } barrierTime := BARRIER_TIME_MIN if isAscending { barrierTime = BARRIER_TIME_MAX } var shouldYieldComparator func(rawTime *int64) bool if isAscending { shouldYieldComparator = func(rawTime *int64) bool { if rawTime != nil && *rawTime < barrierTime { return true } else { return false } } } else { shouldYieldComparator = func(rawTime *int64) bool { if rawTime != nil && *rawTime > barrierTime { return true } else { return false } } } // find the barrier time for _, response := range responses { if shouldYieldComparator(response.NextPointTime) { barrierTime = *response.NextPointTime } } // yield the points from leftover that are safe for _, point := range leftover.Points { if shouldYieldComparator(point.Timestamp) { result.Points = append(result.Points, point) } else { break } } // if they all got added, clear out the leftover if len(leftover.Points) == len(result.Points) { leftover.Points = make([]*protocol.Point, 0) } if barrierTime == BARRIER_TIME_MIN || barrierTime == BARRIER_TIME_MAX { // all the nextPointTimes were nil so we're safe to send everything for _, response := range responses { // if this is the case we know that all responses contained the same // fields. So just append the points if len(response.Series.Fields) == fieldCount { result.Points = append(result.Points, response.Series.Points...) } else { log.Debug("Responses from servers had different numbers of fields.") for _, p := range response.Series.Points { self.normalizePointAndAppend(fieldIndexes, result, response.Series.Fields, p) } } } if len(leftover.Fields) == fieldCount { result.Points = append(result.Points, leftover.Points...) leftover.Points = []*protocol.Point{} } else { log.Debug("Responses from servers had different numbers of fields.") for _, p := range leftover.Points { self.normalizePointAndAppend(fieldIndexes, result, leftover.Fields, p) } } } else { for _, response := range responses { if shouldYieldComparator(response.NextPointTime) { // all points safe to yield if fieldCount == len(response.Series.Fields) { result.Points = append(result.Points, response.Series.Points...) } else { log.Debug("Responses from servers had different numbers of fields.") for _, p := range response.Series.Points { self.normalizePointAndAppend(fieldIndexes, result, response.Series.Fields, p) } } continue } if fieldCount == len(response.Series.Fields) { for i, point := range response.Series.Points { if shouldYieldComparator(point.Timestamp) { result.Points = append(result.Points, point) } else { // since they're returned in order, we can just append these to // the leftover and break out. leftover.Points = append(leftover.Points, response.Series.Points[i:]...) break } } } else { for i, point := range response.Series.Points { if shouldYieldComparator(point.Timestamp) { self.normalizePointAndAppend(fieldIndexes, result, response.Series.Fields, point) } else { // since they're returned in order, we can just append these to // the leftover and break out. for _, point := range response.Series.Points[i:] { self.normalizePointAndAppend(fieldIndexes, leftover, response.Series.Fields, point) } break } } } } } if isAscending { result.SortPointsTimeAscending() leftover.SortPointsTimeAscending() } else { result.SortPointsTimeDescending() leftover.SortPointsTimeDescending() } // Don't yield an empty points array, the engine will think it's the end of the stream. // streamResultsFromChannels will send the empty ones after all channels have returned. if len(result.Points) > 0 { yield(result) } if len(leftover.Points) > 0 { return leftover } return nil }