Beispiel #1
0
func (self *CoordinatorImpl) CommitSeriesData(db string, series *protocol.Series) error {
	lastPointIndex := 0
	now := common.CurrentTime()
	var shardToWrite cluster.Shard
	for _, point := range series.Points {
		if point.Timestamp == nil {
			point.Timestamp = &now
		}
	}

	lastTime := int64(math.MinInt64)
	if len(series.Points) > 0 && *series.Points[0].Timestamp == lastTime {
		// just a hack to make sure lastTime will never equal the first
		// point's timestamp
		lastTime = 0
	}

	// sort the points by timestamp
	series.SortPointsTimeDescending()

	for i, point := range series.Points {
		if *point.Timestamp != lastTime {
			shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *point.Timestamp)
			if err != nil {
				return err
			}
			if shardToWrite == nil {
				shardToWrite = shard
			} else if shardToWrite.Id() != shard.Id() {
				newIndex := i
				newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[lastPointIndex:newIndex]}
				if err := self.write(db, newSeries, shardToWrite); err != nil {
					return err
				}
				lastPointIndex = newIndex
				shardToWrite = shard
			}
			lastTime = *point.Timestamp
		}
	}

	series.Points = series.Points[lastPointIndex:]

	if len(series.Points) > 0 {
		if shardToWrite == nil {
			shardToWrite, _ = self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *series.Points[0].Timestamp)
		}

		err := self.write(db, series, shardToWrite)

		if err != nil {
			log.Error("COORD error writing: ", err)
			return err
		}

		return err
	}

	return nil
}
Beispiel #2
0
// TODO: refactor this for clarity. This got super ugly...
// Function yields all results that are safe to do so ensuring order. Returns all results that must wait for more from the servers.
func (self *CoordinatorImpl) yieldResultsForSeries(isAscending bool, leftover *protocol.Series, responses []*protocol.Response, yield func(*protocol.Series) error) *protocol.Series {
	// results can come from different servers. Some of which won't know about fields that other servers may know about.
	// We need to normalize all this so that all fields are represented and the other field values are null.
	// Give each unique field name an index. We'll use this map later to construct the results and make sure that
	// the response objects have their fields in the result.
	fieldIndexes := make(map[string]int)
	for _, response := range responses {
		for _, name := range response.Series.Fields {
			if _, hasField := fieldIndexes[name]; !hasField {
				fieldIndexes[name] = len(fieldIndexes)
			}
		}
	}
	fields := make([]string, len(fieldIndexes), len(fieldIndexes))
	for name, index := range fieldIndexes {
		fields[index] = name
	}
	fieldCount := len(fields)

	result := &protocol.Series{Name: responses[0].Series.Name, Fields: fields, Points: make([]*protocol.Point, 0)}
	if leftover == nil {
		leftover = &protocol.Series{Name: responses[0].Series.Name, Fields: fields, Points: make([]*protocol.Point, 0)}
	}

	barrierTime := BARRIER_TIME_MIN
	if isAscending {
		barrierTime = BARRIER_TIME_MAX
	}
	var shouldYieldComparator func(rawTime *int64) bool
	if isAscending {
		shouldYieldComparator = func(rawTime *int64) bool {
			if rawTime != nil && *rawTime < barrierTime {
				return true
			} else {
				return false
			}
		}
	} else {
		shouldYieldComparator = func(rawTime *int64) bool {
			if rawTime != nil && *rawTime > barrierTime {
				return true
			} else {
				return false
			}
		}
	}
	// find the barrier time
	for _, response := range responses {
		if shouldYieldComparator(response.NextPointTime) {
			barrierTime = *response.NextPointTime
		}
	}
	// yield the points from leftover that are safe
	for _, point := range leftover.Points {
		if shouldYieldComparator(point.Timestamp) {
			result.Points = append(result.Points, point)
		} else {
			break
		}
	}
	// if they all got added, clear out the leftover
	if len(leftover.Points) == len(result.Points) {
		leftover.Points = make([]*protocol.Point, 0)
	}

	if barrierTime == BARRIER_TIME_MIN || barrierTime == BARRIER_TIME_MAX {
		// all the nextPointTimes were nil so we're safe to send everything
		for _, response := range responses {
			// if this is the case we know that all responses contained the same
			// fields. So just append the points
			if len(response.Series.Fields) == fieldCount {
				result.Points = append(result.Points, response.Series.Points...)
			} else {
				log.Debug("Responses from servers had different numbers of fields.")
				for _, p := range response.Series.Points {
					self.normalizePointAndAppend(fieldIndexes, result, response.Series.Fields, p)
				}
			}
		}
		if len(leftover.Fields) == fieldCount {
			result.Points = append(result.Points, leftover.Points...)
			leftover.Points = []*protocol.Point{}
		} else {
			log.Debug("Responses from servers had different numbers of fields.")
			for _, p := range leftover.Points {
				self.normalizePointAndAppend(fieldIndexes, result, leftover.Fields, p)
			}
		}
	} else {
		for _, response := range responses {
			if shouldYieldComparator(response.NextPointTime) {
				// all points safe to yield
				if fieldCount == len(response.Series.Fields) {
					result.Points = append(result.Points, response.Series.Points...)
				} else {
					log.Debug("Responses from servers had different numbers of fields.")
					for _, p := range response.Series.Points {
						self.normalizePointAndAppend(fieldIndexes, result, response.Series.Fields, p)
					}
				}
				continue
			}

			if fieldCount == len(response.Series.Fields) {
				for i, point := range response.Series.Points {
					if shouldYieldComparator(point.Timestamp) {
						result.Points = append(result.Points, point)
					} else {
						// since they're returned in order, we can just append these to
						// the leftover and break out.
						leftover.Points = append(leftover.Points, response.Series.Points[i:]...)
						break
					}
				}
			} else {
				for i, point := range response.Series.Points {
					if shouldYieldComparator(point.Timestamp) {
						self.normalizePointAndAppend(fieldIndexes, result, response.Series.Fields, point)
					} else {
						// since they're returned in order, we can just append these to
						// the leftover and break out.
						for _, point := range response.Series.Points[i:] {
							self.normalizePointAndAppend(fieldIndexes, leftover, response.Series.Fields, point)
						}
						break
					}
				}
			}
		}
	}

	if isAscending {
		result.SortPointsTimeAscending()
		leftover.SortPointsTimeAscending()
	} else {
		result.SortPointsTimeDescending()
		leftover.SortPointsTimeDescending()
	}

	// Don't yield an empty points array, the engine will think it's the end of the stream.
	// streamResultsFromChannels will send the empty ones after all channels have returned.
	if len(result.Points) > 0 {
		yield(result)
	}
	if len(leftover.Points) > 0 {
		return leftover
	}
	return nil
}