Example #1
0
func (self *CoordinatorImpl) CommitSeriesData(db string, series *protocol.Series) error {
	lastPointIndex := 0
	now := common.CurrentTime()
	var shardToWrite cluster.Shard
	for _, point := range series.Points {
		if point.Timestamp == nil {
			point.Timestamp = &now
		}
	}

	lastTime := int64(math.MinInt64)
	if len(series.Points) > 0 && *series.Points[0].Timestamp == lastTime {
		// just a hack to make sure lastTime will never equal the first
		// point's timestamp
		lastTime = 0
	}

	// sort the points by timestamp
	series.SortPointsTimeDescending()

	for i, point := range series.Points {
		if *point.Timestamp != lastTime {
			shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *point.Timestamp)
			if err != nil {
				return err
			}
			if shardToWrite == nil {
				shardToWrite = shard
			} else if shardToWrite.Id() != shard.Id() {
				newIndex := i
				newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[lastPointIndex:newIndex]}
				if err := self.write(db, newSeries, shardToWrite); err != nil {
					return err
				}
				lastPointIndex = newIndex
				shardToWrite = shard
			}
			lastTime = *point.Timestamp
		}
	}

	series.Points = series.Points[lastPointIndex:]

	if len(series.Points) > 0 {
		if shardToWrite == nil {
			shardToWrite, _ = self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *series.Points[0].Timestamp)
		}

		err := self.write(db, series, shardToWrite)

		if err != nil {
			log.Error("COORD error writing: ", err)
			return err
		}

		return err
	}

	return nil
}
Example #2
0
func (self *CoordinatorImpl) CommitSeriesData(db string, series *protocol.Series) error {
	lastTime := int64(0)
	lastPointIndex := 0
	now := common.CurrentTime()
	var shardToWrite cluster.Shard
	for i, point := range series.Points {
		if point.Timestamp == nil {
			point.Timestamp = &now
		}
		if *point.Timestamp != lastTime {
			shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *point.Timestamp)
			if err != nil {
				return err
			}
			if shardToWrite == nil {
				shardToWrite = shard
			} else if shardToWrite.Id() != shard.Id() {
				newIndex := i + 1
				newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[lastPointIndex:newIndex]}
				self.write(db, newSeries, shardToWrite)
				lastPointIndex = newIndex
				shardToWrite = shard
			}
			lastTime = *point.Timestamp
		}
	}

	series.Points = series.Points[lastPointIndex:]

	if len(series.Points) > 0 {
		if shardToWrite == nil {
			shardToWrite, _ = self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *series.Points[0].Timestamp)
		}

		err := self.write(db, series, shardToWrite)

		if err != nil {
			log.Error("COORD error writing: ", err)
		}

		return err
	}

	return nil
}
Example #3
0
func (self *CoordinatorImpl) WriteSeriesData(user common.User, db string, series *protocol.Series) error {
	if !user.HasWriteAccess(db) {
		return fmt.Errorf("Insufficient permission to write to %s", db)
	}

	now := common.CurrentTime()
	for _, p := range series.Points {
		if p.Timestamp == nil {
			p.Timestamp = &now
			self.sequenceNumberLock.Lock()
			self.currentSequenceNumber += 1
			n := self.currentSequenceNumber
			self.sequenceNumberLock.Unlock()
			p.SequenceNumber = &n
		} else if p.SequenceNumber == nil {
			self.sequenceNumberLock.Lock()
			self.currentSequenceNumber += 1
			n := self.currentSequenceNumber
			self.sequenceNumberLock.Unlock()
			p.SequenceNumber = &n
		}
	}
	return self.datastore.WriteSeriesData(db, series)
}
Example #4
0
func (self *CoordinatorImpl) CommitSeriesData(db string, serieses []*protocol.Series, sync bool) error {
	now := common.CurrentTime()

	shardToSerieses := map[uint32]map[string]*protocol.Series{}
	shardIdToShard := map[uint32]*cluster.ShardData{}

	for _, series := range serieses {
		if len(series.Points) == 0 {
			return fmt.Errorf("Can't write series with zero points.")
		}

		for _, point := range series.Points {
			if point.Timestamp == nil {
				point.Timestamp = &now
			}
		}

		// sort the points by timestamp
		// TODO: this isn't needed anymore
		series.SortPointsTimeDescending()

		for i := 0; i < len(series.Points); {
			shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, series.GetName(), series.Points[i].GetTimestamp())
			if err != nil {
				return err
			}
			firstIndex := i
			timestamp := series.Points[i].GetTimestamp()
			for ; i < len(series.Points) && series.Points[i].GetTimestamp() == timestamp; i++ {
				// add all points with the same timestamp
			}
			newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[firstIndex:i:i]}

			shardIdToShard[shard.Id()] = shard
			shardSerieses := shardToSerieses[shard.Id()]
			if shardSerieses == nil {
				shardSerieses = map[string]*protocol.Series{}
				shardToSerieses[shard.Id()] = shardSerieses
			}
			seriesName := series.GetName()
			s := shardSerieses[seriesName]
			if s == nil {
				shardSerieses[seriesName] = newSeries
				continue
			}
			s.Points = append(s.Points, newSeries.Points...)
		}
	}

	for id, serieses := range shardToSerieses {
		shard := shardIdToShard[id]

		seriesesSlice := make([]*protocol.Series, 0, len(serieses))
		for _, s := range serieses {
			seriesesSlice = append(seriesesSlice, s)
		}

		err := self.write(db, seriesesSlice, shard, sync)
		if err != nil {
			log.Error("COORD error writing: ", err)
			return err
		}
	}

	return nil
}
Example #5
0
func (self *CoordinatorImpl) CommitSeriesData(db string, series *protocol.Series) error {
	// break the series object into separate ones based on their ring location

	// if times server assigned, all the points will go to the same place
	serverAssignedTime := true
	now := common.CurrentTime()

	// assign sequence numbers
	lastNumber, err := self.datastore.AtomicIncrement(POINT_SEQUENCE_NUMBER_KEY, len(series.Points))
	if err != nil {
		return err
	}
	lastNumber = lastNumber - uint64(len(series.Points)-1)
	for _, p := range series.Points {
		if p.Timestamp == nil {
			p.Timestamp = &now
		} else {
			serverAssignedTime = false
		}
		if p.SequenceNumber == nil {
			n := self.sequenceNumberWithServerId(lastNumber)
			lastNumber++
			p.SequenceNumber = &n
		}
	}

	// if it's a single server setup, we don't need to bother with getting ring
	// locations or logging requests or any of that, so just write to the local db and be done.
	if self.clusterConfiguration.IsSingleServer() {
		err := self.writeSeriesToLocalStore(&db, series)
		return err
	}

	if serverAssignedTime {
		location := common.RingLocation(&db, series.Name, series.Points[0].Timestamp)
		i := self.clusterConfiguration.GetServerIndexByLocation(&location)
		return self.handleClusterWrite(&i, &db, series)
	}

	// TODO: make this more efficient and not suck so much
	// not all the same, so break things up

	seriesToServerIndex := make(map[int]*protocol.Series)
	for _, p := range series.Points {
		location := common.RingLocation(&db, series.Name, p.Timestamp)
		i := self.clusterConfiguration.GetServerIndexByLocation(&location)
		s := seriesToServerIndex[i]
		if s == nil {
			s = &protocol.Series{Name: series.Name, Fields: series.Fields, Points: make([]*protocol.Point, 0)}
			seriesToServerIndex[i] = s
		}
		s.Points = append(s.Points, p)
	}

	for serverIndex, s := range seriesToServerIndex {
		err := self.handleClusterWrite(&serverIndex, &db, s)
		if err != nil {
			return err
		}
	}

	return nil
}