func (self *CoordinatorImpl) CommitSeriesData(db string, series *protocol.Series) error { lastPointIndex := 0 now := common.CurrentTime() var shardToWrite cluster.Shard for _, point := range series.Points { if point.Timestamp == nil { point.Timestamp = &now } } lastTime := int64(math.MinInt64) if len(series.Points) > 0 && *series.Points[0].Timestamp == lastTime { // just a hack to make sure lastTime will never equal the first // point's timestamp lastTime = 0 } // sort the points by timestamp series.SortPointsTimeDescending() for i, point := range series.Points { if *point.Timestamp != lastTime { shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *point.Timestamp) if err != nil { return err } if shardToWrite == nil { shardToWrite = shard } else if shardToWrite.Id() != shard.Id() { newIndex := i newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[lastPointIndex:newIndex]} if err := self.write(db, newSeries, shardToWrite); err != nil { return err } lastPointIndex = newIndex shardToWrite = shard } lastTime = *point.Timestamp } } series.Points = series.Points[lastPointIndex:] if len(series.Points) > 0 { if shardToWrite == nil { shardToWrite, _ = self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *series.Points[0].Timestamp) } err := self.write(db, series, shardToWrite) if err != nil { log.Error("COORD error writing: ", err) return err } return err } return nil }
func (self *CoordinatorImpl) CommitSeriesData(db string, series *protocol.Series) error { lastTime := int64(0) lastPointIndex := 0 now := common.CurrentTime() var shardToWrite cluster.Shard for i, point := range series.Points { if point.Timestamp == nil { point.Timestamp = &now } if *point.Timestamp != lastTime { shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *point.Timestamp) if err != nil { return err } if shardToWrite == nil { shardToWrite = shard } else if shardToWrite.Id() != shard.Id() { newIndex := i + 1 newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[lastPointIndex:newIndex]} self.write(db, newSeries, shardToWrite) lastPointIndex = newIndex shardToWrite = shard } lastTime = *point.Timestamp } } series.Points = series.Points[lastPointIndex:] if len(series.Points) > 0 { if shardToWrite == nil { shardToWrite, _ = self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, *series.Name, *series.Points[0].Timestamp) } err := self.write(db, series, shardToWrite) if err != nil { log.Error("COORD error writing: ", err) } return err } return nil }
func (self *CoordinatorImpl) WriteSeriesData(user common.User, db string, series *protocol.Series) error { if !user.HasWriteAccess(db) { return fmt.Errorf("Insufficient permission to write to %s", db) } now := common.CurrentTime() for _, p := range series.Points { if p.Timestamp == nil { p.Timestamp = &now self.sequenceNumberLock.Lock() self.currentSequenceNumber += 1 n := self.currentSequenceNumber self.sequenceNumberLock.Unlock() p.SequenceNumber = &n } else if p.SequenceNumber == nil { self.sequenceNumberLock.Lock() self.currentSequenceNumber += 1 n := self.currentSequenceNumber self.sequenceNumberLock.Unlock() p.SequenceNumber = &n } } return self.datastore.WriteSeriesData(db, series) }
func (self *CoordinatorImpl) CommitSeriesData(db string, serieses []*protocol.Series, sync bool) error { now := common.CurrentTime() shardToSerieses := map[uint32]map[string]*protocol.Series{} shardIdToShard := map[uint32]*cluster.ShardData{} for _, series := range serieses { if len(series.Points) == 0 { return fmt.Errorf("Can't write series with zero points.") } for _, point := range series.Points { if point.Timestamp == nil { point.Timestamp = &now } } // sort the points by timestamp // TODO: this isn't needed anymore series.SortPointsTimeDescending() for i := 0; i < len(series.Points); { shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, series.GetName(), series.Points[i].GetTimestamp()) if err != nil { return err } firstIndex := i timestamp := series.Points[i].GetTimestamp() for ; i < len(series.Points) && series.Points[i].GetTimestamp() == timestamp; i++ { // add all points with the same timestamp } newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[firstIndex:i:i]} shardIdToShard[shard.Id()] = shard shardSerieses := shardToSerieses[shard.Id()] if shardSerieses == nil { shardSerieses = map[string]*protocol.Series{} shardToSerieses[shard.Id()] = shardSerieses } seriesName := series.GetName() s := shardSerieses[seriesName] if s == nil { shardSerieses[seriesName] = newSeries continue } s.Points = append(s.Points, newSeries.Points...) } } for id, serieses := range shardToSerieses { shard := shardIdToShard[id] seriesesSlice := make([]*protocol.Series, 0, len(serieses)) for _, s := range serieses { seriesesSlice = append(seriesesSlice, s) } err := self.write(db, seriesesSlice, shard, sync) if err != nil { log.Error("COORD error writing: ", err) return err } } return nil }
func (self *CoordinatorImpl) CommitSeriesData(db string, series *protocol.Series) error { // break the series object into separate ones based on their ring location // if times server assigned, all the points will go to the same place serverAssignedTime := true now := common.CurrentTime() // assign sequence numbers lastNumber, err := self.datastore.AtomicIncrement(POINT_SEQUENCE_NUMBER_KEY, len(series.Points)) if err != nil { return err } lastNumber = lastNumber - uint64(len(series.Points)-1) for _, p := range series.Points { if p.Timestamp == nil { p.Timestamp = &now } else { serverAssignedTime = false } if p.SequenceNumber == nil { n := self.sequenceNumberWithServerId(lastNumber) lastNumber++ p.SequenceNumber = &n } } // if it's a single server setup, we don't need to bother with getting ring // locations or logging requests or any of that, so just write to the local db and be done. if self.clusterConfiguration.IsSingleServer() { err := self.writeSeriesToLocalStore(&db, series) return err } if serverAssignedTime { location := common.RingLocation(&db, series.Name, series.Points[0].Timestamp) i := self.clusterConfiguration.GetServerIndexByLocation(&location) return self.handleClusterWrite(&i, &db, series) } // TODO: make this more efficient and not suck so much // not all the same, so break things up seriesToServerIndex := make(map[int]*protocol.Series) for _, p := range series.Points { location := common.RingLocation(&db, series.Name, p.Timestamp) i := self.clusterConfiguration.GetServerIndexByLocation(&location) s := seriesToServerIndex[i] if s == nil { s = &protocol.Series{Name: series.Name, Fields: series.Fields, Points: make([]*protocol.Point, 0)} seriesToServerIndex[i] = s } s.Points = append(s.Points, p) } for serverIndex, s := range seriesToServerIndex { err := self.handleClusterWrite(&serverIndex, &db, s) if err != nil { return err } } return nil }