func setGroupOnPoint(p models.Point, allDimensions bool, dimensions []string) models.Point { if allDimensions { dimensions = models.SortedKeys(p.Tags) } p.Group = models.TagsToGroupID(dimensions, p.Tags) p.Dimensions = dimensions return p }
// Replay the batch data from a single source func (r *replayBatchSource) replayBatchFromData(data io.ReadCloser, batch BatchCollector, recTime bool) { defer batch.Close() defer data.Close() in := bufio.NewScanner(data) // Find relative times start := time.Time{} var diff time.Duration zero := r.clck.Zero() for in.Scan() { var b models.Batch err := json.Unmarshal(in.Bytes(), &b) if err != nil { r.allErrs <- err return } if len(b.Points) == 0 { // do nothing continue } b.Group = models.TagsToGroupID(models.SortedKeys(b.Tags), b.Tags) if start.IsZero() { start = b.Points[0].Time diff = zero.Sub(start) } // Add tags to all points if len(b.Tags) > 0 { for i := range b.Points { if len(b.Points[i].Tags) == 0 { b.Points[i].Tags = b.Tags } } } var lastTime time.Time if !recTime { for i := range b.Points { b.Points[i].Time = b.Points[i].Time.Add(diff).UTC() } lastTime = b.Points[len(b.Points)-1].Time } else { lastTime = b.Points[len(b.Points)-1].Time.Add(diff).UTC() } r.clck.Until(lastTime) b.TMax = b.Points[len(b.Points)-1].Time batch.CollectBatch(b) } r.allErrs <- in.Err() }
func (g *GroupByNode) runGroupBy([]byte) error { switch g.Wants() { case pipeline.StreamEdge: for pt, ok := g.ins[0].NextPoint(); ok; pt, ok = g.ins[0].NextPoint() { g.timer.Start() pt = setGroupOnPoint(pt, g.allDimensions, g.dimensions) g.timer.Stop() for _, child := range g.outs { err := child.CollectPoint(pt) if err != nil { return err } } } default: var lastTime time.Time groups := make(map[models.GroupID]*models.Batch) for b, ok := g.ins[0].NextBatch(); ok; b, ok = g.ins[0].NextBatch() { g.timer.Start() if !b.TMax.Equal(lastTime) { lastTime = b.TMax // Emit all groups for id, group := range groups { for _, child := range g.outs { err := child.CollectBatch(*group) if err != nil { return err } } // Remove from groups delete(groups, id) } } for _, p := range b.Points { var dims []string if g.allDimensions { dims = models.SortedKeys(p.Tags) } else { dims = g.dimensions } groupID := models.TagsToGroupID(dims, p.Tags) group, ok := groups[groupID] if !ok { tags := make(map[string]string, len(dims)) for _, dim := range dims { tags[dim] = p.Tags[dim] } group = &models.Batch{ Name: b.Name, Group: groupID, TMax: b.TMax, Tags: tags, } groups[groupID] = group } group.Points = append(group.Points, p) } g.timer.Stop() } } return nil }
// The purpose of this method is to match more specific points // with the less specific points as they arrive. // // Where 'more specific' means, that a point has more dimensions than the join.on dimensions. func (j *JoinNode) matchPoints(p srcPoint) { j.mu.Lock() defer j.mu.Unlock() if !j.allReported { j.reported[p.src] = true j.allReported = len(j.reported) == len(j.ins) } t := p.p.PointTime().Round(j.j.Tolerance) if j.lowMark.IsZero() || t.Before(j.lowMark) { j.lowMark = t } groupId := models.TagsToGroupID(j.j.Dimensions, p.p.PointTags()) if len(p.p.PointDimensions()) > len(j.j.Dimensions) { // We have a specific point, find its cached match and send both to group matches := j.matchGroupsBuffer[groupId] matched := false for _, match := range matches { if match.p.PointTime().Round(j.j.Tolerance).Equal(t) { j.sendMatchPoint(p, match) matched = true } } if !matched { // Cache this point for when its match arrives j.specificGroupsBuffer[groupId] = append(j.specificGroupsBuffer[groupId], p) } // Purge cached match points if j.allReported { for id, cached := range j.matchGroupsBuffer { var i int l := len(cached) for i = 0; i < l; i++ { if !cached[i].p.PointTime().Round(j.j.Tolerance).Before(j.lowMark) { break } } j.matchGroupsBuffer[id] = cached[i:] } } } else { // Cache match point. j.matchGroupsBuffer[groupId] = append(j.matchGroupsBuffer[groupId], p) // Send all specific points, that match, to the group. var i int buf := j.specificGroupsBuffer[groupId] l := len(buf) for i = 0; i < l; i++ { if buf[i].p.PointTime().Round(j.j.Tolerance).Equal(t) { j.sendMatchPoint(buf[i], p) } else { break } } // Remove all sent points j.specificGroupsBuffer[groupId] = buf[i:] } }