Beispiel #1
0
func setGroupOnPoint(p models.Point, allDimensions bool, dimensions []string) models.Point {
	if allDimensions {
		dimensions = models.SortedKeys(p.Tags)
	}
	p.Group = models.TagsToGroupID(dimensions, p.Tags)
	p.Dimensions = dimensions
	return p
}
Beispiel #2
0
// Replay the batch data from a single source
func (r *replayBatchSource) replayBatchFromData(data io.ReadCloser, batch BatchCollector, recTime bool) {
	defer batch.Close()
	defer data.Close()

	in := bufio.NewScanner(data)

	// Find relative times
	start := time.Time{}
	var diff time.Duration
	zero := r.clck.Zero()

	for in.Scan() {
		var b models.Batch
		err := json.Unmarshal(in.Bytes(), &b)
		if err != nil {
			r.allErrs <- err
			return
		}
		if len(b.Points) == 0 {
			// do nothing
			continue
		}
		b.Group = models.TagsToGroupID(models.SortedKeys(b.Tags), b.Tags)

		if start.IsZero() {
			start = b.Points[0].Time
			diff = zero.Sub(start)
		}
		// Add tags to all points
		if len(b.Tags) > 0 {
			for i := range b.Points {
				if len(b.Points[i].Tags) == 0 {
					b.Points[i].Tags = b.Tags
				}
			}
		}
		var lastTime time.Time
		if !recTime {
			for i := range b.Points {
				b.Points[i].Time = b.Points[i].Time.Add(diff).UTC()
			}
			lastTime = b.Points[len(b.Points)-1].Time
		} else {
			lastTime = b.Points[len(b.Points)-1].Time.Add(diff).UTC()
		}
		r.clck.Until(lastTime)
		b.TMax = b.Points[len(b.Points)-1].Time
		batch.CollectBatch(b)
	}
	r.allErrs <- in.Err()
}
Beispiel #3
0
func (g *GroupByNode) runGroupBy([]byte) error {
	switch g.Wants() {
	case pipeline.StreamEdge:
		for pt, ok := g.ins[0].NextPoint(); ok; pt, ok = g.ins[0].NextPoint() {
			g.timer.Start()
			pt = setGroupOnPoint(pt, g.allDimensions, g.dimensions)
			g.timer.Stop()
			for _, child := range g.outs {
				err := child.CollectPoint(pt)
				if err != nil {
					return err
				}
			}
		}
	default:
		var lastTime time.Time
		groups := make(map[models.GroupID]*models.Batch)
		for b, ok := g.ins[0].NextBatch(); ok; b, ok = g.ins[0].NextBatch() {
			g.timer.Start()
			if !b.TMax.Equal(lastTime) {
				lastTime = b.TMax
				// Emit all groups
				for id, group := range groups {
					for _, child := range g.outs {
						err := child.CollectBatch(*group)
						if err != nil {
							return err
						}
					}
					// Remove from groups
					delete(groups, id)
				}
			}
			for _, p := range b.Points {
				var dims []string
				if g.allDimensions {
					dims = models.SortedKeys(p.Tags)
				} else {
					dims = g.dimensions
				}
				groupID := models.TagsToGroupID(dims, p.Tags)
				group, ok := groups[groupID]
				if !ok {
					tags := make(map[string]string, len(dims))
					for _, dim := range dims {
						tags[dim] = p.Tags[dim]
					}
					group = &models.Batch{
						Name:  b.Name,
						Group: groupID,
						TMax:  b.TMax,
						Tags:  tags,
					}
					groups[groupID] = group
				}
				group.Points = append(group.Points, p)
			}
			g.timer.Stop()
		}
	}
	return nil
}
Beispiel #4
0
// The purpose of this method is to match more specific points
// with the less specific points as they arrive.
//
// Where 'more specific' means, that a point has more dimensions than the join.on dimensions.
func (j *JoinNode) matchPoints(p srcPoint) {
	j.mu.Lock()
	defer j.mu.Unlock()

	if !j.allReported {
		j.reported[p.src] = true
		j.allReported = len(j.reported) == len(j.ins)
	}
	t := p.p.PointTime().Round(j.j.Tolerance)
	if j.lowMark.IsZero() || t.Before(j.lowMark) {
		j.lowMark = t
	}

	groupId := models.TagsToGroupID(j.j.Dimensions, p.p.PointTags())
	if len(p.p.PointDimensions()) > len(j.j.Dimensions) {
		// We have a specific point, find its cached match and send both to group
		matches := j.matchGroupsBuffer[groupId]
		matched := false
		for _, match := range matches {
			if match.p.PointTime().Round(j.j.Tolerance).Equal(t) {
				j.sendMatchPoint(p, match)
				matched = true
			}
		}
		if !matched {
			// Cache this point for when its match arrives
			j.specificGroupsBuffer[groupId] = append(j.specificGroupsBuffer[groupId], p)
		}

		// Purge cached match points
		if j.allReported {
			for id, cached := range j.matchGroupsBuffer {
				var i int
				l := len(cached)
				for i = 0; i < l; i++ {
					if !cached[i].p.PointTime().Round(j.j.Tolerance).Before(j.lowMark) {
						break
					}
				}
				j.matchGroupsBuffer[id] = cached[i:]
			}
		}

	} else {

		// Cache match point.
		j.matchGroupsBuffer[groupId] = append(j.matchGroupsBuffer[groupId], p)

		// Send all specific points, that match, to the group.
		var i int
		buf := j.specificGroupsBuffer[groupId]
		l := len(buf)
		for i = 0; i < l; i++ {
			if buf[i].p.PointTime().Round(j.j.Tolerance).Equal(t) {
				j.sendMatchPoint(buf[i], p)
			} else {
				break
			}
		}
		// Remove all sent points
		j.specificGroupsBuffer[groupId] = buf[i:]
	}
}