func (bt *bucketTraverser) apply(f func(fv []*protocol.FieldValue, node *Node) error) error {
	for !bt.reachedEnd() {
		timestamp := &protocol.FieldValue{Int64Value: protocol.Int64(bt.current)}
		defaultChildNode := &Node{states: make([]interface{}, bt.aggregators)}
		err := bt.trie.TraverseLevel(bt.levels, func(v []*protocol.FieldValue, node *Node) error {
			childNode := node.GetChildNode(timestamp)
			if childNode == nil {
				childNode = defaultChildNode
			}
			return f(append(v, timestamp), childNode)
		})
		if err != nil {
			return err
		}

		bt.current += bt.step
	}
	return nil
}
// We have three types of queries:
//   1. time() without fill
//   2. time() with fill
//   3. no time()
//
// For (1) we flush as soon as a new bucket start, the prefix tree
// keeps track of the other group by columns without the time
// bucket. We reset the trie once the series is yielded. For (2), we
// keep track of all group by columns with time being the last level
// in the prefix tree. At the end of the query we step through [start
// time, end time] in self.duration steps and get the state from the
// prefix tree, using default values for groups without state in the
// prefix tree. For the last case we keep the groups in the prefix
// tree and on close() we loop through the groups and flush their
// values with a timestamp equal to now()
func (self *AggregatorEngine) aggregateValuesForSeries(series *protocol.Series) (bool, error) {
	for _, aggregator := range self.aggregators {
		if err := aggregator.InitializeFieldsMetadata(series); err != nil {
			return false, err
		}
	}

	seriesState := self.getSeriesState(series.GetName())
	currentRange := seriesState.pointsRange

	includeTimestampInGroup := self.duration != nil && self.isFillQuery
	var group []*protocol.FieldValue
	if !includeTimestampInGroup {
		group = make([]*protocol.FieldValue, len(self.elems))
	} else {
		group = make([]*protocol.FieldValue, len(self.elems)+1)
	}

	for _, point := range series.Points {
		currentRange.UpdateRange(point)

		// this is a groupby with time() and no fill, flush as soon as we
		// start a new bucket
		if self.duration != nil && !self.isFillQuery {
			timestamp := self.getTimestampFromPoint(point)
			// this is the timestamp aggregator
			if seriesState.started && seriesState.lastTimestamp != timestamp {
				self.runAggregatesForTable(series.GetName())
			}
			seriesState.lastTimestamp = timestamp
			seriesState.started = true
		}

		// get the group this point belongs to
		for idx, elem := range self.elems {
			// TODO: create an index from fieldname to index

			// TODO: We shouldn't rely on GetValue() to do arithmetic
			// operations. Instead we should cascade the arithmetic engine
			// with the aggregator engine and possibly add another
			// arithmetic engine to be able to do arithmetics on the
			// resulting aggregated data.
			value, err := GetValue(elem, series.Fields, point)
			if err != nil {
				return false, err
			}
			group[idx] = value
		}

		// if this is a fill() query, add the timestamp at the end
		if includeTimestampInGroup {
			timestamp := self.getTimestampFromPoint(point)
			group[len(self.elems)] = &protocol.FieldValue{Int64Value: protocol.Int64(timestamp)}
		}

		// update the state of the given group
		node := seriesState.trie.GetNode(group)
		var err error
		log4go.Debug("Aggregating for group %v", group)
		for idx, aggregator := range self.aggregators {
			log4go.Debug("Aggregating value for %T for group %v and state %v", aggregator, group, node.states[idx])
			node.states[idx], err = aggregator.AggregatePoint(node.states[idx], point)
			if err != nil {
				return false, err
			}
		}
	}

	return true, nil
}
Exemple #3
0
func (self *QueryEngine) runAggregatesForTable(table string) {
	// TODO: if this is a fill query, step through [start,end] in duration
	// steps and flush the groups for the given bucket

	self.calculateSummariesForTable(table)

	state := self.getSeriesState(table)
	trie := state.trie
	points := make([]*protocol.Point, 0, trie.CountLeafNodes())
	f := func(group []*protocol.FieldValue, node *Node) error {
		points = append(points, self.getValuesForGroup(table, group, node)...)
		return nil
	}

	var err error
	if self.duration != nil && self.fillWithZero {
		timestampRange := state.pointsRange

		// TODO: DRY this
		if self.query.Ascending {
			bucket := self.getTimestampBucket(uint64(timestampRange.startTime))
			for bucket <= timestampRange.endTime {
				timestamp := &protocol.FieldValue{Int64Value: protocol.Int64(bucket)}
				defaultChildNode := &Node{states: make([]interface{}, len(self.aggregators))}
				err = trie.TraverseLevel(len(self.elems), func(v []*protocol.FieldValue, node *Node) error {
					childNode := node.GetChildNode(timestamp)
					if childNode == nil {
						childNode = defaultChildNode
					}
					return f(append(v, timestamp), childNode)
				})
				bucket += self.duration.Nanoseconds() / 1000
			}
		} else {
			bucket := self.getTimestampBucket(uint64(timestampRange.endTime))
			for {
				timestamp := &protocol.FieldValue{Int64Value: protocol.Int64(bucket)}
				defaultChildNode := &Node{states: make([]interface{}, len(self.aggregators))}
				err = trie.TraverseLevel(len(self.elems), func(v []*protocol.FieldValue, node *Node) error {
					childNode := node.GetChildNode(timestamp)
					if childNode == nil {
						childNode = defaultChildNode
					}
					return f(append(v, timestamp), childNode)
				})
				if bucket <= timestampRange.startTime {
					break
				}
				bucket -= self.duration.Nanoseconds() / 1000
			}
		}
	} else {
		err = trie.Traverse(f)
	}
	if err != nil {
		panic(err)
	}
	trie.Clear()
	self.aggregateYield(&protocol.Series{
		Name:   &table,
		Fields: self.fields,
		Points: points,
	})
}
Exemple #4
0
// We have three types of queries:
//   1. time() without fill
//   2. time() with fill
//   3. no time()
//
// For (1) we flush as soon as a new bucket start, the prefix tree
// keeps track of the other group by columns without the time
// bucket. We reset the trie once the series is yielded. For (2), we
// keep track of all group by columns with time being the last level
// in the prefix tree. At the end of the query we step through [start
// time, end time] in self.duration steps and get the state from the
// prefix tree, using default values for groups without state in the
// prefix tree. For the last case we keep the groups in the prefix
// tree and on close() we loop through the groups and flush their
// values with a timestamp equal to now()
func (self *QueryEngine) aggregateValuesForSeries(series *protocol.Series) error {
	for _, aggregator := range self.aggregators {
		if err := aggregator.InitializeFieldsMetadata(series); err != nil {
			return err
		}
	}

	seriesState := self.getSeriesState(series.GetName())
	currentRange := seriesState.pointsRange

	includeTimestampInGroup := self.duration != nil && self.fillWithZero
	var group []*protocol.FieldValue
	if !includeTimestampInGroup {
		group = make([]*protocol.FieldValue, len(self.elems))
	} else {
		group = make([]*protocol.FieldValue, len(self.elems)+1)
	}

	for _, point := range series.Points {
		currentRange.UpdateRange(point)

		// this is a groupby with time() and no fill, flush as soon as we
		// start a new bucket
		if self.duration != nil && !self.fillWithZero {
			timestamp := self.getTimestampFromPoint(point)
			// this is the timestamp aggregator
			if seriesState.started && seriesState.lastTimestamp != timestamp {
				self.runAggregatesForTable(series.GetName())
			}
			seriesState.lastTimestamp = timestamp
			seriesState.started = true
		}

		// get the group this point belongs to
		for idx, elem := range self.elems {
			// TODO: create an index from fieldname to index
			value, err := GetValue(elem, series.Fields, point)
			if err != nil {
				return err
			}
			group[idx] = value
		}

		// if this is a fill() query, add the timestamp at the end
		if includeTimestampInGroup {
			timestamp := self.getTimestampFromPoint(point)
			group[len(self.elems)] = &protocol.FieldValue{Int64Value: protocol.Int64(timestamp)}
		}

		// update the state of the given group
		node := seriesState.trie.GetNode(group)
		var err error
		for idx, aggregator := range self.aggregators {
			node.states[idx], err = aggregator.AggregatePoint(node.states[idx], point)
			if err != nil {
				return err
			}
		}
	}

	return nil
}
Exemple #5
0
func (self *TrieTestSuite) TestTrie(c *C) {
	trie := NewTrie(2, 1)

	firstValue := []*protocol.FieldValue{
		{StringValue: protocol.String("some_value")},
		{Int64Value: protocol.Int64(1)},
	}
	firstNode := trie.GetNode(firstValue)
	c.Assert(firstNode, NotNil)
	c.Assert(trie.GetNode(firstValue), DeepEquals, firstNode)
	c.Assert(trie.CountLeafNodes(), Equals, 1)

	secondValue := []*protocol.FieldValue{
		{StringValue: protocol.String("some_value")},
		{Int64Value: protocol.Int64(2)},
	}
	secondNode := trie.GetNode(secondValue)
	c.Assert(secondNode, NotNil)
	c.Assert(trie.GetNode(secondValue), DeepEquals, secondNode)
	c.Assert(trie.CountLeafNodes(), Equals, 2)

	thirdValue := []*protocol.FieldValue{
		{StringValue: protocol.String("another_value")},
		{Int64Value: protocol.Int64(1)},
	}
	thirdNode := trie.GetNode(thirdValue)
	c.Assert(thirdNode, NotNil)
	c.Assert(trie.GetNode(thirdValue), DeepEquals, thirdNode)
	c.Assert(trie.CountLeafNodes(), Equals, 3)

	nodes := 0
	orderValues := [][]*protocol.FieldValue{thirdValue, firstValue, secondValue}
	c.Assert(trie.Traverse(func(v []*protocol.FieldValue, _ *Node) error {
		c.Assert(v, DeepEquals, orderValues[nodes])
		nodes++
		return nil
	}), IsNil)
	c.Assert(nodes, Equals, trie.CountLeafNodes())

	// make sure TraverseLevel work as expected
	ns := []*Node{}
	c.Assert(trie.TraverseLevel(0, func(_ []*protocol.FieldValue, n *Node) error {
		ns = append(ns, n)
		return nil
	}), IsNil)
	c.Assert(ns, HasLen, 1) // should return the root node only
	c.Assert(ns[0].value, IsNil)

	ns = []*Node{}
	c.Assert(trie.TraverseLevel(1, func(_ []*protocol.FieldValue, n *Node) error {
		ns = append(ns, n)
		return nil
	}), IsNil)
	c.Assert(ns, HasLen, 2) // should return the root node only
	c.Assert(ns[0].value.GetStringValue(), Equals, "another_value")
	c.Assert(ns[1].value.GetStringValue(), Equals, "some_value")

	c.Assert(ns[0].GetChildNode(&protocol.FieldValue{Int64Value: protocol.Int64(1)}).isLeaf, Equals, true)
	c.Assert(ns[0].GetChildNode(&protocol.FieldValue{Int64Value: protocol.Int64(2)}), IsNil)

	c.Assert(ns[1].GetChildNode(&protocol.FieldValue{Int64Value: protocol.Int64(1)}).isLeaf, Equals, true)
	c.Assert(ns[1].GetChildNode(&protocol.FieldValue{Int64Value: protocol.Int64(2)}).isLeaf, Equals, true)
}