func (e *Engine) CreateIterator(opt influxql.IteratorOptions) (influxql.Iterator, error) {
if call, ok := opt.Expr.(*influxql.Call); ok {
refOpt := opt
refOpt.Expr = call.Args[0].(*influxql.VarRef)
inputs, err := e.createVarRefIterator(refOpt)
if err != nil {
return nil, err
}
input := influxql.NewMergeIterator(inputs, opt)
if opt.InterruptCh != nil {
input = influxql.NewInterruptIterator(input, opt.InterruptCh)
}
return influxql.NewCallIterator(input, opt)
}
itrs, err := e.createVarRefIterator(opt)
if err != nil {
return nil, err
}
itr := influxql.NewSortedMergeIterator(itrs, opt)
if opt.InterruptCh != nil {
itr = influxql.NewInterruptIterator(itr, opt.InterruptCh)
}
return itr, nil
}
func TestSortedMergeIterator_Nil(t *testing.T) {
itr := influxql.NewSortedMergeIterator([]influxql.Iterator{nil}, influxql.IteratorOptions{}).(influxql.FloatIterator)
if p := itr.Next(); p != nil {
t.Fatalf("unexpected point: %#v", p)
}
itr.Close()
}
func (e *Engine) CreateIterator(opt influxql.IteratorOptions) (influxql.Iterator, error) {
if call, ok := opt.Expr.(*influxql.Call); ok {
refOpt := opt
refOpt.Expr = call.Args[0].(*influxql.VarRef)
aggregate := true
if opt.Interval.IsZero() {
switch call.Name {
case "first":
aggregate = false
refOpt.Limit = 1
refOpt.Ascending = true
case "last":
aggregate = false
refOpt.Limit = 1
refOpt.Ascending = false
}
}
inputs, err := e.createVarRefIterator(refOpt, aggregate)
if err != nil {
return nil, err
} else if len(inputs) == 0 {
return nil, nil
}
// Wrap each series in a call iterator.
for i, input := range inputs {
if opt.InterruptCh != nil {
input = influxql.NewInterruptIterator(input, opt.InterruptCh)
}
itr, err := influxql.NewCallIterator(input, opt)
if err != nil {
return nil, err
}
inputs[i] = itr
}
return influxql.NewParallelMergeIterator(inputs, opt, runtime.GOMAXPROCS(0)), nil
}
itrs, err := e.createVarRefIterator(opt, false)
if err != nil {
return nil, err
}
itr := influxql.NewSortedMergeIterator(itrs, opt)
if itr != nil && opt.InterruptCh != nil {
itr = influxql.NewInterruptIterator(itr, opt.InterruptCh)
}
return itr, nil
}
func TestSortedMergeIterator_Cast_Float(t *testing.T) {
inputs := []influxql.Iterator{
&IntegerIterator{Points: []influxql.IntegerPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 0, Value: 1},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 12, Value: 3},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 30, Value: 4},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 1, Value: 2},
{Name: "mem", Tags: ParseTags("host=B"), Time: 4, Value: 8},
}},
&FloatIterator{Points: []influxql.FloatPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 20, Value: 7},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 11, Value: 5},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 13, Value: 6},
{Name: "mem", Tags: ParseTags("host=A"), Time: 25, Value: 9},
}},
}
itr := influxql.NewSortedMergeIterator(inputs, influxql.IteratorOptions{
Interval: influxql.Interval{
Duration: 10 * time.Nanosecond,
},
Ascending: true,
})
if a, err := Iterators([]influxql.Iterator{itr}).ReadAll(); err != nil {
t.Fatalf("unexpected error: %s", err)
} else if !deep.Equal(a, [][]influxql.Point{
{&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0, Value: 1}},
{&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 12, Value: 3}},
{&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 20, Value: 7}},
{&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 30, Value: 4}},
{&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 1, Value: 2}},
{&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 11, Value: 5}},
{&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 13, Value: 6}},
{&influxql.FloatPoint{Name: "mem", Tags: ParseTags("host=A"), Time: 25, Value: 9}},
{&influxql.FloatPoint{Name: "mem", Tags: ParseTags("host=B"), Time: 4, Value: 8}},
}) {
t.Errorf("unexpected points: %s", spew.Sdump(a))
}
for i, input := range inputs {
switch input := input.(type) {
case *FloatIterator:
if !input.Closed {
t.Errorf("iterator %d not closed", i)
}
case *IntegerIterator:
if !input.Closed {
t.Errorf("iterator %d not closed", i)
}
}
}
}
// Ensure that a set of iterators can be merged together, sorted by name/tag.
func TestSortedMergeIterator_Boolean(t *testing.T) {
inputs := []*BooleanIterator{
{Points: []influxql.BooleanPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 0, Value: true},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 12, Value: true},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 30, Value: false},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 1, Value: false},
{Name: "mem", Tags: ParseTags("host=B"), Time: 4, Value: true},
}},
{Points: []influxql.BooleanPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 20, Value: true},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 11, Value: true},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 13, Value: false},
{Name: "mem", Tags: ParseTags("host=A"), Time: 25, Value: true},
}},
{Points: []influxql.BooleanPoint{}},
}
itr := influxql.NewSortedMergeIterator(BooleanIterators(inputs), influxql.IteratorOptions{
Interval: influxql.Interval{
Duration: 10 * time.Nanosecond,
},
Ascending: true,
})
if a, err := Iterators([]influxql.Iterator{itr}).ReadAll(); err != nil {
t.Fatalf("unexpected error: %s", err)
} else if !deep.Equal(a, [][]influxql.Point{
{&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0, Value: true}},
{&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 12, Value: true}},
{&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 20, Value: true}},
{&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 30, Value: false}},
{&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 1, Value: false}},
{&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 11, Value: true}},
{&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 13, Value: false}},
{&influxql.BooleanPoint{Name: "mem", Tags: ParseTags("host=A"), Time: 25, Value: true}},
{&influxql.BooleanPoint{Name: "mem", Tags: ParseTags("host=B"), Time: 4, Value: true}},
}) {
t.Errorf("unexpected points: %s", spew.Sdump(a))
}
for i, input := range inputs {
if !input.Closed {
t.Errorf("iterator %d not closed", i)
}
}
}
// createVarRefIterator creates an iterator for a variable reference.
func (e *Engine) createVarRefIterator(opt influxql.IteratorOptions) ([]influxql.Iterator, error) {
ref, _ := opt.Expr.(*influxql.VarRef)
var itrs []influxql.Iterator
if err := func() error {
mms := tsdb.Measurements(e.index.MeasurementsByName(influxql.Sources(opt.Sources).Names()))
for _, mm := range mms {
// Determine tagsets for this measurement based on dimensions and filters.
tagSets, err := mm.TagSets(opt.Dimensions, opt.Condition)
if err != nil {
return err
}
// Calculate tag sets and apply SLIMIT/SOFFSET.
tagSets = influxql.LimitTagSets(tagSets, opt.SLimit, opt.SOffset)
for _, t := range tagSets {
inputs, err := e.createTagSetIterators(ref, mm, t, opt)
if err != nil {
return err
}
if len(inputs) > 0 && (opt.Limit > 0 || opt.Offset > 0) {
var itr influxql.Iterator
if opt.MergeSorted() {
itr = influxql.NewSortedMergeIterator(inputs, opt)
} else {
itr = influxql.NewMergeIterator(inputs, opt)
}
itrs = append(itrs, newLimitIterator(itr, opt))
} else {
itrs = append(itrs, inputs...)
}
}
}
return nil
}(); err != nil {
influxql.Iterators(itrs).Close()
return nil, err
}
return itrs, nil
}
// Ensure that a set of iterators can be merged together, sorted by name/tag.
func TestSortedMergeIterator_String(t *testing.T) {
inputs := []*StringIterator{
{Points: []influxql.StringPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 0, Value: "a"},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 12, Value: "c"},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 30, Value: "d"},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 1, Value: "b"},
{Name: "mem", Tags: ParseTags("host=B"), Time: 4, Value: "h"},
}},
{Points: []influxql.StringPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 20, Value: "g"},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 11, Value: "e"},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 13, Value: "f"},
{Name: "mem", Tags: ParseTags("host=A"), Time: 25, Value: "i"},
}},
{Points: []influxql.StringPoint{}},
}
itr := influxql.NewSortedMergeIterator(StringIterators(inputs), influxql.IteratorOptions{
Interval: influxql.Interval{
Duration: 10 * time.Nanosecond,
},
Ascending: true,
})
if a := Iterators([]influxql.Iterator{itr}).ReadAll(); !deep.Equal(a, [][]influxql.Point{
{&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0, Value: "a"}},
{&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 12, Value: "c"}},
{&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 20, Value: "g"}},
{&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 30, Value: "d"}},
{&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 1, Value: "b"}},
{&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 11, Value: "e"}},
{&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 13, Value: "f"}},
{&influxql.StringPoint{Name: "mem", Tags: ParseTags("host=A"), Time: 25, Value: "i"}},
{&influxql.StringPoint{Name: "mem", Tags: ParseTags("host=B"), Time: 4, Value: "h"}},
}) {
t.Errorf("unexpected points: %s", spew.Sdump(a))
}
for i, input := range inputs {
if !input.Closed {
t.Errorf("iterator %d not closed", i)
}
}
}
// Ensure that a set of iterators can be merged together, sorted by name/tag.
func TestSortedMergeIterator_Boolean(t *testing.T) {
inputs := []*BooleanIterator{
{Points: []influxql.BooleanPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 0, Value: true},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 12, Value: true},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 30, Value: false},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 1, Value: false},
{Name: "mem", Tags: ParseTags("host=B"), Time: 4, Value: true},
}},
{Points: []influxql.BooleanPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 20, Value: true},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 11, Value: true},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 13, Value: false},
{Name: "mem", Tags: ParseTags("host=A"), Time: 25, Value: true},
}},
{Points: []influxql.BooleanPoint{}},
}
itr := influxql.NewSortedMergeIterator(BooleanIterators(inputs), influxql.IteratorOptions{
Interval: influxql.Interval{
Duration: 10 * time.Nanosecond,
},
Ascending: true,
})
if a, ok := CompareBooleanIterator(itr, []influxql.BooleanPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 0, Value: true},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 12, Value: true},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 20, Value: true},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 30, Value: false},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 1, Value: false},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 11, Value: true},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 13, Value: false},
{Name: "mem", Tags: ParseTags("host=A"), Time: 25, Value: true},
{Name: "mem", Tags: ParseTags("host=B"), Time: 4, Value: true},
}); !ok {
t.Errorf("unexpected points: %s", spew.Sdump(a))
}
for i, input := range inputs {
if !input.Closed {
t.Errorf("iterator %d not closed", i)
}
}
}
func (e *Engine) CreateIterator(opt influxql.IteratorOptions) (influxql.Iterator, error) {
if call, ok := opt.Expr.(*influxql.Call); ok {
refOpt := opt
refOpt.Expr = call.Args[0].(*influxql.VarRef)
inputs, err := e.createVarRefIterator(refOpt)
if err != nil {
return nil, err
} else if len(inputs) == 0 {
return nil, nil
}
// Wrap each series in a call iterator.
for i, input := range inputs {
if opt.InterruptCh != nil {
input = influxql.NewInterruptIterator(input, opt.InterruptCh)
}
itr, err := influxql.NewCallIterator(input, opt)
if err != nil {
return nil, err
}
inputs[i] = itr
}
return influxql.NewParallelMergeIterator(inputs, opt, runtime.GOMAXPROCS(0)), nil
}
itrs, err := e.createVarRefIterator(opt)
if err != nil {
return nil, err
}
itr := influxql.NewSortedMergeIterator(itrs, opt)
if itr != nil && opt.InterruptCh != nil {
itr = influxql.NewInterruptIterator(itr, opt.InterruptCh)
}
return itr, nil
}
// CreateIterator returns a single combined iterator for the shards.
func (a Shards) CreateIterator(opt influxql.IteratorOptions) (influxql.Iterator, error) {
if influxql.Sources(opt.Sources).HasSystemSource() {
return a.createSystemIterator(opt)
}
// Create iterators for each shard.
// Ensure that they are closed if an error occurs.
itrs := make([]influxql.Iterator, 0, len(a))
if err := func() error {
for _, sh := range a {
itr, err := sh.CreateIterator(opt)
if err != nil {
return err
}
itrs = append(itrs, itr)
}
return nil
}(); err != nil {
influxql.Iterators(itrs).Close()
return nil, err
}
// Merge into a single iterator.
if opt.MergeSorted() {
return influxql.NewSortedMergeIterator(itrs, opt), nil
}
itr := influxql.NewMergeIterator(itrs, opt)
if opt.Expr != nil {
if expr, ok := opt.Expr.(*influxql.Call); ok && expr.Name == "count" {
opt.Expr = &influxql.Call{
Name: "sum",
Args: expr.Args,
}
}
}
return influxql.NewCallIterator(itr, opt), nil
}
func TestSortedMergeIterator_Nil(t *testing.T) {
itr := influxql.NewSortedMergeIterator([]influxql.Iterator{nil}, influxql.IteratorOptions{})
if itr != nil {
t.Fatalf("unexpected iterator: %#v", itr)
}
}
// createVarRefIterator creates an iterator for a variable reference.
func (e *Engine) createVarRefIterator(opt influxql.IteratorOptions) ([]influxql.Iterator, error) {
ref, _ := opt.Expr.(*influxql.VarRef)
var itrs []influxql.Iterator
if err := func() error {
mms := tsdb.Measurements(e.index.MeasurementsByName(influxql.Sources(opt.Sources).Names()))
// Retrieve the maximum number of fields (without time).
conditionFields := make([]string, len(influxql.ExprNames(opt.Condition)))
for _, mm := range mms {
// Determine tagsets for this measurement based on dimensions and filters.
tagSets, err := mm.TagSets(opt.Dimensions, opt.Condition)
if err != nil {
return err
}
// Calculate tag sets and apply SLIMIT/SOFFSET.
tagSets = influxql.LimitTagSets(tagSets, opt.SLimit, opt.SOffset)
for _, t := range tagSets {
inputs := make([]influxql.Iterator, 0, len(t.SeriesKeys))
for i, seriesKey := range t.SeriesKeys {
fields := 0
if t.Filters[i] != nil {
// Retrieve non-time fields from this series filter and filter out tags.
for _, f := range influxql.ExprNames(t.Filters[i]) {
conditionFields[fields] = f
fields++
}
}
input, err := e.createVarRefSeriesIterator(ref, mm, seriesKey, t, t.Filters[i], conditionFields[:fields], opt)
if err != nil {
return err
} else if input == nil {
continue
}
inputs = append(inputs, input)
}
if len(inputs) > 0 && (opt.Limit > 0 || opt.Offset > 0) {
var itr influxql.Iterator
if opt.MergeSorted() {
itr = influxql.NewSortedMergeIterator(inputs, opt)
} else {
itr = influxql.NewMergeIterator(inputs, opt)
}
itrs = append(itrs, newLimitIterator(itr, opt))
} else {
itrs = append(itrs, inputs...)
}
}
}
return nil
}(); err != nil {
influxql.Iterators(itrs).Close()
return nil, err
}
return itrs, nil
}