// Ensure engine can create an iterator with a condition.
func TestEngine_CreateIterator_Condition(t *testing.T) {
t.Parallel()
e := MustOpenEngine()
defer e.Close()
e.Index().CreateMeasurementIndexIfNotExists("cpu")
e.Index().Measurement("cpu").SetFieldName("X")
e.Index().Measurement("cpu").SetFieldName("Y")
e.MeasurementFields("cpu").CreateFieldIfNotExists("value", influxql.Float, false)
e.MeasurementFields("cpu").CreateFieldIfNotExists("X", influxql.Float, false)
e.MeasurementFields("cpu").CreateFieldIfNotExists("Y", influxql.Float, false)
si := e.Index().CreateSeriesIndexIfNotExists("cpu", tsdb.NewSeries("cpu,host=A", models.NewTags(map[string]string{"host": "A"})))
si.AssignShard(1)
if err := e.WritePointsString(
`cpu,host=A value=1.1 1000000000`,
`cpu,host=A X=10 1000000000`,
`cpu,host=A Y=100 1000000000`,
`cpu,host=A value=1.2 2000000000`,
`cpu,host=A value=1.3 3000000000`,
`cpu,host=A X=20 3000000000`,
`cpu,host=A Y=200 3000000000`,
); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
itr, err := e.CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Dimensions: []string{"host"},
Condition: influxql.MustParseExpr(`X = 10 OR Y > 150`),
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
Ascending: true,
})
if err != nil {
t.Fatal(err)
}
fitr := itr.(influxql.FloatIterator)
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(0): %v", err)
} else if !reflect.DeepEqual(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 1000000000, Value: 1.1}) {
t.Fatalf("unexpected point(0): %v", p)
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected point(1): %v", err)
} else if !reflect.DeepEqual(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 3000000000, Value: 1.3}) {
t.Fatalf("unexpected point(1): %v", p)
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("expected eof, got error: %v", err)
} else if p != nil {
t.Fatalf("expected eof: %v", p)
}
}
func benchmarkEngineCreateIteratorCount(b *testing.B, pointN int) {
benchmarkIterator(b, influxql.IteratorOptions{
Expr: influxql.MustParseExpr("count(value)"),
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
Ascending: true,
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
}, pointN)
}
// Ensure a shard can create iterators for its underlying data.
func TestShard_CreateIterator(t *testing.T) {
sh := MustOpenShard()
defer sh.Close()
sh.MustWritePointsString(`
cpu,host=serverA,region=uswest value=100 0
cpu,host=serverA,region=uswest value=50,val2=5 10
cpu,host=serverB,region=uswest value=25 0
`)
// Create iterator.
itr, err := sh.CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Aux: []string{"val2"},
Dimensions: []string{"host"},
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
Ascending: true,
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
})
if err != nil {
t.Fatal(err)
}
defer itr.Close()
fitr := itr.(influxql.FloatIterator)
// Read values from iterator.
if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverA"}),
Time: time.Unix(0, 0).UnixNano(),
Value: 100,
Aux: []interface{}{(*float64)(nil)},
}) {
t.Fatalf("unexpected point(0): %s", spew.Sdump(p))
}
if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverA"}),
Time: time.Unix(10, 0).UnixNano(),
Value: 50,
Aux: []interface{}{float64(5)},
}) {
t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
}
if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverB"}),
Time: time.Unix(0, 0).UnixNano(),
Value: 25,
Aux: []interface{}{math.NaN()},
}) {
t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
}
}
// Ensure engine can create an iterator with auxilary fields.
func TestEngine_CreateIterator_Aux(t *testing.T) {
t.Parallel()
e := MustOpenEngine()
defer e.Close()
e.Index().CreateMeasurementIndexIfNotExists("cpu")
e.MeasurementFields("cpu").CreateFieldIfNotExists("value", influxql.Float, false)
e.MeasurementFields("cpu").CreateFieldIfNotExists("F", influxql.Float, false)
si := e.Index().CreateSeriesIndexIfNotExists("cpu", tsdb.NewSeries("cpu,host=A", models.NewTags(map[string]string{"host": "A"})))
si.AssignShard(1)
if err := e.WritePointsString(
`cpu,host=A value=1.1 1000000000`,
`cpu,host=A F=100 1000000000`,
`cpu,host=A value=1.2 2000000000`,
`cpu,host=A value=1.3 3000000000`,
`cpu,host=A F=200 3000000000`,
); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
itr, err := e.CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Aux: []influxql.VarRef{{Val: "F"}},
Dimensions: []string{"host"},
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
Ascending: true,
})
if err != nil {
t.Fatal(err)
}
fitr := itr.(influxql.FloatIterator)
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(0): %v", err)
} else if !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 1000000000, Value: 1.1, Aux: []interface{}{float64(100)}}) {
t.Fatalf("unexpected point(0): %v", p)
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(1): %v", err)
} else if !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 2000000000, Value: 1.2, Aux: []interface{}{(*float64)(nil)}}) {
t.Fatalf("unexpected point(1): %v", p)
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(2): %v", err)
} else if !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 3000000000, Value: 1.3, Aux: []interface{}{float64(200)}}) {
t.Fatalf("unexpected point(2): %v", p)
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("expected eof, got error: %v", err)
} else if p != nil {
t.Fatalf("expected eof: %v", p)
}
}
// Ensure shards can create iterators.
func TestShards_CreateIterator(t *testing.T) {
s := MustOpenStore()
defer s.Close()
// Create shard #0 with data.
s.MustCreateShardWithData("db0", "rp0", 0,
`cpu,host=serverA value=1 0`,
`cpu,host=serverA value=2 10`,
`cpu,host=serverB value=3 20`,
)
// Create shard #1 with data.
s.MustCreateShardWithData("db0", "rp0", 1,
`cpu,host=serverA value=1 30`,
`mem,host=serverA value=2 40`, // skip: wrong source
`cpu,host=serverC value=3 60`,
)
// Create iterator.
itr, err := tsdb.Shards(s.Shards([]uint64{0, 1})).CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Dimensions: []string{"host"},
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
Ascending: true,
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
})
if err != nil {
t.Fatal(err)
}
defer itr.Close()
fitr := itr.(influxql.FloatIterator)
// Read values from iterator. The host=serverA points should come first.
if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=serverA"), Time: time.Unix(0, 0).UnixNano(), Value: 1}) {
t.Fatalf("unexpected point(0): %s", spew.Sdump(p))
} else if p = fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=serverA"), Time: time.Unix(10, 0).UnixNano(), Value: 2}) {
t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
} else if p = fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=serverA"), Time: time.Unix(30, 0).UnixNano(), Value: 1}) {
t.Fatalf("unexpected point(2): %s", spew.Sdump(p))
}
// Next the host=serverB point.
if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=serverB"), Time: time.Unix(20, 0).UnixNano(), Value: 3}) {
t.Fatalf("unexpected point(3): %s", spew.Sdump(p))
}
// And finally the host=serverC point.
if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=serverC"), Time: time.Unix(60, 0).UnixNano(), Value: 3}) {
t.Fatalf("unexpected point(4): %s", spew.Sdump(p))
}
// Then an EOF should occur.
if p := fitr.Next(); p != nil {
t.Fatalf("expected eof, got: %s", spew.Sdump(p))
}
}
func benchmarkEngineCreateIteratorLimit(b *testing.B, pointN int) {
benchmarkIterator(b, influxql.IteratorOptions{
Expr: influxql.MustParseExpr("value"),
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
Dimensions: []string{"host"},
Ascending: true,
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
Limit: 10,
}, pointN)
}
// Ensure engine can create an descending iterator for cached values.
func TestEngine_CreateIterator_TSM_Descending(t *testing.T) {
t.Parallel()
e := MustOpenEngine()
defer e.Close()
e.Index().CreateMeasurementIndexIfNotExists("cpu")
e.MeasurementFields("cpu").CreateFieldIfNotExists("value", influxql.Float, false)
e.Index().CreateSeriesIndexIfNotExists("cpu", tsdb.NewSeries("cpu,host=A", map[string]string{"host": "A"}))
if err := e.WritePointsString(
`cpu,host=A value=1.1 1000000000`,
`cpu,host=A value=1.2 2000000000`,
`cpu,host=A value=1.3 3000000000`,
); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
e.MustWriteSnapshot()
itr, err := e.CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Dimensions: []string{"host"},
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
Ascending: false,
})
if err != nil {
t.Fatal(err)
}
fitr := itr.(influxql.FloatIterator)
if p := fitr.Next(); !reflect.DeepEqual(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 3000000000, Value: 1.3}) {
t.Fatalf("unexpected point(0): %v", p)
}
if p := fitr.Next(); !reflect.DeepEqual(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 2000000000, Value: 1.2}) {
t.Fatalf("unexpected point(1): %v", p)
}
if p := fitr.Next(); !reflect.DeepEqual(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 1000000000, Value: 1.1}) {
t.Fatalf("unexpected point(2): %v", p)
}
if p := fitr.Next(); p != nil {
t.Fatalf("expected eof: %v", p)
}
}
// Ensure binary expression names can be evaluated.
func TestBinaryExprName(t *testing.T) {
for i, tt := range []struct {
expr string
name string
}{
{expr: `value + 1`, name: `value`},
{expr: `"user" / total`, name: `user_total`},
{expr: `("user" + total) / total`, name: `user_total_total`},
} {
expr := influxql.MustParseExpr(tt.expr)
switch expr := expr.(type) {
case *influxql.BinaryExpr:
name := influxql.BinaryExprName(expr)
if name != tt.name {
t.Errorf("%d. unexpected name %s, got %s", i, name, tt.name)
}
default:
t.Errorf("%d. unexpected expr type: %T", i, expr)
}
}
}
// Ensure a shard can create iterators for its underlying data.
func TestShard_CreateIterator_Descending(t *testing.T) {
sh := NewShard()
// Calling CreateIterator when the engine is not open will return
// ErrEngineClosed.
_, got := sh.CreateIterator(influxql.IteratorOptions{})
if exp := tsdb.ErrEngineClosed; got != exp {
t.Fatalf("got %v, expected %v", got, exp)
}
if err := sh.Open(); err != nil {
t.Fatal(err)
}
defer sh.Close()
sh.MustWritePointsString(`
cpu,host=serverA,region=uswest value=100 0
cpu,host=serverA,region=uswest value=50,val2=5 10
cpu,host=serverB,region=uswest value=25 0
`)
// Create iterator.
itr, err := sh.CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Aux: []string{"val2"},
Dimensions: []string{"host"},
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
Ascending: false,
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
})
if err != nil {
t.Fatal(err)
}
defer itr.Close()
fitr := itr.(influxql.FloatIterator)
// Read values from iterator.
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(0): %s", err)
} else if !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverB"}),
Time: time.Unix(0, 0).UnixNano(),
Value: 25,
Aux: []interface{}{(*float64)(nil)},
}) {
t.Fatalf("unexpected point(0): %s", spew.Sdump(p))
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(1): %s", err)
} else if !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverA"}),
Time: time.Unix(10, 0).UnixNano(),
Value: 50,
Aux: []interface{}{float64(5)},
}) {
t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(2): %s", err)
} else if !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverA"}),
Time: time.Unix(0, 0).UnixNano(),
Value: 100,
Aux: []interface{}{(*float64)(nil)},
}) {
t.Fatalf("unexpected point(2): %s", spew.Sdump(p))
}
}
// Ensure the store can backup a shard and another store can restore it.
func TestStore_BackupRestoreShard(t *testing.T) {
s0, s1 := MustOpenStore(), MustOpenStore()
defer s0.Close()
defer s1.Close()
// Create shard with data.
s0.MustCreateShardWithData("db0", "rp0", 100,
`cpu value=1 0`,
`cpu value=2 10`,
`cpu value=3 20`,
)
// Backup shard to a buffer.
var buf bytes.Buffer
if err := s0.BackupShard(100, time.Time{}, &buf); err != nil {
t.Fatal(err)
}
// Create the shard on the other store and restore from buffer.
if err := s1.CreateShard("db0", "rp0", 100); err != nil {
t.Fatal(err)
}
if err := s1.RestoreShard(100, &buf); err != nil {
t.Fatal(err)
}
// Read data from
itr, err := s1.Shard(100).CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Sources: []influxql.Source{&influxql.Measurement{
Name: "cpu",
Database: "db0",
RetentionPolicy: "rp0",
}},
Ascending: true,
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
})
if err != nil {
t.Fatal(err)
}
fitr := itr.(influxql.FloatIterator)
// Read values from iterator. The host=serverA points should come first.
p, e := fitr.Next()
if e != nil {
t.Fatal(e)
}
if !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Time: time.Unix(0, 0).UnixNano(), Value: 1}) {
t.Fatalf("unexpected point(0): %s", spew.Sdump(p))
}
p, e = fitr.Next()
if e != nil {
t.Fatal(e)
}
if !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Time: time.Unix(10, 0).UnixNano(), Value: 2}) {
t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
}
p, e = fitr.Next()
if e != nil {
t.Fatal(e)
}
if !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Time: time.Unix(20, 0).UnixNano(), Value: 3}) {
t.Fatalf("unexpected point(2): %s", spew.Sdump(p))
}
}