// Ensure a shard can create iterators for its underlying data.
func TestShard_CreateIterator(t *testing.T) {
sh := MustOpenShard()
defer sh.Close()
sh.MustWritePointsString(`
cpu,host=serverA,region=uswest value=100 0
cpu,host=serverA,region=uswest value=50,val2=5 10
cpu,host=serverB,region=uswest value=25 0
`)
// Create iterator.
itr, err := sh.CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Aux: []string{"val2"},
Dimensions: []string{"host"},
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
Ascending: true,
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
})
if err != nil {
t.Fatal(err)
}
defer itr.Close()
fitr := itr.(influxql.FloatIterator)
// Read values from iterator.
if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverA"}),
Time: time.Unix(0, 0).UnixNano(),
Value: 100,
Aux: []interface{}{(*float64)(nil)},
}) {
t.Fatalf("unexpected point(0): %s", spew.Sdump(p))
}
if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverA"}),
Time: time.Unix(10, 0).UnixNano(),
Value: 50,
Aux: []interface{}{float64(5)},
}) {
t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
}
if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverB"}),
Time: time.Unix(0, 0).UnixNano(),
Value: 25,
Aux: []interface{}{math.NaN()},
}) {
t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
}
}
func (e *Engine) SeriesKeys(opt influxql.IteratorOptions) (influxql.SeriesList, error) {
seriesList := influxql.SeriesList{}
mms := tsdb.Measurements(e.index.MeasurementsByName(influxql.Sources(opt.Sources).Names()))
for _, mm := range mms {
// Determine tagsets for this measurement based on dimensions and filters.
tagSets, err := mm.TagSets(opt.Dimensions, opt.Condition)
if err != nil {
return nil, err
}
// Calculate tag sets and apply SLIMIT/SOFFSET.
tagSets = influxql.LimitTagSets(tagSets, opt.SLimit, opt.SOffset)
for _, t := range tagSets {
series := influxql.Series{
Name: mm.Name,
Tags: influxql.NewTags(t.Tags),
Aux: make([]influxql.DataType, len(opt.Aux)),
}
// Determine the aux field types.
for _, seriesKey := range t.SeriesKeys {
tags := influxql.NewTags(e.index.TagsForSeries(seriesKey))
for i, field := range opt.Aux {
typ := func() influxql.DataType {
mf := e.measurementFields[mm.Name]
if mf == nil {
return influxql.Unknown
}
f := mf.Field(field)
if f == nil {
return influxql.Unknown
}
return f.Type
}()
if typ == influxql.Unknown {
if v := tags.Value(field); v != "" {
// All tags are strings.
typ = influxql.String
}
}
if typ != influxql.Unknown {
if series.Aux[i] == influxql.Unknown || typ < series.Aux[i] {
series.Aux[i] = typ
}
}
}
}
seriesList = append(seriesList, series)
}
}
return seriesList, nil
}
// NewFloatCursorIterator returns a new instance of FloatCursorIterator.
func NewFloatCursorIterator(name string, tagMap map[string]string, cur Cursor, opt influxql.IteratorOptions) *FloatCursorIterator {
// Extract variable reference if available.
var ref *influxql.VarRef
if opt.Expr != nil {
ref = opt.Expr.(*influxql.VarRef)
}
// Only allocate aux values if we have any requested.
var aux []interface{}
if len(opt.Aux) > 0 {
aux = make([]interface{}, len(opt.Aux))
}
// Convert to influxql tags.
tags := influxql.NewTags(tagMap)
// Determine initial seek position based on sort direction.
seek := opt.StartTime
if !opt.Ascending {
seek = opt.EndTime
}
return &FloatCursorIterator{
point: influxql.FloatPoint{
Name: name,
Tags: tags.Subset(opt.Dimensions),
Aux: aux,
},
opt: opt,
ref: ref,
tags: tags,
cursor: newBufCursor(cur, seek),
}
}
func (a *booleanPointBulkAggregator) AggregatePoint(p *models.Point) error {
value, ok := p.Fields[a.field]
if !ok {
return fmt.Errorf("field %s missing from point cannot aggregate", a.field)
}
typed, ok := value.(bool)
if !ok {
return fmt.Errorf("field %s has wrong type: got %T exp bool", a.field, value)
}
ap := &influxql.BooleanPoint{
Name: p.Name,
Tags: influxql.NewTags(p.Tags),
Time: p.Time.UnixNano(),
Value: typed,
}
if a.topBottomInfo != nil {
// We need to populate the Aux fields
booleanPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
}
if a.isSimpleSelector {
ap.Aux = []interface{}{p.Tags, p.Fields}
}
a.aggregator.AggregateBoolean(ap)
return nil
}
func (a *booleanPointBulkAggregator) AggregateBatch(b *models.Batch) error {
slice := make([]influxql.BooleanPoint, len(b.Points))
for i, p := range b.Points {
value, ok := p.Fields[a.field]
if !ok {
return fmt.Errorf("field %s missing from point cannot aggregate", a.field)
}
typed, ok := value.(bool)
if !ok {
return fmt.Errorf("field %s has wrong type: got %T exp bool", a.field, value)
}
slice[i] = influxql.BooleanPoint{
Name: b.Name,
Tags: influxql.NewTags(p.Tags),
Time: p.Time.UnixNano(),
Value: typed,
}
if a.topBottomInfo != nil {
// We need to populate the Aux fields
booleanPopulateAuxFieldsAndTags(&slice[i], a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
}
if a.isSimpleSelector {
slice[i].Aux = []interface{}{p.Tags, p.Fields}
}
}
a.aggregator.AggregateBooleanBulk(slice)
return nil
}
func (a *stringPointAggregator) AggregateBatch(b *models.Batch) error {
for _, p := range b.Points {
value, ok := p.Fields[a.field]
if !ok {
return fmt.Errorf("field %s missing from point cannot aggregate", a.field)
}
typed, ok := value.(string)
if !ok {
return fmt.Errorf("field %s has wrong type: got %T exp string", a.field, value)
}
ap := &influxql.StringPoint{
Name: b.Name,
Tags: influxql.NewTags(p.Tags),
Time: p.Time.UnixNano(),
Value: typed,
}
if a.topBottomInfo != nil {
// We need to populate the Aux fields
stringPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
}
if a.isSimpleSelector {
ap.Aux = []interface{}{p.Tags, p.Fields}
}
a.aggregator.AggregateString(ap)
}
return nil
}
// ParseTags returns an instance of Tags for a comma-delimited list of key/values.
func ParseTags(s string) influxql.Tags {
m := make(map[string]string)
for _, kv := range strings.Split(s, ",") {
a := strings.Split(kv, "=")
m[a[0]] = a[1]
}
return influxql.NewTags(m)
}
func (a *floatPointBulkAggregator) AggregatePoint(p *models.Point) {
ap := &influxql.FloatPoint{
Name: p.Name,
Tags: influxql.NewTags(p.Tags),
Time: p.Time.UnixNano(),
Value: p.Fields[a.field].(float64),
}
if a.topBottomInfo != nil {
// We need to populate the Aux fields
floatPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
}
a.aggregator.AggregateFloat(ap)
}
// Ensure that a subset can be created from a tag set.
func TestTags_Subset(t *testing.T) {
tags := influxql.NewTags(map[string]string{"a": "0", "b": "1", "c": "2"})
subset := tags.Subset([]string{"b", "c", "d"})
if keys := subset.Keys(); !reflect.DeepEqual(keys, []string{"b", "c", "d"}) {
t.Fatalf("unexpected keys: %+v", keys)
} else if v := subset.Value("a"); v != "" {
t.Fatalf("unexpected 'a' value: %s", v)
} else if v := subset.Value("b"); v != "1" {
t.Fatalf("unexpected 'b' value: %s", v)
} else if v := subset.Value("c"); v != "2" {
t.Fatalf("unexpected 'c' value: %s", v)
} else if v := subset.Value("d"); v != "" {
t.Fatalf("unexpected 'd' value: %s", v)
}
}
func (a *floatPointAggregator) AggregateBatch(b *models.Batch) {
for _, p := range b.Points {
ap := &influxql.FloatPoint{
Name: b.Name,
Tags: influxql.NewTags(p.Tags),
Time: p.Time.UnixNano(),
Value: p.Fields[a.field].(float64),
}
if a.topBottomInfo != nil {
// We need to populate the Aux fields
floatPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
}
a.aggregator.AggregateFloat(ap)
}
}
func (a *floatPointBulkAggregator) AggregateBatch(b *models.Batch) {
slice := make([]influxql.FloatPoint, len(b.Points))
for i, p := range b.Points {
slice[i] = influxql.FloatPoint{
Name: b.Name,
Tags: influxql.NewTags(p.Tags),
Time: p.Time.UnixNano(),
Value: p.Fields[a.field].(float64),
}
if a.topBottomInfo != nil {
// We need to populate the Aux fields
floatPopulateAuxFieldsAndTags(&slice[i], a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
}
}
a.aggregator.AggregateFloatBulk(slice)
}
// createVarRefSeriesIterator creates an iterator for a variable reference for a series.
func (e *Engine) createVarRefSeriesIterator(ref *influxql.VarRef, mm *tsdb.Measurement, seriesKey string, t *influxql.TagSet, filter influxql.Expr, conditionFields []influxql.VarRef, opt influxql.IteratorOptions) (influxql.Iterator, error) {
tags := influxql.NewTags(e.index.TagsForSeries(seriesKey).Map())
// Create options specific for this series.
itrOpt := opt
itrOpt.Condition = filter
// Build auxilary cursors.
// Tag values should be returned if the field doesn't exist.
var aux []cursorAt
if len(opt.Aux) > 0 {
aux = make([]cursorAt, len(opt.Aux))
for i, ref := range opt.Aux {
// Create cursor from field if a tag wasn't requested.
if ref.Type != influxql.Tag {
cur := e.buildCursor(mm.Name, seriesKey, &ref, opt)
if cur != nil {
aux[i] = newBufCursor(cur, opt.Ascending)
continue
}
// If a field was requested, use a nil cursor of the requested type.
switch ref.Type {
case influxql.Float, influxql.AnyField:
aux[i] = &floatNilLiteralCursor{}
continue
case influxql.Integer:
aux[i] = &integerNilLiteralCursor{}
continue
case influxql.String:
aux[i] = &stringNilLiteralCursor{}
continue
case influxql.Boolean:
aux[i] = &booleanNilLiteralCursor{}
continue
}
}
// If field doesn't exist, use the tag value.
if v := tags.Value(ref.Val); v == "" {
// However, if the tag value is blank then return a null.
aux[i] = &stringNilLiteralCursor{}
} else {
aux[i] = &stringLiteralCursor{value: v}
}
}
}
// Build conditional field cursors.
// If a conditional field doesn't exist then ignore the series.
var conds []cursorAt
if len(conditionFields) > 0 {
conds = make([]cursorAt, len(conditionFields))
for i, ref := range conditionFields {
// Create cursor from field if a tag wasn't requested.
if ref.Type != influxql.Tag {
cur := e.buildCursor(mm.Name, seriesKey, &ref, opt)
if cur != nil {
conds[i] = newBufCursor(cur, opt.Ascending)
continue
}
// If a field was requested, use a nil cursor of the requested type.
switch ref.Type {
case influxql.Float, influxql.AnyField:
conds[i] = &floatNilLiteralCursor{}
continue
case influxql.Integer:
conds[i] = &integerNilLiteralCursor{}
continue
case influxql.String:
conds[i] = &stringNilLiteralCursor{}
continue
case influxql.Boolean:
conds[i] = &booleanNilLiteralCursor{}
continue
}
}
// If field doesn't exist, use the tag value.
if v := tags.Value(ref.Val); v == "" {
// However, if the tag value is blank then return a null.
conds[i] = &stringNilLiteralCursor{}
} else {
conds[i] = &stringLiteralCursor{value: v}
}
}
}
condNames := influxql.VarRefs(conditionFields).Strings()
// Limit tags to only the dimensions selected.
tags = tags.Subset(opt.Dimensions)
// If it's only auxiliary fields then it doesn't matter what type of iterator we use.
if ref == nil {
return newFloatIterator(mm.Name, tags, itrOpt, nil, aux, conds, condNames), nil
}
// Build main cursor.
cur := e.buildCursor(mm.Name, seriesKey, ref, opt)
// If the field doesn't exist then don't build an iterator.
if cur == nil {
return nil, nil
}
switch cur := cur.(type) {
case floatCursor:
return newFloatIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil
case integerCursor:
return newIntegerIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil
case stringCursor:
return newStringIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil
case booleanCursor:
return newBooleanIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil
default:
panic("unreachable")
}
}
// Ensure a shard can create iterators for its underlying data.
func TestShard_CreateIterator_Descending(t *testing.T) {
sh := NewShard()
// Calling CreateIterator when the engine is not open will return
// ErrEngineClosed.
_, got := sh.CreateIterator(influxql.IteratorOptions{})
if exp := tsdb.ErrEngineClosed; got != exp {
t.Fatalf("got %v, expected %v", got, exp)
}
if err := sh.Open(); err != nil {
t.Fatal(err)
}
defer sh.Close()
sh.MustWritePointsString(`
cpu,host=serverA,region=uswest value=100 0
cpu,host=serverA,region=uswest value=50,val2=5 10
cpu,host=serverB,region=uswest value=25 0
`)
// Create iterator.
itr, err := sh.CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Aux: []string{"val2"},
Dimensions: []string{"host"},
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
Ascending: false,
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
})
if err != nil {
t.Fatal(err)
}
defer itr.Close()
fitr := itr.(influxql.FloatIterator)
// Read values from iterator.
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(0): %s", err)
} else if !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverB"}),
Time: time.Unix(0, 0).UnixNano(),
Value: 25,
Aux: []interface{}{(*float64)(nil)},
}) {
t.Fatalf("unexpected point(0): %s", spew.Sdump(p))
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(1): %s", err)
} else if !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverA"}),
Time: time.Unix(10, 0).UnixNano(),
Value: 50,
Aux: []interface{}{float64(5)},
}) {
t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(2): %s", err)
} else if !deep.Equal(p, &influxql.FloatPoint{
Name: "cpu",
Tags: influxql.NewTags(map[string]string{"host": "serverA"}),
Time: time.Unix(0, 0).UnixNano(),
Value: 100,
Aux: []interface{}{(*float64)(nil)},
}) {
t.Fatalf("unexpected point(2): %s", spew.Sdump(p))
}
}
// Ensure that tags can return a unique id.
func TestTags_ID(t *testing.T) {
tags := influxql.NewTags(map[string]string{"foo": "bar", "baz": "bat"})
if id := tags.ID(); id != "baz\x00foo\x00bat\x00bar" {
t.Fatalf("unexpected id: %q", id)
}
}