// Ensure a shard can create iterators for its underlying data. func TestShard_CreateIterator(t *testing.T) { sh := MustOpenShard() defer sh.Close() sh.MustWritePointsString(` cpu,host=serverA,region=uswest value=100 0 cpu,host=serverA,region=uswest value=50,val2=5 10 cpu,host=serverB,region=uswest value=25 0 `) // Create iterator. itr, err := sh.CreateIterator(influxql.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Aux: []string{"val2"}, Dimensions: []string{"host"}, Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}}, Ascending: true, StartTime: influxql.MinTime, EndTime: influxql.MaxTime, }) if err != nil { t.Fatal(err) } defer itr.Close() fitr := itr.(influxql.FloatIterator) // Read values from iterator. if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{ Name: "cpu", Tags: influxql.NewTags(map[string]string{"host": "serverA"}), Time: time.Unix(0, 0).UnixNano(), Value: 100, Aux: []interface{}{(*float64)(nil)}, }) { t.Fatalf("unexpected point(0): %s", spew.Sdump(p)) } if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{ Name: "cpu", Tags: influxql.NewTags(map[string]string{"host": "serverA"}), Time: time.Unix(10, 0).UnixNano(), Value: 50, Aux: []interface{}{float64(5)}, }) { t.Fatalf("unexpected point(1): %s", spew.Sdump(p)) } if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{ Name: "cpu", Tags: influxql.NewTags(map[string]string{"host": "serverB"}), Time: time.Unix(0, 0).UnixNano(), Value: 25, Aux: []interface{}{math.NaN()}, }) { t.Fatalf("unexpected point(1): %s", spew.Sdump(p)) } }
func (e *Engine) SeriesKeys(opt influxql.IteratorOptions) (influxql.SeriesList, error) { seriesList := influxql.SeriesList{} mms := tsdb.Measurements(e.index.MeasurementsByName(influxql.Sources(opt.Sources).Names())) for _, mm := range mms { // Determine tagsets for this measurement based on dimensions and filters. tagSets, err := mm.TagSets(opt.Dimensions, opt.Condition) if err != nil { return nil, err } // Calculate tag sets and apply SLIMIT/SOFFSET. tagSets = influxql.LimitTagSets(tagSets, opt.SLimit, opt.SOffset) for _, t := range tagSets { series := influxql.Series{ Name: mm.Name, Tags: influxql.NewTags(t.Tags), Aux: make([]influxql.DataType, len(opt.Aux)), } // Determine the aux field types. for _, seriesKey := range t.SeriesKeys { tags := influxql.NewTags(e.index.TagsForSeries(seriesKey)) for i, field := range opt.Aux { typ := func() influxql.DataType { mf := e.measurementFields[mm.Name] if mf == nil { return influxql.Unknown } f := mf.Field(field) if f == nil { return influxql.Unknown } return f.Type }() if typ == influxql.Unknown { if v := tags.Value(field); v != "" { // All tags are strings. typ = influxql.String } } if typ != influxql.Unknown { if series.Aux[i] == influxql.Unknown || typ < series.Aux[i] { series.Aux[i] = typ } } } } seriesList = append(seriesList, series) } } return seriesList, nil }
// NewFloatCursorIterator returns a new instance of FloatCursorIterator. func NewFloatCursorIterator(name string, tagMap map[string]string, cur Cursor, opt influxql.IteratorOptions) *FloatCursorIterator { // Extract variable reference if available. var ref *influxql.VarRef if opt.Expr != nil { ref = opt.Expr.(*influxql.VarRef) } // Only allocate aux values if we have any requested. var aux []interface{} if len(opt.Aux) > 0 { aux = make([]interface{}, len(opt.Aux)) } // Convert to influxql tags. tags := influxql.NewTags(tagMap) // Determine initial seek position based on sort direction. seek := opt.StartTime if !opt.Ascending { seek = opt.EndTime } return &FloatCursorIterator{ point: influxql.FloatPoint{ Name: name, Tags: tags.Subset(opt.Dimensions), Aux: aux, }, opt: opt, ref: ref, tags: tags, cursor: newBufCursor(cur, seek), } }
func (a *booleanPointBulkAggregator) AggregatePoint(p *models.Point) error { value, ok := p.Fields[a.field] if !ok { return fmt.Errorf("field %s missing from point cannot aggregate", a.field) } typed, ok := value.(bool) if !ok { return fmt.Errorf("field %s has wrong type: got %T exp bool", a.field, value) } ap := &influxql.BooleanPoint{ Name: p.Name, Tags: influxql.NewTags(p.Tags), Time: p.Time.UnixNano(), Value: typed, } if a.topBottomInfo != nil { // We need to populate the Aux fields booleanPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags) } if a.isSimpleSelector { ap.Aux = []interface{}{p.Tags, p.Fields} } a.aggregator.AggregateBoolean(ap) return nil }
func (a *booleanPointBulkAggregator) AggregateBatch(b *models.Batch) error { slice := make([]influxql.BooleanPoint, len(b.Points)) for i, p := range b.Points { value, ok := p.Fields[a.field] if !ok { return fmt.Errorf("field %s missing from point cannot aggregate", a.field) } typed, ok := value.(bool) if !ok { return fmt.Errorf("field %s has wrong type: got %T exp bool", a.field, value) } slice[i] = influxql.BooleanPoint{ Name: b.Name, Tags: influxql.NewTags(p.Tags), Time: p.Time.UnixNano(), Value: typed, } if a.topBottomInfo != nil { // We need to populate the Aux fields booleanPopulateAuxFieldsAndTags(&slice[i], a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags) } if a.isSimpleSelector { slice[i].Aux = []interface{}{p.Tags, p.Fields} } } a.aggregator.AggregateBooleanBulk(slice) return nil }
func (a *stringPointAggregator) AggregateBatch(b *models.Batch) error { for _, p := range b.Points { value, ok := p.Fields[a.field] if !ok { return fmt.Errorf("field %s missing from point cannot aggregate", a.field) } typed, ok := value.(string) if !ok { return fmt.Errorf("field %s has wrong type: got %T exp string", a.field, value) } ap := &influxql.StringPoint{ Name: b.Name, Tags: influxql.NewTags(p.Tags), Time: p.Time.UnixNano(), Value: typed, } if a.topBottomInfo != nil { // We need to populate the Aux fields stringPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags) } if a.isSimpleSelector { ap.Aux = []interface{}{p.Tags, p.Fields} } a.aggregator.AggregateString(ap) } return nil }
// ParseTags returns an instance of Tags for a comma-delimited list of key/values. func ParseTags(s string) influxql.Tags { m := make(map[string]string) for _, kv := range strings.Split(s, ",") { a := strings.Split(kv, "=") m[a[0]] = a[1] } return influxql.NewTags(m) }
func (a *floatPointBulkAggregator) AggregatePoint(p *models.Point) { ap := &influxql.FloatPoint{ Name: p.Name, Tags: influxql.NewTags(p.Tags), Time: p.Time.UnixNano(), Value: p.Fields[a.field].(float64), } if a.topBottomInfo != nil { // We need to populate the Aux fields floatPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags) } a.aggregator.AggregateFloat(ap) }
// Ensure that a subset can be created from a tag set. func TestTags_Subset(t *testing.T) { tags := influxql.NewTags(map[string]string{"a": "0", "b": "1", "c": "2"}) subset := tags.Subset([]string{"b", "c", "d"}) if keys := subset.Keys(); !reflect.DeepEqual(keys, []string{"b", "c", "d"}) { t.Fatalf("unexpected keys: %+v", keys) } else if v := subset.Value("a"); v != "" { t.Fatalf("unexpected 'a' value: %s", v) } else if v := subset.Value("b"); v != "1" { t.Fatalf("unexpected 'b' value: %s", v) } else if v := subset.Value("c"); v != "2" { t.Fatalf("unexpected 'c' value: %s", v) } else if v := subset.Value("d"); v != "" { t.Fatalf("unexpected 'd' value: %s", v) } }
func (a *floatPointAggregator) AggregateBatch(b *models.Batch) { for _, p := range b.Points { ap := &influxql.FloatPoint{ Name: b.Name, Tags: influxql.NewTags(p.Tags), Time: p.Time.UnixNano(), Value: p.Fields[a.field].(float64), } if a.topBottomInfo != nil { // We need to populate the Aux fields floatPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags) } a.aggregator.AggregateFloat(ap) } }
func (a *floatPointBulkAggregator) AggregateBatch(b *models.Batch) { slice := make([]influxql.FloatPoint, len(b.Points)) for i, p := range b.Points { slice[i] = influxql.FloatPoint{ Name: b.Name, Tags: influxql.NewTags(p.Tags), Time: p.Time.UnixNano(), Value: p.Fields[a.field].(float64), } if a.topBottomInfo != nil { // We need to populate the Aux fields floatPopulateAuxFieldsAndTags(&slice[i], a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags) } } a.aggregator.AggregateFloatBulk(slice) }
// createVarRefSeriesIterator creates an iterator for a variable reference for a series. func (e *Engine) createVarRefSeriesIterator(ref *influxql.VarRef, mm *tsdb.Measurement, seriesKey string, t *influxql.TagSet, filter influxql.Expr, conditionFields []influxql.VarRef, opt influxql.IteratorOptions) (influxql.Iterator, error) { tags := influxql.NewTags(e.index.TagsForSeries(seriesKey).Map()) // Create options specific for this series. itrOpt := opt itrOpt.Condition = filter // Build auxilary cursors. // Tag values should be returned if the field doesn't exist. var aux []cursorAt if len(opt.Aux) > 0 { aux = make([]cursorAt, len(opt.Aux)) for i, ref := range opt.Aux { // Create cursor from field if a tag wasn't requested. if ref.Type != influxql.Tag { cur := e.buildCursor(mm.Name, seriesKey, &ref, opt) if cur != nil { aux[i] = newBufCursor(cur, opt.Ascending) continue } // If a field was requested, use a nil cursor of the requested type. switch ref.Type { case influxql.Float, influxql.AnyField: aux[i] = &floatNilLiteralCursor{} continue case influxql.Integer: aux[i] = &integerNilLiteralCursor{} continue case influxql.String: aux[i] = &stringNilLiteralCursor{} continue case influxql.Boolean: aux[i] = &booleanNilLiteralCursor{} continue } } // If field doesn't exist, use the tag value. if v := tags.Value(ref.Val); v == "" { // However, if the tag value is blank then return a null. aux[i] = &stringNilLiteralCursor{} } else { aux[i] = &stringLiteralCursor{value: v} } } } // Build conditional field cursors. // If a conditional field doesn't exist then ignore the series. var conds []cursorAt if len(conditionFields) > 0 { conds = make([]cursorAt, len(conditionFields)) for i, ref := range conditionFields { // Create cursor from field if a tag wasn't requested. if ref.Type != influxql.Tag { cur := e.buildCursor(mm.Name, seriesKey, &ref, opt) if cur != nil { conds[i] = newBufCursor(cur, opt.Ascending) continue } // If a field was requested, use a nil cursor of the requested type. switch ref.Type { case influxql.Float, influxql.AnyField: conds[i] = &floatNilLiteralCursor{} continue case influxql.Integer: conds[i] = &integerNilLiteralCursor{} continue case influxql.String: conds[i] = &stringNilLiteralCursor{} continue case influxql.Boolean: conds[i] = &booleanNilLiteralCursor{} continue } } // If field doesn't exist, use the tag value. if v := tags.Value(ref.Val); v == "" { // However, if the tag value is blank then return a null. conds[i] = &stringNilLiteralCursor{} } else { conds[i] = &stringLiteralCursor{value: v} } } } condNames := influxql.VarRefs(conditionFields).Strings() // Limit tags to only the dimensions selected. tags = tags.Subset(opt.Dimensions) // If it's only auxiliary fields then it doesn't matter what type of iterator we use. if ref == nil { return newFloatIterator(mm.Name, tags, itrOpt, nil, aux, conds, condNames), nil } // Build main cursor. cur := e.buildCursor(mm.Name, seriesKey, ref, opt) // If the field doesn't exist then don't build an iterator. if cur == nil { return nil, nil } switch cur := cur.(type) { case floatCursor: return newFloatIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil case integerCursor: return newIntegerIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil case stringCursor: return newStringIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil case booleanCursor: return newBooleanIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil default: panic("unreachable") } }
// Ensure a shard can create iterators for its underlying data. func TestShard_CreateIterator_Descending(t *testing.T) { sh := NewShard() // Calling CreateIterator when the engine is not open will return // ErrEngineClosed. _, got := sh.CreateIterator(influxql.IteratorOptions{}) if exp := tsdb.ErrEngineClosed; got != exp { t.Fatalf("got %v, expected %v", got, exp) } if err := sh.Open(); err != nil { t.Fatal(err) } defer sh.Close() sh.MustWritePointsString(` cpu,host=serverA,region=uswest value=100 0 cpu,host=serverA,region=uswest value=50,val2=5 10 cpu,host=serverB,region=uswest value=25 0 `) // Create iterator. itr, err := sh.CreateIterator(influxql.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Aux: []string{"val2"}, Dimensions: []string{"host"}, Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}}, Ascending: false, StartTime: influxql.MinTime, EndTime: influxql.MaxTime, }) if err != nil { t.Fatal(err) } defer itr.Close() fitr := itr.(influxql.FloatIterator) // Read values from iterator. if p, err := fitr.Next(); err != nil { t.Fatalf("unexpected error(0): %s", err) } else if !deep.Equal(p, &influxql.FloatPoint{ Name: "cpu", Tags: influxql.NewTags(map[string]string{"host": "serverB"}), Time: time.Unix(0, 0).UnixNano(), Value: 25, Aux: []interface{}{(*float64)(nil)}, }) { t.Fatalf("unexpected point(0): %s", spew.Sdump(p)) } if p, err := fitr.Next(); err != nil { t.Fatalf("unexpected error(1): %s", err) } else if !deep.Equal(p, &influxql.FloatPoint{ Name: "cpu", Tags: influxql.NewTags(map[string]string{"host": "serverA"}), Time: time.Unix(10, 0).UnixNano(), Value: 50, Aux: []interface{}{float64(5)}, }) { t.Fatalf("unexpected point(1): %s", spew.Sdump(p)) } if p, err := fitr.Next(); err != nil { t.Fatalf("unexpected error(2): %s", err) } else if !deep.Equal(p, &influxql.FloatPoint{ Name: "cpu", Tags: influxql.NewTags(map[string]string{"host": "serverA"}), Time: time.Unix(0, 0).UnixNano(), Value: 100, Aux: []interface{}{(*float64)(nil)}, }) { t.Fatalf("unexpected point(2): %s", spew.Sdump(p)) } }
// Ensure that tags can return a unique id. func TestTags_ID(t *testing.T) { tags := influxql.NewTags(map[string]string{"foo": "bar", "baz": "bat"}) if id := tags.ID(); id != "baz\x00foo\x00bat\x00bar" { t.Fatalf("unexpected id: %q", id) } }