Пример #1
0
// Ensure a shard can create iterators for its underlying data.
func TestShard_CreateIterator(t *testing.T) {
	sh := MustOpenShard()
	defer sh.Close()

	sh.MustWritePointsString(`
cpu,host=serverA,region=uswest value=100 0
cpu,host=serverA,region=uswest value=50,val2=5  10
cpu,host=serverB,region=uswest value=25  0
`)

	// Create iterator.
	itr, err := sh.CreateIterator(influxql.IteratorOptions{
		Expr:       influxql.MustParseExpr(`value`),
		Aux:        []string{"val2"},
		Dimensions: []string{"host"},
		Sources:    []influxql.Source{&influxql.Measurement{Name: "cpu"}},
		Ascending:  true,
		StartTime:  influxql.MinTime,
		EndTime:    influxql.MaxTime,
	})
	if err != nil {
		t.Fatal(err)
	}
	defer itr.Close()
	fitr := itr.(influxql.FloatIterator)

	// Read values from iterator.
	if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{
		Name:  "cpu",
		Tags:  influxql.NewTags(map[string]string{"host": "serverA"}),
		Time:  time.Unix(0, 0).UnixNano(),
		Value: 100,
		Aux:   []interface{}{(*float64)(nil)},
	}) {
		t.Fatalf("unexpected point(0): %s", spew.Sdump(p))
	}

	if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{
		Name:  "cpu",
		Tags:  influxql.NewTags(map[string]string{"host": "serverA"}),
		Time:  time.Unix(10, 0).UnixNano(),
		Value: 50,
		Aux:   []interface{}{float64(5)},
	}) {
		t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
	}

	if p := fitr.Next(); !deep.Equal(p, &influxql.FloatPoint{
		Name:  "cpu",
		Tags:  influxql.NewTags(map[string]string{"host": "serverB"}),
		Time:  time.Unix(0, 0).UnixNano(),
		Value: 25,
		Aux:   []interface{}{math.NaN()},
	}) {
		t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
	}
}
Пример #2
0
func (e *Engine) SeriesKeys(opt influxql.IteratorOptions) (influxql.SeriesList, error) {
	seriesList := influxql.SeriesList{}
	mms := tsdb.Measurements(e.index.MeasurementsByName(influxql.Sources(opt.Sources).Names()))
	for _, mm := range mms {
		// Determine tagsets for this measurement based on dimensions and filters.
		tagSets, err := mm.TagSets(opt.Dimensions, opt.Condition)
		if err != nil {
			return nil, err
		}

		// Calculate tag sets and apply SLIMIT/SOFFSET.
		tagSets = influxql.LimitTagSets(tagSets, opt.SLimit, opt.SOffset)
		for _, t := range tagSets {
			series := influxql.Series{
				Name: mm.Name,
				Tags: influxql.NewTags(t.Tags),
				Aux:  make([]influxql.DataType, len(opt.Aux)),
			}

			// Determine the aux field types.
			for _, seriesKey := range t.SeriesKeys {
				tags := influxql.NewTags(e.index.TagsForSeries(seriesKey))
				for i, field := range opt.Aux {
					typ := func() influxql.DataType {
						mf := e.measurementFields[mm.Name]
						if mf == nil {
							return influxql.Unknown
						}

						f := mf.Field(field)
						if f == nil {
							return influxql.Unknown
						}
						return f.Type
					}()

					if typ == influxql.Unknown {
						if v := tags.Value(field); v != "" {
							// All tags are strings.
							typ = influxql.String
						}
					}

					if typ != influxql.Unknown {
						if series.Aux[i] == influxql.Unknown || typ < series.Aux[i] {
							series.Aux[i] = typ
						}
					}
				}
			}
			seriesList = append(seriesList, series)
		}
	}
	return seriesList, nil
}
Пример #3
0
// NewFloatCursorIterator returns a new instance of FloatCursorIterator.
func NewFloatCursorIterator(name string, tagMap map[string]string, cur Cursor, opt influxql.IteratorOptions) *FloatCursorIterator {
	// Extract variable reference if available.
	var ref *influxql.VarRef
	if opt.Expr != nil {
		ref = opt.Expr.(*influxql.VarRef)
	}

	// Only allocate aux values if we have any requested.
	var aux []interface{}
	if len(opt.Aux) > 0 {
		aux = make([]interface{}, len(opt.Aux))
	}

	// Convert to influxql tags.
	tags := influxql.NewTags(tagMap)

	// Determine initial seek position based on sort direction.
	seek := opt.StartTime
	if !opt.Ascending {
		seek = opt.EndTime
	}

	return &FloatCursorIterator{
		point: influxql.FloatPoint{
			Name: name,
			Tags: tags.Subset(opt.Dimensions),
			Aux:  aux,
		},
		opt:    opt,
		ref:    ref,
		tags:   tags,
		cursor: newBufCursor(cur, seek),
	}
}
Пример #4
0
func (a *booleanPointBulkAggregator) AggregatePoint(p *models.Point) error {
	value, ok := p.Fields[a.field]
	if !ok {
		return fmt.Errorf("field %s missing from point cannot aggregate", a.field)
	}
	typed, ok := value.(bool)
	if !ok {
		return fmt.Errorf("field %s has wrong type: got %T exp bool", a.field, value)
	}
	ap := &influxql.BooleanPoint{
		Name:  p.Name,
		Tags:  influxql.NewTags(p.Tags),
		Time:  p.Time.UnixNano(),
		Value: typed,
	}
	if a.topBottomInfo != nil {
		// We need to populate the Aux fields
		booleanPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
	}

	if a.isSimpleSelector {
		ap.Aux = []interface{}{p.Tags, p.Fields}
	}

	a.aggregator.AggregateBoolean(ap)
	return nil
}
Пример #5
0
func (a *booleanPointBulkAggregator) AggregateBatch(b *models.Batch) error {
	slice := make([]influxql.BooleanPoint, len(b.Points))
	for i, p := range b.Points {
		value, ok := p.Fields[a.field]
		if !ok {
			return fmt.Errorf("field %s missing from point cannot aggregate", a.field)
		}
		typed, ok := value.(bool)
		if !ok {
			return fmt.Errorf("field %s has wrong type: got %T exp bool", a.field, value)
		}
		slice[i] = influxql.BooleanPoint{
			Name:  b.Name,
			Tags:  influxql.NewTags(p.Tags),
			Time:  p.Time.UnixNano(),
			Value: typed,
		}
		if a.topBottomInfo != nil {
			// We need to populate the Aux fields
			booleanPopulateAuxFieldsAndTags(&slice[i], a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
		}

		if a.isSimpleSelector {
			slice[i].Aux = []interface{}{p.Tags, p.Fields}
		}
	}
	a.aggregator.AggregateBooleanBulk(slice)
	return nil
}
Пример #6
0
func (a *stringPointAggregator) AggregateBatch(b *models.Batch) error {
	for _, p := range b.Points {
		value, ok := p.Fields[a.field]
		if !ok {
			return fmt.Errorf("field %s missing from point cannot aggregate", a.field)
		}
		typed, ok := value.(string)
		if !ok {
			return fmt.Errorf("field %s has wrong type: got %T exp string", a.field, value)
		}
		ap := &influxql.StringPoint{
			Name:  b.Name,
			Tags:  influxql.NewTags(p.Tags),
			Time:  p.Time.UnixNano(),
			Value: typed,
		}
		if a.topBottomInfo != nil {
			// We need to populate the Aux fields
			stringPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
		}

		if a.isSimpleSelector {
			ap.Aux = []interface{}{p.Tags, p.Fields}
		}

		a.aggregator.AggregateString(ap)
	}
	return nil
}
Пример #7
0
// ParseTags returns an instance of Tags for a comma-delimited list of key/values.
func ParseTags(s string) influxql.Tags {
	m := make(map[string]string)
	for _, kv := range strings.Split(s, ",") {
		a := strings.Split(kv, "=")
		m[a[0]] = a[1]
	}
	return influxql.NewTags(m)
}
Пример #8
0
func (a *floatPointBulkAggregator) AggregatePoint(p *models.Point) {
	ap := &influxql.FloatPoint{
		Name:  p.Name,
		Tags:  influxql.NewTags(p.Tags),
		Time:  p.Time.UnixNano(),
		Value: p.Fields[a.field].(float64),
	}
	if a.topBottomInfo != nil {
		// We need to populate the Aux fields
		floatPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
	}
	a.aggregator.AggregateFloat(ap)
}
Пример #9
0
// Ensure that a subset can be created from a tag set.
func TestTags_Subset(t *testing.T) {
	tags := influxql.NewTags(map[string]string{"a": "0", "b": "1", "c": "2"})
	subset := tags.Subset([]string{"b", "c", "d"})
	if keys := subset.Keys(); !reflect.DeepEqual(keys, []string{"b", "c", "d"}) {
		t.Fatalf("unexpected keys: %+v", keys)
	} else if v := subset.Value("a"); v != "" {
		t.Fatalf("unexpected 'a' value: %s", v)
	} else if v := subset.Value("b"); v != "1" {
		t.Fatalf("unexpected 'b' value: %s", v)
	} else if v := subset.Value("c"); v != "2" {
		t.Fatalf("unexpected 'c' value: %s", v)
	} else if v := subset.Value("d"); v != "" {
		t.Fatalf("unexpected 'd' value: %s", v)
	}
}
Пример #10
0
func (a *floatPointAggregator) AggregateBatch(b *models.Batch) {
	for _, p := range b.Points {
		ap := &influxql.FloatPoint{
			Name:  b.Name,
			Tags:  influxql.NewTags(p.Tags),
			Time:  p.Time.UnixNano(),
			Value: p.Fields[a.field].(float64),
		}
		if a.topBottomInfo != nil {
			// We need to populate the Aux fields
			floatPopulateAuxFieldsAndTags(ap, a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
		}
		a.aggregator.AggregateFloat(ap)
	}
}
Пример #11
0
func (a *floatPointBulkAggregator) AggregateBatch(b *models.Batch) {
	slice := make([]influxql.FloatPoint, len(b.Points))
	for i, p := range b.Points {
		slice[i] = influxql.FloatPoint{
			Name:  b.Name,
			Tags:  influxql.NewTags(p.Tags),
			Time:  p.Time.UnixNano(),
			Value: p.Fields[a.field].(float64),
		}
		if a.topBottomInfo != nil {
			// We need to populate the Aux fields
			floatPopulateAuxFieldsAndTags(&slice[i], a.topBottomInfo.FieldsAndTags, p.Fields, p.Tags)
		}
	}
	a.aggregator.AggregateFloatBulk(slice)
}
Пример #12
0
// createVarRefSeriesIterator creates an iterator for a variable reference for a series.
func (e *Engine) createVarRefSeriesIterator(ref *influxql.VarRef, mm *tsdb.Measurement, seriesKey string, t *influxql.TagSet, filter influxql.Expr, conditionFields []influxql.VarRef, opt influxql.IteratorOptions) (influxql.Iterator, error) {
	tags := influxql.NewTags(e.index.TagsForSeries(seriesKey).Map())

	// Create options specific for this series.
	itrOpt := opt
	itrOpt.Condition = filter

	// Build auxilary cursors.
	// Tag values should be returned if the field doesn't exist.
	var aux []cursorAt
	if len(opt.Aux) > 0 {
		aux = make([]cursorAt, len(opt.Aux))
		for i, ref := range opt.Aux {
			// Create cursor from field if a tag wasn't requested.
			if ref.Type != influxql.Tag {
				cur := e.buildCursor(mm.Name, seriesKey, &ref, opt)
				if cur != nil {
					aux[i] = newBufCursor(cur, opt.Ascending)
					continue
				}

				// If a field was requested, use a nil cursor of the requested type.
				switch ref.Type {
				case influxql.Float, influxql.AnyField:
					aux[i] = &floatNilLiteralCursor{}
					continue
				case influxql.Integer:
					aux[i] = &integerNilLiteralCursor{}
					continue
				case influxql.String:
					aux[i] = &stringNilLiteralCursor{}
					continue
				case influxql.Boolean:
					aux[i] = &booleanNilLiteralCursor{}
					continue
				}
			}

			// If field doesn't exist, use the tag value.
			if v := tags.Value(ref.Val); v == "" {
				// However, if the tag value is blank then return a null.
				aux[i] = &stringNilLiteralCursor{}
			} else {
				aux[i] = &stringLiteralCursor{value: v}
			}
		}
	}

	// Build conditional field cursors.
	// If a conditional field doesn't exist then ignore the series.
	var conds []cursorAt
	if len(conditionFields) > 0 {
		conds = make([]cursorAt, len(conditionFields))
		for i, ref := range conditionFields {
			// Create cursor from field if a tag wasn't requested.
			if ref.Type != influxql.Tag {
				cur := e.buildCursor(mm.Name, seriesKey, &ref, opt)
				if cur != nil {
					conds[i] = newBufCursor(cur, opt.Ascending)
					continue
				}

				// If a field was requested, use a nil cursor of the requested type.
				switch ref.Type {
				case influxql.Float, influxql.AnyField:
					conds[i] = &floatNilLiteralCursor{}
					continue
				case influxql.Integer:
					conds[i] = &integerNilLiteralCursor{}
					continue
				case influxql.String:
					conds[i] = &stringNilLiteralCursor{}
					continue
				case influxql.Boolean:
					conds[i] = &booleanNilLiteralCursor{}
					continue
				}
			}

			// If field doesn't exist, use the tag value.
			if v := tags.Value(ref.Val); v == "" {
				// However, if the tag value is blank then return a null.
				conds[i] = &stringNilLiteralCursor{}
			} else {
				conds[i] = &stringLiteralCursor{value: v}
			}
		}
	}
	condNames := influxql.VarRefs(conditionFields).Strings()

	// Limit tags to only the dimensions selected.
	tags = tags.Subset(opt.Dimensions)

	// If it's only auxiliary fields then it doesn't matter what type of iterator we use.
	if ref == nil {
		return newFloatIterator(mm.Name, tags, itrOpt, nil, aux, conds, condNames), nil
	}

	// Build main cursor.
	cur := e.buildCursor(mm.Name, seriesKey, ref, opt)

	// If the field doesn't exist then don't build an iterator.
	if cur == nil {
		return nil, nil
	}

	switch cur := cur.(type) {
	case floatCursor:
		return newFloatIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil
	case integerCursor:
		return newIntegerIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil
	case stringCursor:
		return newStringIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil
	case booleanCursor:
		return newBooleanIterator(mm.Name, tags, itrOpt, cur, aux, conds, condNames), nil
	default:
		panic("unreachable")
	}
}
Пример #13
0
// Ensure a shard can create iterators for its underlying data.
func TestShard_CreateIterator_Descending(t *testing.T) {
	sh := NewShard()

	// Calling CreateIterator when the engine is not open will return
	// ErrEngineClosed.
	_, got := sh.CreateIterator(influxql.IteratorOptions{})
	if exp := tsdb.ErrEngineClosed; got != exp {
		t.Fatalf("got %v, expected %v", got, exp)
	}

	if err := sh.Open(); err != nil {
		t.Fatal(err)
	}
	defer sh.Close()

	sh.MustWritePointsString(`
cpu,host=serverA,region=uswest value=100 0
cpu,host=serverA,region=uswest value=50,val2=5  10
cpu,host=serverB,region=uswest value=25  0
`)

	// Create iterator.
	itr, err := sh.CreateIterator(influxql.IteratorOptions{
		Expr:       influxql.MustParseExpr(`value`),
		Aux:        []string{"val2"},
		Dimensions: []string{"host"},
		Sources:    []influxql.Source{&influxql.Measurement{Name: "cpu"}},
		Ascending:  false,
		StartTime:  influxql.MinTime,
		EndTime:    influxql.MaxTime,
	})
	if err != nil {
		t.Fatal(err)
	}
	defer itr.Close()
	fitr := itr.(influxql.FloatIterator)

	// Read values from iterator.
	if p, err := fitr.Next(); err != nil {
		t.Fatalf("unexpected error(0): %s", err)
	} else if !deep.Equal(p, &influxql.FloatPoint{
		Name:  "cpu",
		Tags:  influxql.NewTags(map[string]string{"host": "serverB"}),
		Time:  time.Unix(0, 0).UnixNano(),
		Value: 25,
		Aux:   []interface{}{(*float64)(nil)},
	}) {
		t.Fatalf("unexpected point(0): %s", spew.Sdump(p))
	}

	if p, err := fitr.Next(); err != nil {
		t.Fatalf("unexpected error(1): %s", err)
	} else if !deep.Equal(p, &influxql.FloatPoint{
		Name:  "cpu",
		Tags:  influxql.NewTags(map[string]string{"host": "serverA"}),
		Time:  time.Unix(10, 0).UnixNano(),
		Value: 50,
		Aux:   []interface{}{float64(5)},
	}) {
		t.Fatalf("unexpected point(1): %s", spew.Sdump(p))
	}

	if p, err := fitr.Next(); err != nil {
		t.Fatalf("unexpected error(2): %s", err)
	} else if !deep.Equal(p, &influxql.FloatPoint{
		Name:  "cpu",
		Tags:  influxql.NewTags(map[string]string{"host": "serverA"}),
		Time:  time.Unix(0, 0).UnixNano(),
		Value: 100,
		Aux:   []interface{}{(*float64)(nil)},
	}) {
		t.Fatalf("unexpected point(2): %s", spew.Sdump(p))
	}
}
Пример #14
0
// Ensure that tags can return a unique id.
func TestTags_ID(t *testing.T) {
	tags := influxql.NewTags(map[string]string{"foo": "bar", "baz": "bat"})
	if id := tags.ID(); id != "baz\x00foo\x00bat\x00bar" {
		t.Fatalf("unexpected id: %q", id)
	}
}