Пример #1
0
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (interface{}, error) {
	timerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
	if err != nil {
		return nil, err
	}
	hasTimeout := context.Timeout != 0
	var cancellable api.Cancellable
	if hasTimeout {
		cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
	} else {
		cancellable = api.NewCancellable()
	}
	r := context.Registry
	if r == nil {
		r = registry.Default()
	}

	defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
	evaluationContext := function.EvaluationContext{
		API:          context.API,
		FetchLimit:   function.NewFetchCounter(context.FetchLimit),
		MultiBackend: context.Backend,
		Predicate:    cmd.predicate,
		SampleMethod: cmd.context.SampleMethod,
		Timerange:    timerange,
		Cancellable:  cancellable,
		Profiler:     context.Profiler,
		Registry:     r,
	}
	if hasTimeout {
		timeout := time.After(context.Timeout)
		results := make(chan interface{})
		errors := make(chan error)
		go func() {
			result, err := evaluateExpressions(evaluationContext, cmd.expressions)
			if err != nil {
				errors <- err
			} else {
				results <- result
			}
		}()
		select {
		case <-timeout:
			return nil, fmt.Errorf("Timeout while executing the query.") // timeout.
		case result := <-results:
			return result, nil
		case err := <-errors:
			return nil, err
		}
	} else {
		return evaluateExpressions(evaluationContext, cmd.expressions)
	}
}
Пример #2
0
func TestMovingAverage(t *testing.T) {
	fakeAPI := mocks.NewFakeMetricMetadataAPI()
	fakeAPI.AddPairWithoutGraphite(api.TaggedMetric{"series", api.NewTagSet()})

	fakeBackend := movingAverageBackend{}
	timerange, err := api.NewTimerange(1200, 1500, 100)
	if err != nil {
		t.Fatalf(err.Error())
	}

	expression := &functionExpression{
		functionName: "transform.moving_average",
		groupBy:      []string{},
		arguments: []function.Expression{
			&metricFetchExpression{"series", api.TruePredicate},
			durationExpression{"300ms", 300 * time.Millisecond},
		},
	}

	backend := fakeBackend
	result, err := evaluateToSeriesList(expression,
		&function.EvaluationContext{
			MetricMetadataAPI:         fakeAPI,
			TimeseriesStorageAPI:      backend,
			Timerange:                 timerange,
			SampleMethod:              api.SampleMean,
			FetchLimit:                function.NewFetchCounter(1000),
			Registry:                  registry.Default(),
			Cancellable:               api.NewCancellable(),
			OptimizationConfiguration: optimize.NewOptimizationConfiguration(),
		})
	if err != nil {
		t.Errorf(err.Error())
	}

	expected := []float64{4, 3, 11.0 / 3, 5}

	if len(result.Series) != 1 {
		t.Fatalf("expected exactly 1 returned series")
	}
	if len(result.Series[0].Values) != len(expected) {
		t.Fatalf("expected exactly %d values in returned series, but got %d", len(expected), len(result.Series[0].Values))
	}
	const eps = 1e-7
	for i := range expected {
		if math.Abs(result.Series[0].Values[i]-expected[i]) > eps {
			t.Fatalf("expected %+v but got %+v", expected, result.Series[0].Values)
		}
	}
}
func TestMovingAverage(t *testing.T) {
	fakeAPI := mocks.NewFakeApi()
	fakeAPI.AddPair(api.TaggedMetric{"series", api.NewTagSet()}, "series")

	fakeBackend := movingAverageBackend{}
	timerange, err := api.NewTimerange(1200, 1500, 100)
	if err != nil {
		t.Fatalf(err.Error())
	}

	expression := &functionExpression{
		functionName: "transform.moving_average",
		groupBy:      []string{},
		arguments: []function.Expression{
			&metricFetchExpression{"series", api.TruePredicate},
			stringExpression{"300ms"},
		},
	}

	result, err := evaluateToSeriesList(expression,
		function.EvaluationContext{
			API:          fakeAPI,
			MultiBackend: backend.NewSequentialMultiBackend(fakeBackend),
			Timerange:    timerange,
			SampleMethod: api.SampleMean,
			FetchLimit:   function.NewFetchCounter(1000),
			Registry:     registry.Default(),
		})
	if err != nil {
		t.Errorf(err.Error())
	}

	expected := []float64{4, 3, 11.0 / 3, 5}
	if len(result.Series) != 1 {
		t.Fatalf("expected exactly 1 returned series")
	}
	if len(result.Series[0].Values) != len(expected) {
		t.Fatalf("expected exactly %d values in returned series", len(expected))
	}
	const eps = 1e-7
	for i := range expected {
		if math.Abs(result.Series[0].Values[i]-expected[i]) > eps {
			t.Fatalf("expected %+v but got %+v", expected, result.Series[0].Values)
		}
	}
}
Пример #4
0
func Test_ScalarExpression(t *testing.T) {
	timerangeA, err := api.NewTimerange(0, 10, 2)
	if err != nil {
		t.Fatalf("invalid timerange used for testcase")
		return
	}
	for _, test := range []struct {
		expr           scalarExpression
		timerange      api.Timerange
		expectedSeries []api.Timeseries
	}{
		{
			scalarExpression{5},
			timerangeA,
			[]api.Timeseries{
				api.Timeseries{
					Values: []float64{5.0, 5.0, 5.0, 5.0, 5.0, 5.0},
					TagSet: api.NewTagSet(),
				},
			},
		},
	} {
		a := assert.New(t).Contextf("%+v", test)
		result, err := evaluateToSeriesList(test.expr, &function.EvaluationContext{
			TimeseriesStorageAPI: FakeBackend{},
			Timerange:            test.timerange,
			SampleMethod:         api.SampleMean,
			FetchLimit:           function.NewFetchCounter(1000),
			Registry:             registry.Default(),
		})

		if err != nil {
			t.Fatalf("failed to convert number into serieslist")
		}

		a.EqInt(len(result.Series), len(test.expectedSeries))

		for i := 0; i < len(result.Series); i++ {
			a.Eq(result.Series[i].Values, test.expectedSeries[i].Values)
		}
	}
}
Пример #5
0
func Test_evaluateBinaryOperation(t *testing.T) {
	emptyContext := &function.EvaluationContext{
		TimeseriesStorageAPI: FakeBackend{},
		MetricMetadataAPI:    nil,
		Timerange:            api.Timerange{},
		SampleMethod:         api.SampleMean,
		Predicate:            nil,
		FetchLimit:           function.NewFetchCounter(1000),
		Cancellable:          api.NewCancellable(),
	}
	for _, test := range []struct {
		context              *function.EvaluationContext
		functionName         string
		left                 api.SeriesList
		right                api.SeriesList
		evalFunction         func(float64, float64) float64
		expectSuccess        bool
		expectedResultValues [][]float64
	}{
		{
			emptyContext,
			"add",
			api.SeriesList{
				[]api.Timeseries{
					{
						Values: []float64{1, 2, 3},
						TagSet: api.TagSet{},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			api.SeriesList{
				[]api.Timeseries{
					{
						Values: []float64{4, 5, 1},
						TagSet: api.TagSet{},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			func(left, right float64) float64 { return left + right },
			true,
			[][]float64{{5, 7, 4}},
		},
		{
			emptyContext,
			"subtract",
			api.SeriesList{
				[]api.Timeseries{
					{
						Values: []float64{1, 2, 3},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			api.SeriesList{
				[]api.Timeseries{
					{
						Values: []float64{4, 5, 1},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			func(left, right float64) float64 { return left - right },
			true,
			[][]float64{{-3, -3, 2}},
		},
		{
			emptyContext,
			"add",
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{1, 2, 3},
						TagSet: api.TagSet{
							"env":  "production",
							"host": "#1",
						},
					},
					api.Timeseries{
						Values: []float64{7, 7, 7},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#2",
						},
					},
					api.Timeseries{
						Values: []float64{1, 0, 2},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#3",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{5, 5, 5},
						TagSet: api.TagSet{
							"env": "staging",
						},
					},
					api.Timeseries{
						Values: []float64{10, 100, 1000},
						TagSet: api.TagSet{
							"env": "production",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			func(left, right float64) float64 { return left + right },
			true,
			[][]float64{{11, 102, 1003}, {12, 12, 12}, {6, 5, 7}},
		},
		{
			emptyContext,
			"add",
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{1, 2, 3},
						TagSet: api.TagSet{
							"env":  "production",
							"host": "#1",
						},
					},
					api.Timeseries{
						Values: []float64{4, 5, 6},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#2",
						},
					},
					api.Timeseries{
						Values: []float64{7, 8, 9},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#3",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{2, 2, 2},
						TagSet: api.TagSet{
							"env": "staging",
						},
					},
					api.Timeseries{
						Values: []float64{3, 3, 3},
						TagSet: api.TagSet{
							"env": "staging",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			func(left, right float64) float64 { return left * right },
			true,
			[][]float64{{8, 10, 12}, {14, 16, 18}, {12, 15, 18}, {21, 24, 27}},
		},
		{
			emptyContext,
			"add",
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{103, 103, 103},
						TagSet: api.TagSet{
							"env":  "production",
							"host": "#1",
						},
					},
					api.Timeseries{
						Values: []float64{203, 203, 203},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#2",
						},
					},
					api.Timeseries{
						Values: []float64{303, 303, 303},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#3",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{1, 2, 3},
						TagSet: api.TagSet{
							"env": "staging",
						},
					},
					api.Timeseries{
						Values: []float64{3, 0, 3},
						TagSet: api.TagSet{
							"env": "production",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			func(left, right float64) float64 { return left - right },
			true,
			[][]float64{{100, 103, 100}, {202, 201, 200}, {302, 301, 300}},
		},
	} {
		a := assert.New(t).Contextf("%+v", test)

		metricFun := registry.NewOperator(test.functionName, test.evalFunction)

		value, err := metricFun.Evaluate(test.context, []function.Expression{&LiteralSeriesExpression{test.left}, &LiteralSeriesExpression{test.right}}, []string{}, false)
		if err != nil {
			a.EqBool(err == nil, test.expectSuccess)
			continue
		}

		result, err := value.ToSeriesList(test.context.Timerange)
		if err != nil {
			a.EqBool(err == nil, test.expectSuccess)
			continue
		}

		// Our expected list should be the same length as the actual one:
		a.EqInt(len(result.Series), len(test.expectedResultValues))

		// The "expected" results are only true up to permutation (since guessing the order they'll come out of `join()` is hard)
		// Provided that they're all unique then we just need to check that every member that's expected can be found
		// This is a bit more annoying:

		equal := func(left, right []float64) bool {
			if len(left) != len(right) {
				return false
			}
			for i := range left {
				if left[i] != right[i] {
					return false
				}
			}
			return true
		}

		for _, expectedMember := range test.expectedResultValues {
			found := false
			// check that expectedMember is inside our result list
			// look for it inside result.Series
			for _, resultMember := range result.Series {
				if equal(resultMember.Values, expectedMember) {
					found = true
					break
				}
			}
			if !found {
				t.Fatalf("got %+v for test %+v", result, test)
			}
		}

	}
}
Пример #6
0
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (interface{}, error) {
	timerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
	if err != nil {
		return nil, err
	}
	slotLimit := context.SlotLimit
	defaultLimit := 1000
	if slotLimit == 0 {
		slotLimit = defaultLimit // the default limit
	}

	smallestResolution := timerange.Duration() / time.Duration(slotLimit-2)
	// ((end + res/2) - (start - res/2)) / res + 1 <= slots // make adjustments for a snap that moves the endpoints
	// (do some algebra)
	// (end - start + res) + res <= slots * res
	// end - start <= res * (slots - 2)
	// so
	// res >= (end - start) / (slots - 2)

	// Update the timerange by applying the insights of the storage API:
	chosenResolution := context.TimeseriesStorageAPI.ChooseResolution(timerange, smallestResolution)

	chosenTimerange, err := api.NewSnappedTimerange(timerange.Start(), timerange.End(), int64(chosenResolution/time.Millisecond))
	if err != nil {
		return nil, err
	}

	if chosenTimerange.Slots() > slotLimit {
		return nil, function.NewLimitError(
			"Requested number of data points exceeds the configured limit",
			chosenTimerange.Slots(), slotLimit)
	}
	hasTimeout := context.Timeout != 0
	var cancellable api.Cancellable
	if hasTimeout {
		cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
	} else {
		cancellable = api.NewCancellable()
	}
	r := context.Registry
	if r == nil {
		r = registry.Default()
	}

	defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
	evaluationContext := function.EvaluationContext{
		MetricMetadataAPI:         context.MetricMetadataAPI,
		FetchLimit:                function.NewFetchCounter(context.FetchLimit),
		TimeseriesStorageAPI:      context.TimeseriesStorageAPI,
		Predicate:                 cmd.predicate,
		SampleMethod:              cmd.context.SampleMethod,
		Timerange:                 timerange,
		Cancellable:               cancellable,
		Registry:                  r,
		Profiler:                  context.Profiler,
		OptimizationConfiguration: context.OptimizationConfiguration,
	}

	if hasTimeout {
		timeout := time.After(context.Timeout)
		results := make(chan interface{})
		errors := make(chan error)
		go func() {
			result, err := function.EvaluateMany(evaluationContext, cmd.expressions)
			if err != nil {
				errors <- err
			} else {
				results <- result
			}
		}()
		select {
		case <-timeout:
			return nil, function.NewLimitError("Timeout while executing the query.",
				context.Timeout, context.Timeout)
		case result := <-results:
			return result, nil
		case err := <-errors:
			return nil, err
		}
	} else {
		values, err := function.EvaluateMany(evaluationContext, cmd.expressions)
		if err != nil {
			return nil, err
		}
		lists := make([]api.SeriesList, len(values))
		for i := range values {
			lists[i], err = values[i].ToSeriesList(evaluationContext.Timerange)
			if err != nil {
				return nil, err
			}
		}
		return lists, nil
	}
}
Пример #7
0
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (interface{}, error) {
	timerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
	if err != nil {
		return nil, err
	}
	slotLimit := context.SlotLimit
	defaultLimit := 1000
	if slotLimit == 0 {
		slotLimit = defaultLimit // the default limit
	}
	if timerange.Slots() > slotLimit {
		return nil, function.NewLimitError(
			"Requested number of data points exceeds the configured limit",
			timerange.Slots(), slotLimit)
	}
	hasTimeout := context.Timeout != 0
	var cancellable api.Cancellable
	if hasTimeout {
		cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
	} else {
		cancellable = api.NewCancellable()
	}
	r := context.Registry
	if r == nil {
		r = registry.Default()
	}

	defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
	evaluationContext := function.EvaluationContext{
		API:          context.API,
		FetchLimit:   function.NewFetchCounter(context.FetchLimit),
		MultiBackend: context.Backend,
		Predicate:    cmd.predicate,
		SampleMethod: cmd.context.SampleMethod,
		Timerange:    timerange,
		Cancellable:  cancellable,
		Profiler:     context.Profiler,
		Registry:     r,
	}
	if hasTimeout {
		timeout := time.After(context.Timeout)
		results := make(chan interface{})
		errors := make(chan error)
		go func() {
			result, err := function.EvaluateMany(evaluationContext, cmd.expressions)
			if err != nil {
				errors <- err
			} else {
				results <- result
			}
		}()
		select {
		case <-timeout:
			return nil, function.NewLimitError("Timeout while executing the query.",
				context.Timeout, context.Timeout)
		case result := <-results:
			return result, nil
		case err := <-errors:
			return nil, err
		}
	} else {
		values, err := function.EvaluateMany(evaluationContext, cmd.expressions)
		if err != nil {
			return nil, err
		}
		lists := make([]api.SeriesList, len(values))
		for i := range values {
			lists[i], err = values[i].ToSeriesList(evaluationContext.Timerange)
			if err != nil {
				return nil, err
			}
		}
		return lists, nil
	}
}
Пример #8
0
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (CommandResult, error) {
	userTimerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
	if err != nil {
		return CommandResult{}, err
	}
	slotLimit := context.SlotLimit
	defaultLimit := 1000
	if slotLimit == 0 {
		slotLimit = defaultLimit // the default limit
	}

	smallestResolution := userTimerange.Duration() / time.Duration(slotLimit-2)
	// ((end + res/2) - (start - res/2)) / res + 1 <= slots // make adjustments for a snap that moves the endpoints
	// (do some algebra)
	// (end - start + res) + res <= slots * res
	// end - start <= res * (slots - 2)
	// so
	// res >= (end - start) / (slots - 2)

	// Update the timerange by applying the insights of the storage API:
	chosenResolution := context.TimeseriesStorageAPI.ChooseResolution(userTimerange, smallestResolution)

	chosenTimerange, err := api.NewSnappedTimerange(userTimerange.Start(), userTimerange.End(), int64(chosenResolution/time.Millisecond))
	if err != nil {
		return CommandResult{}, err
	}

	if chosenTimerange.Slots() > slotLimit {
		return CommandResult{}, function.NewLimitError(
			"Requested number of data points exceeds the configured limit",
			chosenTimerange.Slots(), slotLimit)
	}
	hasTimeout := context.Timeout != 0
	var cancellable api.Cancellable
	if hasTimeout {
		cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
	} else {
		cancellable = api.NewCancellable()
	}
	r := context.Registry
	if r == nil {
		r = registry.Default()
	}

	defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
	evaluationContext := function.EvaluationContext{
		MetricMetadataAPI:         context.MetricMetadataAPI,
		FetchLimit:                function.NewFetchCounter(context.FetchLimit),
		TimeseriesStorageAPI:      context.TimeseriesStorageAPI,
		Predicate:                 cmd.predicate,
		SampleMethod:              cmd.context.SampleMethod,
		Timerange:                 chosenTimerange,
		Cancellable:               cancellable,
		Registry:                  r,
		Profiler:                  context.Profiler,
		OptimizationConfiguration: context.OptimizationConfiguration,
		EvaluationNotes:           []string{},
		UserSpecifiableConfig:     context.UserSpecifiableConfig,
	}

	timeout := (<-chan time.Time)(nil)
	if hasTimeout {
		// A nil channel will just block forever
		timeout = time.After(context.Timeout)
	}

	results := make(chan []function.Value, 1)
	errors := make(chan error, 1)
	// Goroutines are never garbage collected, so we need to provide capacity so that the send always succeeds.
	go func() {
		// Evaluate the result, and send it along the goroutines.
		result, err := function.EvaluateMany(&evaluationContext, cmd.expressions)
		if err != nil {
			errors <- err
			return
		}
		results <- result
	}()
	select {
	case <-timeout:
		return CommandResult{}, function.NewLimitError("Timeout while executing the query.",
			context.Timeout, context.Timeout)
	case err := <-errors:
		return CommandResult{}, err
	case result := <-results:
		lists := make([]api.SeriesList, len(result))
		for i := range result {
			lists[i], err = result[i].ToSeriesList(evaluationContext.Timerange)
			if err != nil {
				return CommandResult{}, err
			}
		}
		description := map[string][]string{}
		for _, list := range lists {
			for _, series := range list.Series {
				for key, value := range series.TagSet {
					description[key] = append(description[key], value)
				}
			}
		}
		for key, values := range description {
			natural_sort.Sort(values)
			filtered := []string{}
			for i := range values {
				if i == 0 || values[i-1] != values[i] {
					filtered = append(filtered, values[i])
				}
			}
			description[key] = filtered
		}
		return CommandResult{
			Body: lists,
			Metadata: map[string]interface{}{
				"description": description,
				"notes":       evaluationContext.EvaluationNotes,
			},
		}, nil
	}
}