Ejemplo n.º 1
0
func newSuite() Suite {
	b := fakeBackend{make(chan struct{}, 10)}
	suite := Suite{
		backend:      b,
		multiBackend: NewParallelMultiBackend(b, 4),
		cancellable:  api.NewCancellable(),
	}
	suite.waitGroup.Add(1)
	return suite
}
Ejemplo n.º 2
0
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (interface{}, error) {
	timerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
	if err != nil {
		return nil, err
	}
	hasTimeout := context.Timeout != 0
	var cancellable api.Cancellable
	if hasTimeout {
		cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
	} else {
		cancellable = api.NewCancellable()
	}
	r := context.Registry
	if r == nil {
		r = registry.Default()
	}

	defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
	evaluationContext := function.EvaluationContext{
		API:          context.API,
		FetchLimit:   function.NewFetchCounter(context.FetchLimit),
		MultiBackend: context.Backend,
		Predicate:    cmd.predicate,
		SampleMethod: cmd.context.SampleMethod,
		Timerange:    timerange,
		Cancellable:  cancellable,
		Profiler:     context.Profiler,
		Registry:     r,
	}
	if hasTimeout {
		timeout := time.After(context.Timeout)
		results := make(chan interface{})
		errors := make(chan error)
		go func() {
			result, err := evaluateExpressions(evaluationContext, cmd.expressions)
			if err != nil {
				errors <- err
			} else {
				results <- result
			}
		}()
		select {
		case <-timeout:
			return nil, fmt.Errorf("Timeout while executing the query.") // timeout.
		case result := <-results:
			return result, nil
		case err := <-errors:
			return nil, err
		}
	} else {
		return evaluateExpressions(evaluationContext, cmd.expressions)
	}
}
Ejemplo n.º 3
0
func TestMovingAverage(t *testing.T) {
	fakeAPI := mocks.NewFakeMetricMetadataAPI()
	fakeAPI.AddPairWithoutGraphite(api.TaggedMetric{"series", api.NewTagSet()})

	fakeBackend := movingAverageBackend{}
	timerange, err := api.NewTimerange(1200, 1500, 100)
	if err != nil {
		t.Fatalf(err.Error())
	}

	expression := &functionExpression{
		functionName: "transform.moving_average",
		groupBy:      []string{},
		arguments: []function.Expression{
			&metricFetchExpression{"series", api.TruePredicate},
			durationExpression{"300ms", 300 * time.Millisecond},
		},
	}

	backend := fakeBackend
	result, err := evaluateToSeriesList(expression,
		&function.EvaluationContext{
			MetricMetadataAPI:         fakeAPI,
			TimeseriesStorageAPI:      backend,
			Timerange:                 timerange,
			SampleMethod:              api.SampleMean,
			FetchLimit:                function.NewFetchCounter(1000),
			Registry:                  registry.Default(),
			Cancellable:               api.NewCancellable(),
			OptimizationConfiguration: optimize.NewOptimizationConfiguration(),
		})
	if err != nil {
		t.Errorf(err.Error())
	}

	expected := []float64{4, 3, 11.0 / 3, 5}

	if len(result.Series) != 1 {
		t.Fatalf("expected exactly 1 returned series")
	}
	if len(result.Series[0].Values) != len(expected) {
		t.Fatalf("expected exactly %d values in returned series, but got %d", len(expected), len(result.Series[0].Values))
	}
	const eps = 1e-7
	for i := range expected {
		if math.Abs(result.Series[0].Values[i]-expected[i]) > eps {
			t.Fatalf("expected %+v but got %+v", expected, result.Series[0].Values)
		}
	}
}
Ejemplo n.º 4
0
func Test_evaluateBinaryOperation(t *testing.T) {
	emptyContext := &function.EvaluationContext{
		TimeseriesStorageAPI: FakeBackend{},
		MetricMetadataAPI:    nil,
		Timerange:            api.Timerange{},
		SampleMethod:         api.SampleMean,
		Predicate:            nil,
		FetchLimit:           function.NewFetchCounter(1000),
		Cancellable:          api.NewCancellable(),
	}
	for _, test := range []struct {
		context              *function.EvaluationContext
		functionName         string
		left                 api.SeriesList
		right                api.SeriesList
		evalFunction         func(float64, float64) float64
		expectSuccess        bool
		expectedResultValues [][]float64
	}{
		{
			emptyContext,
			"add",
			api.SeriesList{
				[]api.Timeseries{
					{
						Values: []float64{1, 2, 3},
						TagSet: api.TagSet{},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			api.SeriesList{
				[]api.Timeseries{
					{
						Values: []float64{4, 5, 1},
						TagSet: api.TagSet{},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			func(left, right float64) float64 { return left + right },
			true,
			[][]float64{{5, 7, 4}},
		},
		{
			emptyContext,
			"subtract",
			api.SeriesList{
				[]api.Timeseries{
					{
						Values: []float64{1, 2, 3},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			api.SeriesList{
				[]api.Timeseries{
					{
						Values: []float64{4, 5, 1},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			func(left, right float64) float64 { return left - right },
			true,
			[][]float64{{-3, -3, 2}},
		},
		{
			emptyContext,
			"add",
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{1, 2, 3},
						TagSet: api.TagSet{
							"env":  "production",
							"host": "#1",
						},
					},
					api.Timeseries{
						Values: []float64{7, 7, 7},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#2",
						},
					},
					api.Timeseries{
						Values: []float64{1, 0, 2},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#3",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{5, 5, 5},
						TagSet: api.TagSet{
							"env": "staging",
						},
					},
					api.Timeseries{
						Values: []float64{10, 100, 1000},
						TagSet: api.TagSet{
							"env": "production",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			func(left, right float64) float64 { return left + right },
			true,
			[][]float64{{11, 102, 1003}, {12, 12, 12}, {6, 5, 7}},
		},
		{
			emptyContext,
			"add",
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{1, 2, 3},
						TagSet: api.TagSet{
							"env":  "production",
							"host": "#1",
						},
					},
					api.Timeseries{
						Values: []float64{4, 5, 6},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#2",
						},
					},
					api.Timeseries{
						Values: []float64{7, 8, 9},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#3",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{2, 2, 2},
						TagSet: api.TagSet{
							"env": "staging",
						},
					},
					api.Timeseries{
						Values: []float64{3, 3, 3},
						TagSet: api.TagSet{
							"env": "staging",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			func(left, right float64) float64 { return left * right },
			true,
			[][]float64{{8, 10, 12}, {14, 16, 18}, {12, 15, 18}, {21, 24, 27}},
		},
		{
			emptyContext,
			"add",
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{103, 103, 103},
						TagSet: api.TagSet{
							"env":  "production",
							"host": "#1",
						},
					},
					api.Timeseries{
						Values: []float64{203, 203, 203},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#2",
						},
					},
					api.Timeseries{
						Values: []float64{303, 303, 303},
						TagSet: api.TagSet{
							"env":  "staging",
							"host": "#3",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			api.SeriesList{
				[]api.Timeseries{
					api.Timeseries{
						Values: []float64{1, 2, 3},
						TagSet: api.TagSet{
							"env": "staging",
						},
					},
					api.Timeseries{
						Values: []float64{3, 0, 3},
						TagSet: api.TagSet{
							"env": "production",
						},
					},
				},
				api.Timerange{},
				"",
				"",
			},
			func(left, right float64) float64 { return left - right },
			true,
			[][]float64{{100, 103, 100}, {202, 201, 200}, {302, 301, 300}},
		},
	} {
		a := assert.New(t).Contextf("%+v", test)

		metricFun := registry.NewOperator(test.functionName, test.evalFunction)

		value, err := metricFun.Evaluate(test.context, []function.Expression{&LiteralSeriesExpression{test.left}, &LiteralSeriesExpression{test.right}}, []string{}, false)
		if err != nil {
			a.EqBool(err == nil, test.expectSuccess)
			continue
		}

		result, err := value.ToSeriesList(test.context.Timerange)
		if err != nil {
			a.EqBool(err == nil, test.expectSuccess)
			continue
		}

		// Our expected list should be the same length as the actual one:
		a.EqInt(len(result.Series), len(test.expectedResultValues))

		// The "expected" results are only true up to permutation (since guessing the order they'll come out of `join()` is hard)
		// Provided that they're all unique then we just need to check that every member that's expected can be found
		// This is a bit more annoying:

		equal := func(left, right []float64) bool {
			if len(left) != len(right) {
				return false
			}
			for i := range left {
				if left[i] != right[i] {
					return false
				}
			}
			return true
		}

		for _, expectedMember := range test.expectedResultValues {
			found := false
			// check that expectedMember is inside our result list
			// look for it inside result.Series
			for _, resultMember := range result.Series {
				if equal(resultMember.Values, expectedMember) {
					found = true
					break
				}
			}
			if !found {
				t.Fatalf("got %+v for test %+v", result, test)
			}
		}

	}
}
Ejemplo n.º 5
0
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (interface{}, error) {
	timerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
	if err != nil {
		return nil, err
	}
	slotLimit := context.SlotLimit
	defaultLimit := 1000
	if slotLimit == 0 {
		slotLimit = defaultLimit // the default limit
	}

	smallestResolution := timerange.Duration() / time.Duration(slotLimit-2)
	// ((end + res/2) - (start - res/2)) / res + 1 <= slots // make adjustments for a snap that moves the endpoints
	// (do some algebra)
	// (end - start + res) + res <= slots * res
	// end - start <= res * (slots - 2)
	// so
	// res >= (end - start) / (slots - 2)

	// Update the timerange by applying the insights of the storage API:
	chosenResolution := context.TimeseriesStorageAPI.ChooseResolution(timerange, smallestResolution)

	chosenTimerange, err := api.NewSnappedTimerange(timerange.Start(), timerange.End(), int64(chosenResolution/time.Millisecond))
	if err != nil {
		return nil, err
	}

	if chosenTimerange.Slots() > slotLimit {
		return nil, function.NewLimitError(
			"Requested number of data points exceeds the configured limit",
			chosenTimerange.Slots(), slotLimit)
	}
	hasTimeout := context.Timeout != 0
	var cancellable api.Cancellable
	if hasTimeout {
		cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
	} else {
		cancellable = api.NewCancellable()
	}
	r := context.Registry
	if r == nil {
		r = registry.Default()
	}

	defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
	evaluationContext := function.EvaluationContext{
		MetricMetadataAPI:         context.MetricMetadataAPI,
		FetchLimit:                function.NewFetchCounter(context.FetchLimit),
		TimeseriesStorageAPI:      context.TimeseriesStorageAPI,
		Predicate:                 cmd.predicate,
		SampleMethod:              cmd.context.SampleMethod,
		Timerange:                 timerange,
		Cancellable:               cancellable,
		Registry:                  r,
		Profiler:                  context.Profiler,
		OptimizationConfiguration: context.OptimizationConfiguration,
	}

	if hasTimeout {
		timeout := time.After(context.Timeout)
		results := make(chan interface{})
		errors := make(chan error)
		go func() {
			result, err := function.EvaluateMany(evaluationContext, cmd.expressions)
			if err != nil {
				errors <- err
			} else {
				results <- result
			}
		}()
		select {
		case <-timeout:
			return nil, function.NewLimitError("Timeout while executing the query.",
				context.Timeout, context.Timeout)
		case result := <-results:
			return result, nil
		case err := <-errors:
			return nil, err
		}
	} else {
		values, err := function.EvaluateMany(evaluationContext, cmd.expressions)
		if err != nil {
			return nil, err
		}
		lists := make([]api.SeriesList, len(values))
		for i := range values {
			lists[i], err = values[i].ToSeriesList(evaluationContext.Timerange)
			if err != nil {
				return nil, err
			}
		}
		return lists, nil
	}
}
Ejemplo n.º 6
0
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (interface{}, error) {
	timerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
	if err != nil {
		return nil, err
	}
	slotLimit := context.SlotLimit
	defaultLimit := 1000
	if slotLimit == 0 {
		slotLimit = defaultLimit // the default limit
	}
	if timerange.Slots() > slotLimit {
		return nil, function.NewLimitError(
			"Requested number of data points exceeds the configured limit",
			timerange.Slots(), slotLimit)
	}
	hasTimeout := context.Timeout != 0
	var cancellable api.Cancellable
	if hasTimeout {
		cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
	} else {
		cancellable = api.NewCancellable()
	}
	r := context.Registry
	if r == nil {
		r = registry.Default()
	}

	defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
	evaluationContext := function.EvaluationContext{
		API:          context.API,
		FetchLimit:   function.NewFetchCounter(context.FetchLimit),
		MultiBackend: context.Backend,
		Predicate:    cmd.predicate,
		SampleMethod: cmd.context.SampleMethod,
		Timerange:    timerange,
		Cancellable:  cancellable,
		Profiler:     context.Profiler,
		Registry:     r,
	}
	if hasTimeout {
		timeout := time.After(context.Timeout)
		results := make(chan interface{})
		errors := make(chan error)
		go func() {
			result, err := function.EvaluateMany(evaluationContext, cmd.expressions)
			if err != nil {
				errors <- err
			} else {
				results <- result
			}
		}()
		select {
		case <-timeout:
			return nil, function.NewLimitError("Timeout while executing the query.",
				context.Timeout, context.Timeout)
		case result := <-results:
			return result, nil
		case err := <-errors:
			return nil, err
		}
	} else {
		values, err := function.EvaluateMany(evaluationContext, cmd.expressions)
		if err != nil {
			return nil, err
		}
		lists := make([]api.SeriesList, len(values))
		for i := range values {
			lists[i], err = values[i].ToSeriesList(evaluationContext.Timerange)
			if err != nil {
				return nil, err
			}
		}
		return lists, nil
	}
}
Ejemplo n.º 7
0
func Test_Blueflood(t *testing.T) {
	timerange, err := api.NewTimerange(12000, 13000, 1000)
	if err != nil {
		t.Fatalf("invalid testcase timerange")
		return
	}
	defaultClientConfig := Config{
		"https://blueflood.url",
		"square",
		make(map[string]int64),
		time.Millisecond,
		0,
	}
	// Not really MIN1440, but that's what default TTLs will get with the Timerange we use
	defaultQueryUrl := "https://blueflood.url/v2.0/square/views/some.key.graphite?from=12000&resolution=MIN1440&select=numPoints%2Caverage&to=14000"

	for _, test := range []struct {
		name               string
		metricMap          map[api.GraphiteMetric]api.TaggedMetric
		queryMetric        api.TaggedMetric
		sampleMethod       api.SampleMethod
		timerange          api.Timerange
		clientConfig       Config
		queryUrl           string
		queryResponse      string
		queryResponseCode  int
		queryDelay         time.Duration
		expectedErrorCode  api.BackendErrorCode
		expectedSeriesList api.Timeseries
	}{
		{
			name: "Success case",
			metricMap: map[api.GraphiteMetric]api.TaggedMetric{
				api.GraphiteMetric("some.key.graphite"): api.TaggedMetric{
					MetricKey: api.MetricKey("some.key"),
					TagSet:    api.ParseTagSet("tag=value"),
				},
			},
			queryMetric: api.TaggedMetric{
				MetricKey: api.MetricKey("some.key"),
				TagSet:    api.ParseTagSet("tag=value"),
			},
			sampleMethod: api.SampleMean,
			timerange:    timerange,
			queryUrl:     defaultQueryUrl,
			clientConfig: defaultClientConfig,
			queryResponse: `{
        "unit": "unknown", 
        "values": [
          {
            "numPoints": 1,
            "timestamp": 12000,
            "average": 5
          },
          {
            "numPoints": 1,
            "timestamp": 13000,
            "average": 3
          }
        ],
        "metadata": {
          "limit": null,
          "next_href": null,
          "count": 2,
          "marker": null
        }
      }`,
			expectedSeriesList: api.Timeseries{
				Values: []float64{5, 3},
				TagSet: api.ParseTagSet("tag=value"),
			},
		},
		{
			name: "Failure case - invalid JSON",
			metricMap: map[api.GraphiteMetric]api.TaggedMetric{
				api.GraphiteMetric("some.key.graphite"): api.TaggedMetric{
					MetricKey: api.MetricKey("some.key"),
					TagSet:    api.ParseTagSet("tag=value"),
				},
			},
			queryMetric: api.TaggedMetric{
				MetricKey: api.MetricKey("some.key"),
				TagSet:    api.ParseTagSet("tag=value"),
			},
			sampleMethod:      api.SampleMean,
			timerange:         timerange,
			clientConfig:      defaultClientConfig,
			queryUrl:          defaultQueryUrl,
			queryResponse:     `{invalid}`,
			expectedErrorCode: api.FetchIOError,
		},
		{
			name: "Failure case - HTTP error",
			metricMap: map[api.GraphiteMetric]api.TaggedMetric{
				api.GraphiteMetric("some.key.graphite"): api.TaggedMetric{
					MetricKey: api.MetricKey("some.key"),
					TagSet:    api.ParseTagSet("tag=value"),
				},
			},
			queryMetric: api.TaggedMetric{
				MetricKey: api.MetricKey("some.key"),
				TagSet:    api.ParseTagSet("tag=value"),
			},
			sampleMethod:      api.SampleMean,
			timerange:         timerange,
			clientConfig:      defaultClientConfig,
			queryUrl:          defaultQueryUrl,
			queryResponse:     `{}`,
			queryResponseCode: 400,
			expectedErrorCode: api.FetchIOError,
		},
		{
			name: "Failure case - timeout",
			metricMap: map[api.GraphiteMetric]api.TaggedMetric{
				api.GraphiteMetric("some.key.graphite"): api.TaggedMetric{
					MetricKey: api.MetricKey("some.key"),
					TagSet:    api.ParseTagSet("tag=value"),
				},
			},
			queryMetric: api.TaggedMetric{
				MetricKey: api.MetricKey("some.key"),
				TagSet:    api.ParseTagSet("tag=value"),
			},
			sampleMethod:      api.SampleMean,
			timerange:         timerange,
			clientConfig:      defaultClientConfig,
			queryUrl:          defaultQueryUrl,
			queryResponse:     `{}`,
			queryDelay:        1 * time.Second,
			expectedErrorCode: api.FetchTimeoutError,
		},
	} {
		a := assert.New(t).Contextf("%s", test.name)

		fakeApi := mocks.NewFakeApi()
		for k, v := range test.metricMap {
			fakeApi.AddPair(v, k)
		}

		fakeHttpClient := mocks.NewFakeHttpClient()
		code := test.queryResponseCode
		if code == 0 {
			code = http.StatusOK
		}
		fakeHttpClient.SetResponse(test.queryUrl, mocks.Response{test.queryResponse, test.queryDelay, code})

		b := NewBlueflood(test.clientConfig).(*blueflood)
		b.client = fakeHttpClient

		seriesList, err := b.FetchSingleSeries(api.FetchSeriesRequest{
			Metric:       test.queryMetric,
			SampleMethod: test.sampleMethod,
			Timerange:    test.timerange,
			API:          fakeApi,
			Cancellable:  api.NewCancellable(),
		})

		if test.expectedErrorCode != 0 {
			if err == nil {
				a.Errorf("Expected error, but was successful.")
				continue
			}
			berr, ok := err.(api.BackendError)
			if !ok {
				a.Errorf("Failed to cast error to BackendError")
				continue
			}
			a.Eq(berr.Code, test.expectedErrorCode)
		} else {
			if err != nil {
				a.CheckError(err)
				continue
			}
			a.Eq(seriesList, test.expectedSeriesList)
		}
	}
}
Ejemplo n.º 8
0
func TestFullResolutionDataFilling(t *testing.T) {
	// The queries have to be relative to "now"
	defaultClientConfig := Config{
		"https://blueflood.url",
		"square",
		make(map[string]int64),
		time.Millisecond,
		14400,
	}

	baseTime := 1438734300000

	regularQueryURL := fmt.Sprintf(
		"https://blueflood.url/v2.0/square/views/some.key.value?from=%d&resolution=MIN5&select=numPoints%%2Caverage&to=%d",
		baseTime-300*1000*10, // 50 minutes ago
		baseTime-300*1000*3,  // 15 minutes ago
	)

	regularResponse := fmt.Sprintf(`{
	  "unit": "unknown",
	  "values": [
	    {
	      "numPoints": 28,
	      "timestamp": %d,
	      "average": 100
	    },
	    {
	      "numPoints": 29,
	      "timestamp": %d,
	      "average": 142
	    },
	    {
	      "numPoints": 27,
	      "timestamp": %d,
	      "average": 138
	    },
	    {
	      "numPoints": 28,
	      "timestamp": %d,
	      "average": 182
	    }
	  ],
	  "metadata": {
	    "limit": null,
	    "next_href": null,
	    "count": 4,
	    "marker": null
	  }
	}`,
		baseTime-300*1000*10, // 50 minutes ago
		baseTime-300*1000*9,  // 45 minutes ago
		baseTime-300*1000*8,  // 40 minutes ago
		baseTime-300*1000*7,  // 35 minutes ago
	)

	fullResolutionQueryURL := fmt.Sprintf(
		"https://blueflood.url/v2.0/square/views/some.key.value?from=%d&resolution=FULL&select=numPoints%%2Caverage&to=%d",
		baseTime-300*1000*10, // 50 minutes ago
		baseTime-300*1000*3,  // 15 minutes ago
	)
	fullResolutionResponse := fmt.Sprintf(`{
	  "unit": "unknown",
	  "values": [
	    {
	      "numPoints": 28,
	      "timestamp": %d,
	      "average": 13
	    },
	    {
	      "numPoints": 29,
	      "timestamp": %d,
	      "average": 16
	    },
	    {
	      "numPoints": 27,
	      "timestamp": %d,
	      "average": 19
	    },
	    {
	      "numPoints": 28,
	      "timestamp": %d,
	      "average": 27
	    }
	  ],
	  "metadata": {
	    "limit": null,
	    "next_href": null,
	    "count": 4,
	    "marker": null
	  }
	}`,
		baseTime-300*1000*6,      // 30m ago
		baseTime-300*1000*5+17,   // 25m ago with random shuffling
		baseTime-300*1000*4+2821, // 20m ago with random shuffling
		baseTime-300*1000*3,      // 15m ago
	)

	fakeHttpClient := mocks.NewFakeHttpClient()
	fakeHttpClient.SetResponse(regularQueryURL, mocks.Response{regularResponse, 0, http.StatusOK})
	fakeHttpClient.SetResponse(fullResolutionQueryURL, mocks.Response{fullResolutionResponse, 0, http.StatusOK})

	fakeApi := mocks.NewFakeApi()
	fakeApi.AddPair(
		api.TaggedMetric{
			MetricKey: api.MetricKey("some.key"),
			TagSet:    api.ParseTagSet("tag=value"),
		},
		api.GraphiteMetric("some.key.value"),
	)

	b := NewBlueflood(defaultClientConfig).(*blueflood)
	b.client = fakeHttpClient

	queryTimerange, err := api.NewSnappedTimerange(
		int64(baseTime)-300*1000*10, // 50 minutes ago
		int64(baseTime)-300*1000*4,  // 20 minutes ago
		300*1000,                    // 5 minute resolution
	)
	if err != nil {
		t.Fatalf("timerange error: %s", err.Error())
	}

	seriesList, err := b.FetchSingleSeries(api.FetchSeriesRequest{
		Metric: api.TaggedMetric{
			MetricKey: api.MetricKey("some.key"),
			TagSet:    api.ParseTagSet("tag=value"),
		},
		SampleMethod: api.SampleMean,
		Timerange:    queryTimerange,
		API:          fakeApi,
		Cancellable:  api.NewCancellable(),
	})
	if err != nil {
		t.Fatalf("Expected success, but got error: %s", err.Error())
	}
	expected := []float64{100, 142, 138, 182, 13, 16, 19}
	if len(seriesList.Values) != len(expected) {
		t.Fatalf("Expected %+v but got %+v", expected, seriesList)
	}
	for i, expect := range expected {
		if seriesList.Values[i] != expect {
			t.Fatalf("Expected %+v but got %+v", expected, seriesList)
		}
	}
}
Ejemplo n.º 9
0
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (CommandResult, error) {
	userTimerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
	if err != nil {
		return CommandResult{}, err
	}
	slotLimit := context.SlotLimit
	defaultLimit := 1000
	if slotLimit == 0 {
		slotLimit = defaultLimit // the default limit
	}

	smallestResolution := userTimerange.Duration() / time.Duration(slotLimit-2)
	// ((end + res/2) - (start - res/2)) / res + 1 <= slots // make adjustments for a snap that moves the endpoints
	// (do some algebra)
	// (end - start + res) + res <= slots * res
	// end - start <= res * (slots - 2)
	// so
	// res >= (end - start) / (slots - 2)

	// Update the timerange by applying the insights of the storage API:
	chosenResolution := context.TimeseriesStorageAPI.ChooseResolution(userTimerange, smallestResolution)

	chosenTimerange, err := api.NewSnappedTimerange(userTimerange.Start(), userTimerange.End(), int64(chosenResolution/time.Millisecond))
	if err != nil {
		return CommandResult{}, err
	}

	if chosenTimerange.Slots() > slotLimit {
		return CommandResult{}, function.NewLimitError(
			"Requested number of data points exceeds the configured limit",
			chosenTimerange.Slots(), slotLimit)
	}
	hasTimeout := context.Timeout != 0
	var cancellable api.Cancellable
	if hasTimeout {
		cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
	} else {
		cancellable = api.NewCancellable()
	}
	r := context.Registry
	if r == nil {
		r = registry.Default()
	}

	defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
	evaluationContext := function.EvaluationContext{
		MetricMetadataAPI:         context.MetricMetadataAPI,
		FetchLimit:                function.NewFetchCounter(context.FetchLimit),
		TimeseriesStorageAPI:      context.TimeseriesStorageAPI,
		Predicate:                 cmd.predicate,
		SampleMethod:              cmd.context.SampleMethod,
		Timerange:                 chosenTimerange,
		Cancellable:               cancellable,
		Registry:                  r,
		Profiler:                  context.Profiler,
		OptimizationConfiguration: context.OptimizationConfiguration,
		EvaluationNotes:           []string{},
		UserSpecifiableConfig:     context.UserSpecifiableConfig,
	}

	timeout := (<-chan time.Time)(nil)
	if hasTimeout {
		// A nil channel will just block forever
		timeout = time.After(context.Timeout)
	}

	results := make(chan []function.Value, 1)
	errors := make(chan error, 1)
	// Goroutines are never garbage collected, so we need to provide capacity so that the send always succeeds.
	go func() {
		// Evaluate the result, and send it along the goroutines.
		result, err := function.EvaluateMany(&evaluationContext, cmd.expressions)
		if err != nil {
			errors <- err
			return
		}
		results <- result
	}()
	select {
	case <-timeout:
		return CommandResult{}, function.NewLimitError("Timeout while executing the query.",
			context.Timeout, context.Timeout)
	case err := <-errors:
		return CommandResult{}, err
	case result := <-results:
		lists := make([]api.SeriesList, len(result))
		for i := range result {
			lists[i], err = result[i].ToSeriesList(evaluationContext.Timerange)
			if err != nil {
				return CommandResult{}, err
			}
		}
		description := map[string][]string{}
		for _, list := range lists {
			for _, series := range list.Series {
				for key, value := range series.TagSet {
					description[key] = append(description[key], value)
				}
			}
		}
		for key, values := range description {
			natural_sort.Sort(values)
			filtered := []string{}
			for i := range values {
				if i == 0 || values[i-1] != values[i] {
					filtered = append(filtered, values[i])
				}
			}
			description[key] = filtered
		}
		return CommandResult{
			Body: lists,
			Metadata: map[string]interface{}{
				"description": description,
				"notes":       evaluationContext.EvaluationNotes,
			},
		}, nil
	}
}
Ejemplo n.º 10
0
func TestFullResolutionDataFilling(t *testing.T) {
	graphite := mocks.FakeGraphiteConverter{
		MetricMap: map[util.GraphiteMetric]api.TaggedMetric{
			util.GraphiteMetric("some.key.value"): api.TaggedMetric{
				MetricKey: api.MetricKey("some.key"),
				TagSet:    api.ParseTagSet("tag=value"),
			},
		},
	}

	fakeApi := mocks.NewFakeMetricMetadataAPI()
	fakeApi.AddPair(
		api.TaggedMetric{
			MetricKey: api.MetricKey("some.key"),
			TagSet:    api.ParseTagSet("tag=value"),
		},
		util.GraphiteMetric("some.key.value"),
		&graphite,
	)

	now := time.Unix(1438734300000, 0)

	baseTime := now.Unix() * 1000
	timeSource := func() time.Time { return now }

	queryTimerange, err := api.NewSnappedTimerange(
		int64(baseTime)-300*1000*10, // 50 minutes ago
		int64(baseTime)-300*1000*4,  // 20 minutes ago
		300*1000,                    // 5 minute resolution
	)

	// The queries have to be relative to "now"
	defaultClientConfig := Config{
		BaseUrl:                 "https://blueflood.url",
		TenantId:                "square",
		Ttls:                    make(map[string]int64),
		Timeout:                 time.Millisecond,
		FullResolutionOverlap:   14400,
		GraphiteMetricConverter: &graphite,
		TimeSource:              timeSource,
	}

	regularQueryURL := fmt.Sprintf(
		"https://blueflood.url/v2.0/square/views/some.key.value?from=%d&resolution=MIN5&select=numPoints%%2Caverage&to=%d",
		queryTimerange.Start(),
		queryTimerange.End()+queryTimerange.ResolutionMillis(),
	)

	regularResponse := fmt.Sprintf(`{
	  "unit": "unknown",
	  "values": [
	    {
	      "numPoints": 28,
	      "timestamp": %d,
	      "average": 100
	    },
	    {
	      "numPoints": 29,
	      "timestamp": %d,
	      "average": 142
	    },
	    {
	      "numPoints": 27,
	      "timestamp": %d,
	      "average": 138
	    },
	    {
	      "numPoints": 28,
	      "timestamp": %d,
	      "average": 182
	    }
	  ],
	  "metadata": {
	    "limit": null,
	    "next_href": null,
	    "count": 4,
	    "marker": null
	  }
	}`,
		baseTime-300*1000*10, // 50 minutes ago
		baseTime-300*1000*9,  // 45 minutes ago
		baseTime-300*1000*8,  // 40 minutes ago
		baseTime-300*1000*7,  // 35 minutes ago
	)

	fullResolutionQueryURL := fmt.Sprintf(
		"https://blueflood.url/v2.0/square/views/some.key.value?from=%d&resolution=FULL&select=numPoints%%2Caverage&to=%d",
		queryTimerange.Start(),
		queryTimerange.End()+queryTimerange.ResolutionMillis(),
	)
	fullResolutionResponse := fmt.Sprintf(`{
	  "unit": "unknown",
	  "values": [
	    {
	      "numPoints": 28,
	      "timestamp": %d,
	      "average": 13
	    },
	    {
	      "numPoints": 29,
	      "timestamp": %d,
	      "average": 16
	    },
	    {
	      "numPoints": 27,
	      "timestamp": %d,
	      "average": 19
	    },
	    {
	      "numPoints": 28,
	      "timestamp": %d,
	      "average": 27
	    }
	  ],
	  "metadata": {
	    "limit": null,
	    "next_href": null,
	    "count": 4,
	    "marker": null
	  }
	}`,
		baseTime-300*1000*6,      // 30m ago
		baseTime-300*1000*5+17,   // 25m ago with random shuffling
		baseTime-300*1000*4+2821, // 20m ago with random shuffling
		baseTime-300*1000*3,      // 15m ago
	)

	fakeHttpClient := mocks.NewFakeHttpClient()
	fakeHttpClient.SetResponse(regularQueryURL, mocks.Response{regularResponse, 0, http.StatusOK})
	fakeHttpClient.SetResponse(fullResolutionQueryURL, mocks.Response{fullResolutionResponse, 0, http.StatusOK})
	defaultClientConfig.HttpClient = fakeHttpClient
	defaultClientConfig.TimeSource = timeSource

	b := NewBlueflood(defaultClientConfig)

	if err != nil {
		t.Fatalf("timerange error: %s", err.Error())
	}

	seriesList, err := b.FetchSingleTimeseries(api.FetchTimeseriesRequest{
		Metric: api.TaggedMetric{
			MetricKey: api.MetricKey("some.key"),
			TagSet:    api.ParseTagSet("tag=value"),
		},
		SampleMethod:   api.SampleMean,
		Timerange:      queryTimerange,
		MetricMetadata: fakeApi,
		Cancellable:    api.NewCancellable(),
	})
	if err != nil {
		t.Fatalf("Expected success, but got error: %s", err.Error())
	}
	expected := []float64{100, 142, 138, 182, 13, 16, 19}
	if len(seriesList.Values) != len(expected) {
		t.Fatalf("Expected %+v but got %+v", expected, seriesList)
	}
	for i, expect := range expected {
		if seriesList.Values[i] != expect {
			t.Fatalf("Expected %+v but got %+v", expected, seriesList)
		}
	}
}
Ejemplo n.º 11
0
func TestIncludeRawPayload(t *testing.T) {
	graphite := mocks.FakeGraphiteConverter{
		MetricMap: map[util.GraphiteMetric]api.TaggedMetric{
			util.GraphiteMetric("some.key.value"): api.TaggedMetric{
				MetricKey: api.MetricKey("some.key"),
				TagSet:    api.ParseTagSet("tag=value"),
			},
		},
	}

	fakeApi := mocks.NewFakeMetricMetadataAPI()
	fakeApi.AddPair(
		api.TaggedMetric{
			MetricKey: api.MetricKey("some.key"),
			TagSet:    api.ParseTagSet("tag=value"),
		},
		util.GraphiteMetric("some.key.value"),
		&graphite,
	)

	now := time.Unix(1438734300000, 0)

	baseTime := now.Unix() * 1000
	timeSource := func() time.Time { return now }

	queryTimerange, err := api.NewSnappedTimerange(
		int64(baseTime)-300*1000*10, // 50 minutes ago
		int64(baseTime)-300*1000*4,  // 20 minutes ago
		300*1000,                    // 5 minute resolution
	)

	// The queries have to be relative to "now"
	defaultClientConfig := Config{
		BaseUrl:                 "https://blueflood.url",
		TenantId:                "square",
		Ttls:                    make(map[string]int64),
		Timeout:                 time.Millisecond,
		FullResolutionOverlap:   14400,
		GraphiteMetricConverter: &graphite,
		TimeSource:              timeSource,
	}

	regularQueryURL := fmt.Sprintf(
		"https://blueflood.url/v2.0/square/views/some.key.value?from=%d&resolution=MIN5&select=numPoints%%2Caverage&to=%d",
		queryTimerange.Start(),
		queryTimerange.End()+queryTimerange.ResolutionMillis(),
	)

	regularResponse := fmt.Sprintf(`{
	  "unit": "unknown",
	  "values": [
	    {
	      "numPoints": 28,
	      "timestamp": %d,
	      "average": 100
	    },
	    {
	      "numPoints": 29,
	      "timestamp": %d,
	      "average": 142
	    },
	    {
	      "numPoints": 27,
	      "timestamp": %d,
	      "average": 138
	    },
	    {
	      "numPoints": 28,
	      "timestamp": %d,
	      "average": 182
	    }
	  ],
	  "metadata": {
	    "limit": null,
	    "next_href": null,
	    "count": 4,
	    "marker": null
	  }
	}`,
		baseTime-300*1000*10, // 50 minutes ago
		baseTime-300*1000*9,  // 45 minutes ago
		baseTime-300*1000*8,  // 40 minutes ago
		baseTime-300*1000*7,  // 35 minutes ago
	)

	fakeHttpClient := mocks.NewFakeHttpClient()
	fakeHttpClient.SetResponse(regularQueryURL, mocks.Response{regularResponse, 0, http.StatusOK})
	// fakeHttpClient.SetResponse(fullResolutionQueryURL, mocks.Response{fullResolutionResponse, 0, http.StatusOK})
	defaultClientConfig.HttpClient = fakeHttpClient
	defaultClientConfig.TimeSource = timeSource

	b := NewBlueflood(defaultClientConfig)
	if err != nil {
		t.Fatalf("timerange error: %s", err.Error())
	}

	userConfig := api.UserSpecifiableConfig{
		IncludeRawData: true,
	}

	timeSeries, err := b.FetchSingleTimeseries(api.FetchTimeseriesRequest{
		Metric: api.TaggedMetric{
			MetricKey: api.MetricKey("some.key"),
			TagSet:    api.ParseTagSet("tag=value"),
		},
		SampleMethod:          api.SampleMean,
		Timerange:             queryTimerange,
		MetricMetadata:        fakeApi,
		Cancellable:           api.NewCancellable(),
		UserSpecifiableConfig: userConfig,
	})
	if err != nil {
		t.Fatalf("Expected success, but got error: %s", err.Error())
	}

	if timeSeries.Raw == nil || string(timeSeries.Raw[0]) != regularResponse {
		t.Fatalf("Didn't fill in the raw result correctly, got: %s\n", string(timeSeries.Raw[0]))
	}
}