Exemplo n.º 1
0
// assertQuery generates a query result from the local test model and compares
// it against the query returned from the server.
func (tm *testModel) assertQuery(
	name string,
	sources []string,
	downsample, agg *tspb.TimeSeriesQueryAggregator,
	derivative *tspb.TimeSeriesQueryDerivative,
	r Resolution,
	sampleDuration, start, end int64,
	expectedDatapointCount, expectedSourceCount int,
) {
	// Query the actual server.
	q := tspb.Query{
		Name:             name,
		Downsampler:      downsample,
		SourceAggregator: agg,
		Derivative:       derivative,
		Sources:          sources,
	}
	actualDatapoints, actualSources, err := tm.DB.Query(context.TODO(), q, r, sampleDuration, start, end)
	if err != nil {
		tm.t.Fatal(err)
	}
	if a, e := len(actualDatapoints), expectedDatapointCount; a != e {
		tm.t.Logf("actual datapoints: %v", actualDatapoints)
		tm.t.Fatal(errors.Errorf("query expected %d datapoints, got %d", e, a))
	}
	if a, e := len(actualSources), expectedSourceCount; a != e {
		tm.t.Fatal(errors.Errorf("query expected %d sources, got %d", e, a))
	}

	// Construct an expected result for comparison.
	var expectedDatapoints []tspb.TimeSeriesDatapoint
	expectedSources := make([]string, 0, 0)
	dataSpans := make(map[string]*dataSpan)

	// If no specific sources were provided, look for data from every source
	// encountered by the test model.
	var sourcesToCheck map[string]struct{}
	if len(sources) == 0 {
		sourcesToCheck = tm.seenSources
	} else {
		sourcesToCheck = make(map[string]struct{})
		for _, s := range sources {
			sourcesToCheck[s] = struct{}{}
		}
	}

	// Iterate over all possible sources which may have data for this query.
	for sourceName := range sourcesToCheck {
		// Iterate over all possible key times at which query data may be present.
		for time := start - (start % r.SlabDuration()); time < end; time += r.SlabDuration() {
			// Construct a key for this source/time and retrieve it from model.
			key := MakeDataKey(name, sourceName, r, time)
			value, ok := tm.modelData[string(key)]
			if !ok {
				continue
			}

			// Add data from the key to the correct dataSpan.
			data, err := value.GetTimeseries()
			if err != nil {
				tm.t.Fatal(err)
			}
			ds, ok := dataSpans[sourceName]
			if !ok {
				ds = &dataSpan{
					startNanos:  start - (start % r.SampleDuration()),
					sampleNanos: r.SampleDuration(),
				}
				dataSpans[sourceName] = ds
				expectedSources = append(expectedSources, sourceName)
			}
			if err := ds.addData(data); err != nil {
				tm.t.Fatal(err)
			}
		}
	}

	// Verify that expected sources match actual sources.
	sort.Strings(expectedSources)
	sort.Strings(actualSources)
	if !reflect.DeepEqual(actualSources, expectedSources) {
		tm.t.Error(errors.Errorf("actual source list: %v, expected: %v", actualSources, expectedSources))
	}

	// Iterate over data in all dataSpans and construct expected datapoints.
	var startOffset int32
	isDerivative := q.GetDerivative() != tspb.TimeSeriesQueryDerivative_NONE
	if isDerivative {
		startOffset = -1
	}
	extractFn, err := getExtractionFunction(q.GetDownsampler())
	if err != nil {
		tm.t.Fatal(err)
	}
	downsampleFn, err := getDownsampleFunction(q.GetDownsampler())
	if err != nil {
		tm.t.Fatal(err)
	}
	var iters aggregatingIterator
	for _, ds := range dataSpans {
		iters = append(iters, newInterpolatingIterator(*ds, startOffset, sampleDuration, extractFn, downsampleFn))
	}

	iters.init()
	if !iters.isValid() {
		if a, e := 0, len(expectedDatapoints); a != e {
			tm.t.Error(errors.Errorf("query had zero datapoints, expected: %v", expectedDatapoints))
		}
		return
	}
	currentVal := func() tspb.TimeSeriesDatapoint {
		var value float64
		switch q.GetSourceAggregator() {
		case tspb.TimeSeriesQueryAggregator_SUM:
			value = iters.sum()
		case tspb.TimeSeriesQueryAggregator_AVG:
			value = iters.avg()
		case tspb.TimeSeriesQueryAggregator_MAX:
			value = iters.max()
		case tspb.TimeSeriesQueryAggregator_MIN:
			value = iters.min()
		default:
			tm.t.Fatalf("unknown query aggregator %s", q.GetSourceAggregator())
		}
		return tspb.TimeSeriesDatapoint{
			TimestampNanos: iters.timestamp(),
			Value:          value,
		}
	}

	var last tspb.TimeSeriesDatapoint
	if isDerivative {
		last = currentVal()
		if iters.offset() < 0 {
			iters.advance()
		}
	}
	for iters.isValid() && iters.timestamp() <= end {
		current := currentVal()
		result := current
		if isDerivative {
			dTime := (current.TimestampNanos - last.TimestampNanos) / int64(time.Second)
			if dTime == 0 {
				result.Value = 0
			} else {
				result.Value = (current.Value - last.Value) / float64(dTime)
			}
			if result.Value < 0 &&
				q.GetDerivative() == tspb.TimeSeriesQueryDerivative_NON_NEGATIVE_DERIVATIVE {
				result.Value = 0
			}
		}
		expectedDatapoints = append(expectedDatapoints, result)
		last = current
		iters.advance()
	}

	if !reflect.DeepEqual(actualDatapoints, expectedDatapoints) {
		tm.t.Error(errors.Errorf("actual datapoints: %v, expected: %v", actualDatapoints, expectedDatapoints))
	}
}
Exemplo n.º 2
0
// Query returns datapoints for the named time series during the supplied time
// span.  Data is returned as a series of consecutive data points.
//
// Raw data is queried only at the queryResolution supplied: if data for the
// named time series is not stored at the given resolution, an empty result will
// be returned.
//
// Raw data is converted into query results through a number of processing
// steps, which are executed in the following order:
//
// 1. Downsampling
// 2. Rate calculation (if requested)
// 3. Interpolation and Aggregation
//
// Raw data stored on the server is already downsampled into samples with
// interval length queryResolution.SampleDuration(); however, Result data can be
// further downsampled into a longer sample intervals based on a provided
// sampleDuration. sampleDuration must have a sample duration which is a
// positive integer multiple of the queryResolution's sample duration. The
// downsampling operation can compute a sum, total, max or min. Each downsampled
// datapoint's timestamp falls in the middle of the sample period it represents.
//
// After downsampling, values can be converted into a rate if requested by the
// query. Each data point's value is replaced by the derivative of the series at
// that timestamp, computed by comparing the datapoint to its predecessor. If a
// query requests a derivative, the returned value for each datapoint is
// expressed in units per second.
//
// If data for the named time series was collected from multiple sources, each
// returned datapoint will represent the sum of datapoints from all sources at
// the same time. The returned string slices contains a list of all sources for
// the metric which were aggregated to produce the result. In the case where one
// series is missing a data point that is present in other series, the missing
// data points for that series will be interpolated using linear interpolation.
func (db *DB) Query(
	ctx context.Context,
	query tspb.Query,
	queryResolution Resolution,
	sampleDuration, startNanos, endNanos int64,
) ([]tspb.TimeSeriesDatapoint, []string, error) {
	// Verify that sampleDuration is a multiple of
	// queryResolution.SampleDuration().
	if sampleDuration < queryResolution.SampleDuration() {
		return nil, nil, fmt.Errorf(
			"sampleDuration %d was not less that queryResolution.SampleDuration %d",
			sampleDuration,
			queryResolution.SampleDuration(),
		)
	}
	if sampleDuration%queryResolution.SampleDuration() != 0 {
		return nil, nil, fmt.Errorf(
			"sampleDuration %d is not a multiple of queryResolution.SampleDuration %d",
			sampleDuration,
			queryResolution.SampleDuration(),
		)
	}

	// Normalize startNanos to a sampleDuration boundary.
	startNanos -= startNanos % sampleDuration

	var rows []client.KeyValue
	if len(query.Sources) == 0 {
		// Based on the supplied timestamps and resolution, construct start and
		// end keys for a scan that will return every key with data relevant to
		// the query.
		startKey := MakeDataKey(query.Name, "" /* source */, queryResolution, startNanos)
		endKey := MakeDataKey(query.Name, "" /* source */, queryResolution, endNanos).PrefixEnd()
		b := &client.Batch{}
		b.Scan(startKey, endKey)

		if err := db.db.Run(ctx, b); err != nil {
			return nil, nil, err
		}
		rows = b.Results[0].Rows
	} else {
		b := &client.Batch{}
		// Iterate over all key timestamps which may contain data for the given
		// sources, based on the given start/end time and the resolution.
		kd := queryResolution.SlabDuration()
		startKeyNanos := startNanos - (startNanos % kd)
		endKeyNanos := endNanos - (endNanos % kd)
		for currentTimestamp := startKeyNanos; currentTimestamp <= endKeyNanos; currentTimestamp += kd {
			for _, source := range query.Sources {
				key := MakeDataKey(query.Name, source, queryResolution, currentTimestamp)
				b.Get(key)
			}
		}
		err := db.db.Run(ctx, b)
		if err != nil {
			return nil, nil, err
		}
		for _, result := range b.Results {
			row := result.Rows[0]
			if row.Value == nil {
				continue
			}
			rows = append(rows, row)
		}
	}

	// Convert the queried source data into a set of data spans, one for each
	// source.
	sourceSpans, err := makeDataSpans(rows, startNanos)
	if err != nil {
		return nil, nil, err
	}

	// Choose an extractor function which will be used to return values from
	// each source for each sample period.
	extractor, err := getExtractionFunction(query.GetDownsampler())
	if err != nil {
		return nil, nil, err
	}

	// Choose downsampler function.
	downsampler, err := getDownsampleFunction(query.GetDownsampler())
	if err != nil {
		return nil, nil, err
	}

	// Create an interpolatingIterator for each dataSpan, adding each iterator
	// into a aggregatingIterator collection. This is also where we compute a
	// list of all sources with data present in the query.
	sources := make([]string, 0, len(sourceSpans))
	iters := make(aggregatingIterator, 0, len(sourceSpans))
	for name, span := range sourceSpans {
		sources = append(sources, name)
		iters = append(iters, newInterpolatingIterator(
			*span, 0, sampleDuration, extractor, downsampler, query.GetDerivative(),
		))
	}

	// Choose an aggregation function to use when taking values from the
	// aggregatingIterator.
	var valueFn func() float64
	switch query.GetSourceAggregator() {
	case tspb.TimeSeriesQueryAggregator_SUM:
		valueFn = iters.sum
	case tspb.TimeSeriesQueryAggregator_AVG:
		valueFn = iters.avg
	case tspb.TimeSeriesQueryAggregator_MAX:
		valueFn = iters.max
	case tspb.TimeSeriesQueryAggregator_MIN:
		valueFn = iters.min
	}

	// Iterate over all requested offsets, recording a value from the
	// aggregatingIterator at each offset encountered. If the query is
	// requesting a derivative, a rate of change is recorded instead of the
	// actual values.
	iters.init()
	if !iters.isValid() {
		// We have no data to return.
		return nil, sources, nil
	}

	var responseData []tspb.TimeSeriesDatapoint

	for iters.isValid() && iters.timestamp() <= endNanos {
		response := tspb.TimeSeriesDatapoint{
			TimestampNanos: iters.timestamp(),
			Value:          valueFn(),
		}
		if query.GetDerivative() != tspb.TimeSeriesQueryDerivative_NONE {
			response.Value = response.Value / float64(sampleDuration) * float64(time.Second.Nanoseconds())
		}
		responseData = append(responseData, response)
		iters.advance()
	}

	return responseData, sources, nil
}