// Query returns datapoints for the named time series during the supplied time
// span. Data is returned as a series of consecutive data points.
//
// Data is queried only at the Resolution supplied: if data for the named time
// series is not stored at the given resolution, an empty result will be
// returned.
//
// All data stored on the server is downsampled to some degree; the data points
// returned represent the average value within a sample period. Each datapoint's
// timestamp falls in the middle of the sample period it represents.
//
// If data for the named time series was collected from multiple sources, each
// returned datapoint will represent the sum of datapoints from all sources at
// the same time. The returned string slices contains a list of all sources for
// the metric which were aggregated to produce the result.
func (db *DB) Query(query tspb.Query, r Resolution, startNanos, endNanos int64) ([]tspb.TimeSeriesDatapoint, []string, error) {
// Normalize startNanos and endNanos the nearest SampleDuration boundary.
startNanos -= startNanos % r.SampleDuration()
var rows []client.KeyValue
if len(query.Sources) == 0 {
// Based on the supplied timestamps and resolution, construct start and end
// keys for a scan that will return every key with data relevant to the
// query.
startKey := MakeDataKey(query.Name, "" /* source */, r, startNanos)
endKey := MakeDataKey(query.Name, "" /* source */, r, endNanos).PrefixEnd()
b := &client.Batch{}
b.Scan(startKey, endKey)
if err := db.db.Run(b); err != nil {
return nil, nil, err
}
rows = b.Results[0].Rows
} else {
b := &client.Batch{}
// Iterate over all key timestamps which may contain data for the given
// sources, based on the given start/end time and the resolution.
kd := r.KeyDuration()
startKeyNanos := startNanos - (startNanos % kd)
endKeyNanos := endNanos - (endNanos % kd)
for currentTimestamp := startKeyNanos; currentTimestamp <= endKeyNanos; currentTimestamp += kd {
for _, source := range query.Sources {
key := MakeDataKey(query.Name, source, r, currentTimestamp)
b.Get(key)
}
}
err := db.db.Run(b)
if err != nil {
return nil, nil, err
}
for _, result := range b.Results {
row := result.Rows[0]
if row.Value == nil {
continue
}
rows = append(rows, row)
}
}
// Convert the queried source data into a set of data spans, one for each
// source.
sourceSpans, err := makeDataSpans(rows, startNanos)
if err != nil {
return nil, nil, err
}
// Compute a downsample function which will be used to return values from
// each source for each sample period.
downsampler, err := getDownsampleFunction(query.GetDownsampler())
if err != nil {
return nil, nil, err
}
// If we are returning a derivative, iteration needs to start at offset -1
// (in order to correctly compute the rate of change at offset 0).
var startOffset int32
isDerivative := query.GetDerivative() != tspb.TimeSeriesQueryDerivative_NONE
if isDerivative {
startOffset = -1
}
// Create an interpolatingIterator for each dataSpan, adding each iterator
// into a unionIterator collection. This is also where we compute a list of
// all sources with data present in the query.
sources := make([]string, 0, len(sourceSpans))
iters := make(unionIterator, 0, len(sourceSpans))
for name, span := range sourceSpans {
sources = append(sources, name)
iters = append(iters, span.newIterator(startOffset, downsampler))
}
// Choose an aggregation function to use when taking values from the
// unionIterator.
var valueFn func() float64
switch query.GetSourceAggregator() {
case tspb.TimeSeriesQueryAggregator_SUM:
valueFn = iters.sum
case tspb.TimeSeriesQueryAggregator_AVG:
valueFn = iters.avg
case tspb.TimeSeriesQueryAggregator_MAX:
valueFn = iters.max
case tspb.TimeSeriesQueryAggregator_MIN:
valueFn = iters.min
}
// Iterate over all requested offsets, recording a value from the
// unionIterator at each offset encountered. If the query is requesting a
// derivative, a rate of change is recorded instead of the actual values.
iters.init()
var last tspb.TimeSeriesDatapoint
if isDerivative {
last = tspb.TimeSeriesDatapoint{
TimestampNanos: iters.timestamp(),
Value: valueFn(),
}
// For derivatives, the iterator was initialized at offset -1 in order
// to calculate the rate of change at offset zero. However, in some
// cases (such as the very first value recorded) offset -1 is not
// available. In this case, we treat the rate-of-change at the first
// offset as zero.
if iters.offset() < 0 {
iters.advance()
}
}
var responseData []tspb.TimeSeriesDatapoint
for iters.isValid() && iters.timestamp() <= endNanos {
current := tspb.TimeSeriesDatapoint{
TimestampNanos: iters.timestamp(),
Value: valueFn(),
}
response := current
if isDerivative {
dTime := (current.TimestampNanos - last.TimestampNanos) / time.Second.Nanoseconds()
if dTime == 0 {
response.Value = 0
} else {
response.Value = (current.Value - last.Value) / float64(dTime)
}
if response.Value < 0 &&
query.GetDerivative() == tspb.TimeSeriesQueryDerivative_NON_NEGATIVE_DERIVATIVE {
response.Value = 0
}
}
responseData = append(responseData, response)
last = current
iters.advance()
}
return responseData, sources, nil
}
// assertQuery generates a query result from the local test model and compares
// it against the query returned from the server.
func (tm *testModel) assertQuery(
name string,
sources []string,
downsample, agg *tspb.TimeSeriesQueryAggregator,
derivative *tspb.TimeSeriesQueryDerivative,
r Resolution,
start, end int64,
expectedDatapointCount, expectedSourceCount int,
) {
// Query the actual server.
q := tspb.Query{
Name: name,
Downsampler: downsample,
SourceAggregator: agg,
Derivative: derivative,
Sources: sources,
}
actualDatapoints, actualSources, err := tm.DB.Query(q, r, start, end)
if err != nil {
tm.t.Fatal(err)
}
if a, e := len(actualDatapoints), expectedDatapointCount; a != e {
tm.t.Logf("actual datapoints: %v", actualDatapoints)
tm.t.Fatal(errors.Errorf("query expected %d datapoints, got %d", e, a))
}
if a, e := len(actualSources), expectedSourceCount; a != e {
tm.t.Fatal(errors.Errorf("query expected %d sources, got %d", e, a))
}
// Construct an expected result for comparison.
var expectedDatapoints []tspb.TimeSeriesDatapoint
expectedSources := make([]string, 0, 0)
dataSpans := make(map[string]*dataSpan)
// If no specific sources were provided, look for data from every source
// encountered by the test model.
var sourcesToCheck map[string]struct{}
if len(sources) == 0 {
sourcesToCheck = tm.seenSources
} else {
sourcesToCheck = make(map[string]struct{})
for _, s := range sources {
sourcesToCheck[s] = struct{}{}
}
}
// Iterate over all possible sources which may have data for this query.
for sourceName := range sourcesToCheck {
// Iterate over all possible key times at which query data may be present.
for time := start - (start % r.KeyDuration()); time < end; time += r.KeyDuration() {
// Construct a key for this source/time and retrieve it from model.
key := MakeDataKey(name, sourceName, r, time)
value, ok := tm.modelData[string(key)]
if !ok {
continue
}
// Add data from the key to the correct dataSpan.
data, err := value.GetTimeseries()
if err != nil {
tm.t.Fatal(err)
}
ds, ok := dataSpans[sourceName]
if !ok {
ds = &dataSpan{
startNanos: start - (start % r.SampleDuration()),
sampleNanos: r.SampleDuration(),
}
dataSpans[sourceName] = ds
expectedSources = append(expectedSources, sourceName)
}
if err := ds.addData(data); err != nil {
tm.t.Fatal(err)
}
}
}
// Iterate over data in all dataSpans and construct expected datapoints.
var startOffset int32
isDerivative := q.GetDerivative() != tspb.TimeSeriesQueryDerivative_NONE
if isDerivative {
startOffset = -1
}
downsampleFn, err := getDownsampleFunction(q.GetDownsampler())
if err != nil {
tm.t.Fatal(err)
}
var iters unionIterator
for _, ds := range dataSpans {
iters = append(iters, ds.newIterator(startOffset, downsampleFn))
}
iters.init()
currentVal := func() tspb.TimeSeriesDatapoint {
var value float64
switch q.GetSourceAggregator() {
case tspb.TimeSeriesQueryAggregator_SUM:
value = iters.sum()
case tspb.TimeSeriesQueryAggregator_AVG:
value = iters.avg()
case tspb.TimeSeriesQueryAggregator_MAX:
value = iters.max()
case tspb.TimeSeriesQueryAggregator_MIN:
value = iters.min()
default:
tm.t.Fatalf("unknown query aggregator %s", q.GetSourceAggregator())
}
return tspb.TimeSeriesDatapoint{
TimestampNanos: iters.timestamp(),
Value: value,
}
}
var last tspb.TimeSeriesDatapoint
if isDerivative {
last = currentVal()
if iters.offset() < 0 {
iters.advance()
}
}
for iters.isValid() && iters.timestamp() <= end {
current := currentVal()
result := current
if isDerivative {
dTime := (current.TimestampNanos - last.TimestampNanos) / int64(time.Second)
if dTime == 0 {
result.Value = 0
} else {
result.Value = (current.Value - last.Value) / float64(dTime)
}
if result.Value < 0 &&
q.GetDerivative() == tspb.TimeSeriesQueryDerivative_NON_NEGATIVE_DERIVATIVE {
result.Value = 0
}
}
expectedDatapoints = append(expectedDatapoints, result)
last = current
iters.advance()
}
sort.Strings(expectedSources)
sort.Strings(actualSources)
if !reflect.DeepEqual(actualSources, expectedSources) {
tm.t.Error(errors.Errorf("actual source list: %v, expected: %v", actualSources, expectedSources))
}
if !reflect.DeepEqual(actualDatapoints, expectedDatapoints) {
tm.t.Error(errors.Errorf("actual datapoints: %v, expected: %v", actualDatapoints, expectedDatapoints))
}
}