// assertModelCorrect asserts that the model data being maintained by this
// testModel is equivalent to the actual time series data stored in the
// engine. If the actual data does not match the model, this method will print
// out detailed information about the differences between the two data sets.
func (tm *testModel) assertModelCorrect() {
actualData := tm.getActualData()
if !reflect.DeepEqual(tm.modelData, actualData) {
// Provide a detailed differencing of the actual data and the expected
// model. This is done by comparing individual keys, and printing human
// readable information about any keys which differ in value between the
// two data sets.
var buf bytes.Buffer
buf.WriteString("Found unexpected differences in model data and actual data:\n")
for k, vActual := range actualData {
n, s, r, ts, err := DecodeDataKey([]byte(k))
if err != nil {
tm.t.Fatal(err)
}
if vModel, ok := tm.modelData[k]; !ok {
fmt.Fprintf(&buf, "\tKey %s/%s@%d, r:%d from actual data was not found in model", n, s, ts, r)
} else {
if !proto.Equal(vActual, vModel) {
fmt.Fprintf(&buf, "\tKey %s/%s@%d, r:%d differs between model and actual:", n, s, ts, r)
if its, err := roachpb.InternalTimeSeriesDataFromValue(vActual); err != nil {
fmt.Fprintf(&buf, "\tActual value is not a valid time series: %v", vActual)
} else {
fmt.Fprintf(&buf, "\tActual value: %v", its)
}
if its, err := roachpb.InternalTimeSeriesDataFromValue(vModel); err != nil {
fmt.Fprintf(&buf, "\tModel value is not a valid time series: %v", vModel)
} else {
fmt.Fprintf(&buf, "\tModel value: %v", its)
}
}
}
}
// Detect keys in model which were not present in the actual data.
for k := range tm.modelData {
n, s, r, ts, err := DecodeDataKey([]byte(k))
if err != nil {
tm.t.Fatal(err)
}
if _, ok := actualData[k]; !ok {
fmt.Fprintf(&buf, "Key %s/%s@%d, r:%d from model was not found in actual data", n, s, ts, r)
}
}
tm.t.Fatal(buf.String())
}
}
// unmarshalTimeSeries unmarshals the time series value stored in the given byte
// array. It is assumed that the time series value was originally marshalled as
// a MVCCMetadata with an inline value.
func unmarshalTimeSeries(t testing.TB, b []byte) *roachpb.InternalTimeSeriesData {
if b == nil {
return nil
}
var mvccValue MVCCMetadata
if err := proto.Unmarshal(b, &mvccValue); err != nil {
t.Fatalf("error unmarshalling time series in text: %s", err.Error())
}
valueTS, err := roachpb.InternalTimeSeriesDataFromValue(mvccValue.Value)
if err != nil {
t.Fatalf("error unmarshalling time series in text: %s", err.Error())
}
return valueTS
}
// MergeInternalTimeSeriesData exports the engine's C++ merge logic for
// InternalTimeSeriesData to higher level packages. This is intended primarily
// for consumption by high level testing of time series functionality.
func MergeInternalTimeSeriesData(sources ...*roachpb.InternalTimeSeriesData) (
*roachpb.InternalTimeSeriesData, error) {
// Wrap each proto in an inlined MVCC value, and marshal each wrapped value
// to bytes. This is the format required by the engine.
srcBytes := make([][]byte, 0, len(sources))
for _, src := range sources {
val, err := src.ToValue()
if err != nil {
return nil, err
}
bytes, err := proto.Marshal(&MVCCMetadata{
Value: val,
})
if err != nil {
return nil, err
}
srcBytes = append(srcBytes, bytes)
}
// Merge every element into a nil byte slice, one at a time.
var (
mergedBytes []byte
err error
)
for _, bytes := range srcBytes {
mergedBytes, err = goMerge(mergedBytes, bytes)
if err != nil {
return nil, err
}
}
// Unmarshal merged bytes and extract the time series value within.
var mvccValue MVCCMetadata
if err := proto.Unmarshal(mergedBytes, &mvccValue); err != nil {
return nil, err
}
mergedTS, err := roachpb.InternalTimeSeriesDataFromValue(mvccValue.Value)
if err != nil {
return nil, err
}
return mergedTS, nil
}
func (tm *testModel) storeInModel(r Resolution, data TimeSeriesData) {
// Note the source, used to construct keys for model queries.
tm.seenSources[data.Source] = struct{}{}
// Process and store data in the model.
internalData, err := data.ToInternal(r.KeyDuration(), r.SampleDuration())
if err != nil {
tm.t.Fatalf("test could not convert time series to internal format: %s", err.Error())
}
for _, idata := range internalData {
key := MakeDataKey(data.Name, data.Source, r, idata.StartTimestampNanos)
keyStr := string(key)
existing, ok := tm.modelData[keyStr]
var newTs *roachpb.InternalTimeSeriesData
if ok {
existingTs, err := roachpb.InternalTimeSeriesDataFromValue(existing)
if err != nil {
tm.t.Fatalf("test could not extract time series from existing model value: %s", err.Error())
}
newTs, err = engine.MergeInternalTimeSeriesData(existingTs, idata)
if err != nil {
tm.t.Fatalf("test could not merge time series into model value: %s", err.Error())
}
} else {
newTs, err = engine.MergeInternalTimeSeriesData(idata)
if err != nil {
tm.t.Fatalf("test could not merge time series into model value: %s", err.Error())
}
}
val, err := newTs.ToValue()
if err != nil {
tm.t.Fatal(err)
}
tm.modelData[keyStr] = val
}
}
// assertQuery generates a query result from the local test model and compares
// it against the query returned from the server.
func (tm *testModel) assertQuery(name string, agg *TimeSeriesQueryAggregator,
r Resolution, start, end int64, expectedDatapointCount int, expectedSourceCount int) {
// Query the actual server.
q := TimeSeriesQueryRequest_Query{
Name: name,
Aggregator: agg,
}
actualDatapoints, actualSources, err := tm.DB.Query(q, r, start, end)
if err != nil {
tm.t.Fatal(err)
}
if a, e := len(actualDatapoints), expectedDatapointCount; a != e {
tm.t.Fatalf("query expected %d datapoints, got %d", e, a)
}
if a, e := len(actualSources), expectedSourceCount; a != e {
tm.t.Fatalf("query expected %d sources, got %d", e, a)
}
// Construct an expected result for comparison.
var expectedDatapoints []*TimeSeriesDatapoint
expectedSources := make([]string, 0, 0)
dataSpans := make(map[string]*dataSpan)
// Iterate over all possible sources which may have data for this query.
for sourceName := range tm.seenSources {
// Iterate over all possible key times at which query data may be present.
for time := start - (start % r.KeyDuration()); time < end; time += r.KeyDuration() {
// Construct a key for this source/time and retrieve it from model.
key := MakeDataKey(name, sourceName, r, time)
value := tm.modelData[string(key)]
if value == nil {
continue
}
// Add data from the key to the correct dataSpan.
data, err := roachpb.InternalTimeSeriesDataFromValue(value)
if err != nil {
tm.t.Fatal(err)
}
ds, ok := dataSpans[sourceName]
if !ok {
ds = &dataSpan{
startNanos: start - (start % r.SampleDuration()),
sampleNanos: r.SampleDuration(),
}
dataSpans[sourceName] = ds
expectedSources = append(expectedSources, sourceName)
}
if err := ds.addData(data); err != nil {
tm.t.Fatal(err)
}
}
}
// Iterate over data in all dataSpans and construct expected datapoints.
var iters unionIterator
for _, ds := range dataSpans {
iters = append(iters, ds.newIterator())
}
iters.init()
for iters.isValid() {
var value float64
switch q.GetAggregator() {
case TimeSeriesQueryAggregator_AVG:
value = iters.avg()
case TimeSeriesQueryAggregator_AVG_RATE:
value = iters.dAvg()
}
expectedDatapoints = append(expectedDatapoints, &TimeSeriesDatapoint{
TimestampNanos: iters.timestamp(),
Value: value,
})
iters.advance()
}
sort.Strings(expectedSources)
sort.Strings(actualSources)
if !reflect.DeepEqual(actualSources, expectedSources) {
tm.t.Errorf("actual source list: %v, expected: %v", actualSources, expectedSources)
}
if !reflect.DeepEqual(actualDatapoints, expectedDatapoints) {
tm.t.Errorf("actual datapoints: %v, expected: %v", actualDatapoints, expectedDatapoints)
}
}
