// assertModelCorrect asserts that the model data being maintained by this
// testModel is equivalent to the actual time series data stored in the
// engine. If the actual data does not match the model, this method will print
// out detailed information about the differences between the two data sets.
func (tm *testModel) assertModelCorrect() {
actualData := tm.getActualData()
if !reflect.DeepEqual(tm.modelData, actualData) {
// Provide a detailed differencing of the actual data and the expected
// model. This is done by comparing individual keys, and printing human
// readable information about any keys which differ in value between the
// two data sets.
var buf bytes.Buffer
buf.WriteString("Found unexpected differences in model data and actual data:\n")
for k, vActual := range actualData {
n, s, r, ts, err := DecodeDataKey([]byte(k))
if err != nil {
tm.t.Fatal(err)
}
if vModel, ok := tm.modelData[k]; !ok {
fmt.Fprintf(&buf, "\tKey %s/%s@%d, r:%d from actual data was not found in model", n, s, ts, r)
} else {
if !gogoproto.Equal(vActual, vModel) {
fmt.Fprintf(&buf, "\tKey %s/%s@%d, r:%d differs between model and actual:", n, s, ts, r)
if its, err := proto.InternalTimeSeriesDataFromValue(vActual); err != nil {
fmt.Fprintf(&buf, "\tActual value is not a valid time series: %v", vActual)
} else {
fmt.Fprintf(&buf, "\tActual value: %v", its)
}
if its, err := proto.InternalTimeSeriesDataFromValue(vModel); err != nil {
fmt.Fprintf(&buf, "\tModel value is not a valid time series: %v", vModel)
} else {
fmt.Fprintf(&buf, "\tModel value: %v", its)
}
}
}
}
// Detect keys in model which were not present in the actual data.
for k := range tm.modelData {
n, s, r, ts, err := DecodeDataKey([]byte(k))
if err != nil {
tm.t.Fatal(err)
}
if _, ok := actualData[k]; !ok {
fmt.Fprintf(&buf, "Key %s/%s@%d, r:%d from model was not found in actual data", n, s, ts, r)
}
}
tm.t.Fatal(buf.String())
}
}
// unmarshalTimeSeries unmarshals the time series value stored in the given byte
// array. It is assumed that the time series value was originally marshalled as
// a MVCCMetadata with an inline value.
func unmarshalTimeSeries(t testing.TB, b []byte) *proto.InternalTimeSeriesData {
if b == nil {
return nil
}
var mvccValue MVCCMetadata
if err := gogoproto.Unmarshal(b, &mvccValue); err != nil {
t.Fatalf("error unmarshalling time series in text: %s", err.Error())
}
valueTS, err := proto.InternalTimeSeriesDataFromValue(mvccValue.Value)
if err != nil {
t.Fatalf("error unmarshalling time series in text: %s", err.Error())
}
return valueTS
}
// MergeInternalTimeSeriesData exports the engine's C++ merge logic for
// InternalTimeSeriesData to higher level packages. This is intended primarily
// for consumption by high level testing of time series functionality.
func MergeInternalTimeSeriesData(sources ...*proto.InternalTimeSeriesData) (
*proto.InternalTimeSeriesData, error) {
// Wrap each proto in an inlined MVCC value, and marshal each wrapped value
// to bytes. This is the format required by the engine.
srcBytes := make([][]byte, 0, len(sources))
for _, src := range sources {
val, err := src.ToValue()
if err != nil {
return nil, err
}
bytes, err := gogoproto.Marshal(&MVCCMetadata{
Value: val,
})
if err != nil {
return nil, err
}
srcBytes = append(srcBytes, bytes)
}
// Merge every element into a nil byte slice, one at a time.
var (
mergedBytes []byte
err error
)
for _, bytes := range srcBytes {
mergedBytes, err = goMerge(mergedBytes, bytes)
if err != nil {
return nil, err
}
}
// Unmarshal merged bytes and extract the time series value within.
var mvccValue MVCCMetadata
if err := gogoproto.Unmarshal(mergedBytes, &mvccValue); err != nil {
return nil, err
}
mergedTS, err := proto.InternalTimeSeriesDataFromValue(mvccValue.Value)
if err != nil {
return nil, err
}
return mergedTS, nil
}
func (tm *testModel) storeInModel(r Resolution, data proto.TimeSeriesData) {
// Note the source, used to construct keys for model queries.
tm.seenSources[data.Source] = true
// Process and store data in the model.
internalData, err := data.ToInternal(r.KeyDuration(), r.SampleDuration())
if err != nil {
tm.t.Fatalf("test could not convert time series to internal format: %s", err.Error())
}
for _, idata := range internalData {
key := MakeDataKey(data.Name, data.Source, r, idata.StartTimestampNanos)
keyStr := string(key)
existing, ok := tm.modelData[keyStr]
var newTs *proto.InternalTimeSeriesData
if ok {
existingTs, err := proto.InternalTimeSeriesDataFromValue(existing)
if err != nil {
tm.t.Fatalf("test could not extract time series from existing model value: %s", err.Error())
}
newTs, err = engine.MergeInternalTimeSeriesData(existingTs, idata)
if err != nil {
tm.t.Fatalf("test could not merge time series into model value: %s", err.Error())
}
} else {
newTs, err = engine.MergeInternalTimeSeriesData(idata)
if err != nil {
tm.t.Fatalf("test could not merge time series into model value: %s", err.Error())
}
}
val, err := newTs.ToValue()
if err != nil {
tm.t.Fatal(err)
}
tm.modelData[keyStr] = val
}
}
// assertQuery generates a query result from the local test model and compares
// it against the query returned from the server.
func (tm *testModel) assertQuery(name string, agg *proto.TimeSeriesQueryAggregator,
r Resolution, start, end int64, expectedDatapointCount int, expectedSourceCount int) {
// Query the actual server.
q := proto.TimeSeriesQueryRequest_Query{
Name: name,
Aggregator: agg,
}
actualDatapoints, actualSources, err := tm.DB.Query(q, r, start, end)
if err != nil {
tm.t.Fatal(err)
}
if a, e := len(actualDatapoints), expectedDatapointCount; a != e {
tm.t.Fatalf("query expected %d datapoints, got %d", e, a)
}
if a, e := len(actualSources), expectedSourceCount; a != e {
tm.t.Fatalf("query expected %d sources, got %d", e, a)
}
// Construct an expected result for comparison.
var expectedDatapoints []*proto.TimeSeriesDatapoint
expectedSources := make([]string, 0, 0)
dataSpans := make(map[string]*dataSpan)
// Iterate over all possible sources which may have data for this query.
for sourceName := range tm.seenSources {
// Iterate over all possible key times at which query data may be present.
for time := start - (start % r.KeyDuration()); time < end; time += r.KeyDuration() {
// Construct a key for this source/time and retrieve it from model.
key := MakeDataKey(name, sourceName, r, time)
value := tm.modelData[string(key)]
if value == nil {
continue
}
// Add data from the key to the correct dataSpan.
data, err := proto.InternalTimeSeriesDataFromValue(value)
if err != nil {
tm.t.Fatal(err)
}
ds, ok := dataSpans[sourceName]
if !ok {
ds = &dataSpan{
startNanos: start - (start % r.SampleDuration()),
sampleNanos: r.SampleDuration(),
}
dataSpans[sourceName] = ds
expectedSources = append(expectedSources, sourceName)
}
if err := ds.addData(data); err != nil {
tm.t.Fatal(err)
}
}
}
// Iterate over data in all dataSpans and construct expected datapoints.
var iters unionIterator
for _, ds := range dataSpans {
iters = append(iters, ds.newIterator())
}
iters.init()
for iters.isValid() {
var value float64
switch q.GetAggregator() {
case proto.TimeSeriesQueryAggregator_AVG:
value = iters.avg()
case proto.TimeSeriesQueryAggregator_AVG_RATE:
value = iters.dAvg()
}
expectedDatapoints = append(expectedDatapoints, &proto.TimeSeriesDatapoint{
TimestampNanos: iters.timestamp(),
Value: value,
})
iters.advance()
}
sort.Strings(expectedSources)
sort.Strings(actualSources)
if !reflect.DeepEqual(actualSources, expectedSources) {
tm.t.Errorf("actual source list: %v, expected: %v", actualSources, expectedSources)
}
if !reflect.DeepEqual(actualDatapoints, expectedDatapoints) {
tm.t.Errorf("actual datapoints: %v, expected: %v", actualDatapoints, expectedDatapoints)
}
}
// TestGoMerge tests the function goMerge but not the integration with
// the storage engines. For that, see the engine tests.
func TestGoMerge(t *testing.T) {
// Let's start with stuff that should go wrong.
badCombinations := []struct {
existing, update []byte
}{
{counter(0), appender("")},
{appender(""), counter(0)},
{counter(0), nil},
{appender(""), nil},
{
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
nil,
},
{
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
appender("a"),
},
{
appender("a"),
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
},
{
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime+1, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
},
{
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime, 100, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
},
}
for i, c := range badCombinations {
_, err := goMerge(c.existing, c.update)
if err == nil {
t.Errorf("goMerge: %d: expected error", i)
}
}
testCasesCounter := []struct {
existing, update, expected int64
wantError bool
}{
{0, 10, 10, false},
{10, 20, 30, false},
{595, -600, -5, false},
// Close to overflow, but not quite there.
{math.MinInt64 + 3, -3, math.MinInt64, false},
{math.MaxInt64, 0, math.MaxInt64, false},
// Overflows.
{math.MaxInt64, 1, 0, true},
{-1, math.MinInt64, 0, true},
}
for i, c := range testCasesCounter {
result, err := goMerge(counter(c.existing), counter(c.update))
if c.wantError {
if err == nil {
t.Errorf("goMerge: %d: wanted error but got success", i)
}
continue
}
if err != nil {
t.Errorf("goMerge error: %d: %v", i, err)
continue
}
var v proto.MVCCMetadata
if err := gogoproto.Unmarshal(result, &v); err != nil {
t.Errorf("goMerge error unmarshalling: %s", err)
continue
}
if v.Value.GetInteger() != c.expected {
t.Errorf("goMerge error: %d: want %v, got %v", i, c.expected, v.Value.GetInteger())
}
}
gibber1, gibber2 := gibberishString(100), gibberishString(200)
testCasesAppender := []struct {
existing, update, expected []byte
}{
{appender(""), appender(""), appender("")},
{nil, appender(""), appender("")},
{nil, nil, mustMarshal(&proto.MVCCMetadata{Value: &proto.Value{}})},
{appender("\n "), appender(" \t "), appender("\n \t ")},
{appender("ქართული"), appender("\nKhartuli"), appender("ქართული\nKhartuli")},
{appender(gibber1), appender(gibber2), appender(gibber1 + gibber2)},
}
for i, c := range testCasesAppender {
result, err := goMerge(c.existing, c.update)
if err != nil {
t.Errorf("goMerge error: %d: %v", i, err)
continue
}
var resultV, expectedV proto.MVCCMetadata
gogoproto.Unmarshal(result, &resultV)
gogoproto.Unmarshal(c.expected, &expectedV)
if !reflect.DeepEqual(resultV, expectedV) {
t.Errorf("goMerge error: %d: want %+v, got %+v", i, expectedV, resultV)
}
}
testCasesTimeSeries := []struct {
existing, update, expected []byte
}{
{
nil,
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
},
{
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime, 1000, []tsIntSample{
{2, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
{2, 1, 5, 5, 5},
}...),
},
{
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
{3, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime, 1000, []tsIntSample{
{2, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
{2, 1, 5, 5, 5},
{3, 1, 5, 5, 5},
}...),
},
{
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 10, 10, 10},
{1, 1, 5, 5, 5},
{2, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 100, 100, 100},
{2, 1, 5, 5, 5},
{3, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 3, 115, 100, 5},
{2, 2, 10, 5, 5},
{3, 1, 5, 5, 5},
}...),
},
{
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime, 1000, []tsIntSample{
{2, 1, 5, 5, 5},
}...),
timeSeriesInt(testtime, 1000, []tsIntSample{
{1, 1, 5, 5, 5},
{2, 1, 5, 5, 5},
}...),
},
{
timeSeriesFloat(testtime, 1000, []tsFloatSample{
{1, 1, 5, 5, 5},
{3, 1, 5, 5, 5},
}...),
timeSeriesFloat(testtime, 1000, []tsFloatSample{
{2, 1, 5, 5, 5},
}...),
timeSeriesFloat(testtime, 1000, []tsFloatSample{
{1, 1, 5, 5, 5},
{2, 1, 5, 5, 5},
{3, 1, 5, 5, 5},
}...),
},
{
timeSeriesFloat(testtime, 1000, []tsFloatSample{
{1, 1, 10, 10, 10},
{1, 1, 5, 5, 5},
{2, 1, 5, 5, 5},
}...),
timeSeriesFloat(testtime, 1000, []tsFloatSample{
{1, 1, 100, 100, 100},
{2, 1, 5, 5, 5},
{3, 1, 5, 5, 5},
}...),
timeSeriesFloat(testtime, 1000, []tsFloatSample{
{1, 3, 115, 100, 5},
{2, 2, 10, 5, 5},
{3, 1, 5, 5, 5},
}...),
},
}
for i, c := range testCasesTimeSeries {
result, err := goMerge(c.existing, c.update)
if err != nil {
t.Errorf("goMerge error: %d: %v", i, err)
continue
}
// Extract the time series and compare.
var resultV, expectedV proto.MVCCMetadata
gogoproto.Unmarshal(result, &resultV)
gogoproto.Unmarshal(c.expected, &expectedV)
resultTS, _ := proto.InternalTimeSeriesDataFromValue(resultV.Value)
expectedTS, _ := proto.InternalTimeSeriesDataFromValue(expectedV.Value)
if !reflect.DeepEqual(resultTS, expectedTS) {
t.Errorf("goMerge error: %d: want %v, got %v", i, expectedTS, resultTS)
}
}
}