func BenchmarkMVCCPutDelete(b *testing.B) {
const cacheSize = 1 << 30 // 1 GB
stopper := stop.NewStopper()
rocksdb := NewInMem(roachpb.Attributes{}, cacheSize, stopper)
defer stopper.Stop()
r := rand.New(rand.NewSource(int64(timeutil.Now().UnixNano())))
value := roachpb.MakeValueFromBytes(randutil.RandBytes(r, 10))
zeroTS := roachpb.ZeroTimestamp
var blockNum int64
for i := 0; i < b.N; i++ {
blockID := r.Int63()
blockNum++
key := encoding.EncodeVarintAscending(nil, blockID)
key = encoding.EncodeVarintAscending(key, blockNum)
if err := MVCCPut(rocksdb, nil, key, zeroTS, value, nil /* txn */); err != nil {
b.Fatal(err)
}
if err := MVCCDelete(rocksdb, nil, key, zeroTS, nil /* txn */); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkMVCCPutDelete_RocksDB(b *testing.B) {
const cacheSize = 1 << 30 // 1 GB
rocksdb, stopper := setupMVCCInMemRocksDB(b, "put_delete")
defer stopper.Stop()
r := rand.New(rand.NewSource(int64(timeutil.Now().UnixNano())))
value := roachpb.MakeValueFromBytes(randutil.RandBytes(r, 10))
zeroTS := roachpb.ZeroTimestamp
var blockNum int64
for i := 0; i < b.N; i++ {
blockID := r.Int63()
blockNum++
key := encoding.EncodeVarintAscending(nil, blockID)
key = encoding.EncodeVarintAscending(key, blockNum)
if err := MVCCPut(context.Background(), rocksdb, nil, key, zeroTS, value, nil /* txn */); err != nil {
b.Fatal(err)
}
if err := MVCCDelete(context.Background(), rocksdb, nil, key, zeroTS, nil /* txn */); err != nil {
b.Fatal(err)
}
}
}
// MakeDataKey creates a time series data key for the given series name, source,
// Resolution and timestamp. The timestamp is expressed in nanoseconds since the
// epoch; it will be truncated to an exact multiple of the supplied
// Resolution's KeyDuration.
func MakeDataKey(name string, source string, r Resolution, timestamp int64) roachpb.Key {
// Normalize timestamp into a timeslot before recording.
timeslot := timestamp / r.KeyDuration()
k := append(roachpb.Key(nil), keyDataPrefix...)
k = encoding.EncodeBytesAscending(k, []byte(name))
k = encoding.EncodeVarintAscending(k, int64(r))
k = encoding.EncodeVarintAscending(k, timeslot)
k = append(k, source...)
return k
}
func writeRandomTimeSeriesDataToRange(
t testing.TB,
store *storage.Store,
rangeID roachpb.RangeID,
keyPrefix []byte,
) (midpoint []byte) {
src := rand.New(rand.NewSource(0))
r := ts.Resolution10s
for i := 0; i < 20; i++ {
var data []tspb.TimeSeriesData
for j := int64(0); j <= src.Int63n(5); j++ {
d := tspb.TimeSeriesData{
Name: "test.random.metric",
Source: "cpu01",
}
for k := int64(0); k <= src.Int63n(10); k++ {
d.Datapoints = append(d.Datapoints, tspb.TimeSeriesDatapoint{
TimestampNanos: src.Int63n(200) * r.KeyDuration(),
Value: src.Float64(),
})
}
data = append(data, d)
}
for _, d := range data {
idatas, err := d.ToInternal(r.KeyDuration(), r.SampleDuration())
if err != nil {
t.Fatal(err)
}
for _, idata := range idatas {
var value roachpb.Value
if err := value.SetProto(&idata); err != nil {
t.Fatal(err)
}
mArgs := roachpb.MergeRequest{
Span: roachpb.Span{
Key: encoding.EncodeVarintAscending(keyPrefix, idata.StartTimestampNanos),
},
Value: value,
}
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeID,
}, &mArgs); pErr != nil {
t.Fatal(pErr)
}
}
}
}
// Return approximate midway point (100 is midway between random timestamps in range [0,200)).
midKey := append([]byte(nil), keyPrefix...)
midKey = encoding.EncodeVarintAscending(midKey, 100*r.KeyDuration())
return keys.MakeRowSentinelKey(midKey)
}
// TestStoreRangeSplitInsideRow verifies an attempt to split a range inside of
// a table row will cause a split at a boundary between rows.
func TestStoreRangeSplitInsideRow(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
// Manually create some the column keys corresponding to the table:
//
// CREATE TABLE t (id STRING PRIMARY KEY, col1 INT, col2 INT)
tableKey := keys.MakeTablePrefix(keys.MaxReservedDescID + 1)
rowKey := roachpb.Key(encoding.EncodeVarintAscending(append([]byte(nil), tableKey...), 1))
rowKey = encoding.EncodeStringAscending(encoding.EncodeVarintAscending(rowKey, 1), "a")
col1Key := keys.MakeFamilyKey(append([]byte(nil), rowKey...), 1)
col2Key := keys.MakeFamilyKey(append([]byte(nil), rowKey...), 2)
// We don't care about the value, so just store any old thing.
if err := store.DB().Put(col1Key, "column 1"); err != nil {
t.Fatal(err)
}
if err := store.DB().Put(col2Key, "column 2"); err != nil {
t.Fatal(err)
}
// Split between col1Key and col2Key by splitting before col2Key.
args := adminSplitArgs(col2Key, col2Key)
_, err := client.SendWrapped(rg1(store), nil, &args)
if err != nil {
t.Fatalf("%s: split unexpected error: %s", col1Key, err)
}
rng1 := store.LookupReplica(col1Key, nil)
rng2 := store.LookupReplica(col2Key, nil)
// Verify the two columns are still on the same range.
if !reflect.DeepEqual(rng1, rng2) {
t.Fatalf("%s: ranges differ: %+v vs %+v", roachpb.Key(col1Key), rng1, rng2)
}
// Verify we split on a row key.
if startKey := rng1.Desc().StartKey; !startKey.Equal(rowKey) {
t.Fatalf("%s: expected split on %s, but found %s",
roachpb.Key(col1Key), roachpb.Key(rowKey), startKey)
}
// Verify the previous range was split on a row key.
rng3 := store.LookupReplica(tableKey, nil)
if endKey := rng3.Desc().EndKey; !endKey.Equal(rowKey) {
t.Fatalf("%s: expected split on %s, but found %s",
roachpb.Key(col1Key), roachpb.Key(rowKey), endKey)
}
}
func (z *zeroSum) monkey(tableID uint32, d time.Duration) {
r := newRand()
zipf := z.accountDistribution(r)
for {
time.Sleep(time.Duration(rand.Float64() * float64(d)))
key := keys.MakeTablePrefix(tableID)
key = encoding.EncodeVarintAscending(key, int64(zipf.Uint64()))
key = keys.MakeRowSentinelKey(key)
switch r.Intn(2) {
case 0:
if err := z.split(z.randNode(r.Intn), key); err != nil {
if strings.Contains(err.Error(), "range is already split at key") ||
strings.Contains(err.Error(), "conflict updating range descriptors") {
continue
}
z.maybeLogError(err)
} else {
atomic.AddUint64(&z.stats.splits, 1)
}
case 1:
if transferred, err := z.transferLease(z.randNode(r.Intn), r, key); err != nil {
z.maybeLogError(err)
} else if transferred {
atomic.AddUint64(&z.stats.transfers, 1)
}
}
}
}
func TestPrettyPrint(t *testing.T) {
tm, _ := time.Parse(time.UnixDate, "Sat Mar 7 11:06:39 UTC 2015")
txnID := uuid.NewV4()
testCases := []struct {
key roachpb.Key
exp string
}{
// local
{StoreIdentKey(), "/Local/Store/storeIdent"},
{StoreGossipKey(), "/Local/Store/gossipBootstrap"},
{SequenceCacheKeyPrefix(roachpb.RangeID(1000001), txnID), fmt.Sprintf(`/Local/RangeID/1000001/r/SequenceCache/%q`, txnID)},
{SequenceCacheKey(roachpb.RangeID(1000001), txnID, uint32(111), uint32(222)), fmt.Sprintf(`/Local/RangeID/1000001/r/SequenceCache/%q/epoch:111/seq:222`, txnID)},
{RaftTombstoneKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftTombstone"},
{RaftAppliedIndexKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftAppliedIndex"},
{RaftTruncatedStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftTruncatedState"},
{RangeLeaderLeaseKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeLeaderLease"},
{RangeStatsKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeStats"},
{RaftHardStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RaftHardState"},
{RaftLastIndexKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RaftLastIndex"},
{RaftLogKey(roachpb.RangeID(1000001), uint64(200001)), "/Local/RangeID/1000001/u/RaftLog/logIndex:200001"},
{RangeLastVerificationTimestampKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RangeLastVerificationTimestamp"},
{MakeRangeKeyPrefix(roachpb.RKey("ok")), `/Local/Range/"ok"`},
{RangeDescriptorKey(roachpb.RKey("111")), `/Local/Range/"111"/RangeDescriptor`},
{RangeTreeNodeKey(roachpb.RKey("111")), `/Local/Range/"111"/RangeTreeNode`},
{TransactionKey(roachpb.Key("111"), txnID), fmt.Sprintf(`/Local/Range/"111"/Transaction/addrKey:/id:%q`, txnID)},
{LocalMax, `/Meta1/""`}, // LocalMax == Meta1Prefix
// system
{makeKey(Meta2Prefix, roachpb.Key("foo")), `/Meta2/"foo"`},
{makeKey(Meta1Prefix, roachpb.Key("foo")), `/Meta1/"foo"`},
{RangeMetaKey(roachpb.RKey("f")), `/Meta2/"f"`},
{StoreStatusKey(2222), "/System/StatusStore/2222"},
{NodeStatusKey(1111), "/System/StatusNode/1111"},
{SystemMax, "/System/Max"},
// key of key
{RangeMetaKey(roachpb.RKey(MakeRangeKeyPrefix(roachpb.RKey("ok")))), `/Meta2/Local/Range/"ok"`},
{RangeMetaKey(roachpb.RKey(makeKey(MakeTablePrefix(42), roachpb.RKey("foo")))), `/Meta2/Table/42/"foo"`},
{RangeMetaKey(roachpb.RKey(makeKey(Meta2Prefix, roachpb.Key("foo")))), `/Meta1/"foo"`},
// table
{UserTableDataMin, "/Table/50"},
{MakeTablePrefix(111), "/Table/111"},
{makeKey(MakeTablePrefix(42), roachpb.RKey("foo")), `/Table/42/"foo"`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeFloatAscending(nil, float64(233.221112)))),
"/Table/42/233.221112"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeFloatDescending(nil, float64(-233.221112)))),
"/Table/42/233.221112"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeFloatAscending(nil, math.Inf(1)))),
"/Table/42/+Inf"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeFloatAscending(nil, math.NaN()))),
"/Table/42/NaN"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeVarintAscending(nil, 1222)),
roachpb.RKey(encoding.EncodeStringAscending(nil, "handsome man"))),
`/Table/42/1222/"handsome man"`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeVarintAscending(nil, 1222))),
`/Table/42/1222`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeVarintDescending(nil, 1222))),
`/Table/42/-1223`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeBytesAscending(nil, []byte{1, 2, 8, 255}))),
`/Table/42/"\x01\x02\b\xff"`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeBytesAscending(nil, []byte{1, 2, 8, 255})),
roachpb.RKey("bar")), `/Table/42/"\x01\x02\b\xff"/"bar"`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeBytesDescending(nil, []byte{1, 2, 8, 255})),
roachpb.RKey("bar")), `/Table/42/"\x01\x02\b\xff"/"bar"`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeNullAscending(nil))), "/Table/42/NULL"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeNotNullAscending(nil))), "/Table/42/#"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeTimeAscending(nil, tm))),
"/Table/42/Sat Mar 7 11:06:39 UTC 2015"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeTimeDescending(nil, tm))),
"/Table/42/Sat Mar 7 11:06:39 UTC 2015"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeDecimalAscending(nil, inf.NewDec(1234, 2)))),
"/Table/42/12.34"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeDecimalDescending(nil, inf.NewDec(1234, 2)))),
"/Table/42/-12.34"},
// others
{makeKey([]byte("")), "/Min"},
{Meta1KeyMax, "/Meta1/Max"},
{Meta2KeyMax, "/Meta2/Max"},
{makeKey(MakeTablePrefix(42), roachpb.RKey([]byte{0x21, 'a', 0x00, 0x02})), "/Table/42/<util/encoding/encoding.go:9999: unknown escape sequence: 0x0 0x2>"},
}
for i, test := range testCases {
keyInfo := MassagePrettyPrintedSpanForTest(PrettyPrint(test.key), nil)
exp := MassagePrettyPrintedSpanForTest(test.exp, nil)
if exp != keyInfo {
t.Errorf("%d: expected %s, got %s", i, exp, keyInfo)
}
if exp != MassagePrettyPrintedSpanForTest(test.key.String(), nil) {
t.Errorf("%d: expected %s, got %s", i, exp, test.key.String())
}
}
}
func TestTableReader(t *testing.T) {
defer leaktest.AfterTest(t)()
_, sqlDB, kvDB, cleanup := sqlutils.SetupServer(t)
defer cleanup()
// Create a table where each row is:
//
// | a | b | sum | s |
// |-----------------------------------------------------------------|
// | rowId/10 | rowId%10 | rowId/10 + rowId%10 | IntToEnglish(rowId) |
aFn := func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row / 10))
}
bFn := func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row % 10))
}
sumFn := func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row/10 + row%10))
}
sqlutils.CreateTable(t, sqlDB, "t",
"a INT, b INT, sum INT, s STRING, PRIMARY KEY (a,b), INDEX bs (b,s)",
99,
sqlutils.ToRowFn(aFn, bFn, sumFn, sqlutils.RowEnglishFn))
td := sqlbase.GetTableDescriptor(kvDB, "test", "t")
makeIndexSpan := func(start, end int) TableReaderSpan {
var span roachpb.Span
prefix := roachpb.Key(sqlbase.MakeIndexKeyPrefix(td.ID, td.Indexes[0].ID))
span.Key = append(prefix, encoding.EncodeVarintAscending(nil, int64(start))...)
span.EndKey = append(span.EndKey, prefix...)
span.EndKey = append(span.EndKey, encoding.EncodeVarintAscending(nil, int64(end))...)
return TableReaderSpan{Span: span}
}
testCases := []struct {
spec TableReaderSpec
expected string
}{
{
spec: TableReaderSpec{
Filter: Expression{Expr: "$2 < 5 AND $1 != 3"}, // sum < 5 && b != 3
OutputColumns: []uint32{0, 1},
},
expected: "[[0 1] [0 2] [0 4] [1 0] [1 1] [1 2] [2 0] [2 1] [2 2] [3 0] [3 1] [4 0]]",
},
{
spec: TableReaderSpec{
Filter: Expression{Expr: "$2 < 5 AND $1 != 3"},
OutputColumns: []uint32{3}, // s
HardLimit: 4,
},
expected: "[['one'] ['two'] ['four'] ['one-zero']]",
},
{
spec: TableReaderSpec{
IndexIdx: 1,
Reverse: true,
Spans: []TableReaderSpan{makeIndexSpan(4, 6)},
Filter: Expression{Expr: "$0 < 3"}, // sum < 8
OutputColumns: []uint32{0, 1},
SoftLimit: 1,
},
expected: "[[2 5] [1 5] [0 5] [2 4] [1 4] [0 4]]",
},
}
for _, c := range testCases {
ts := c.spec
ts.Table = *td
txn := client.NewTxn(context.Background(), *kvDB)
out := &RowBuffer{}
tr, err := newTableReader(&ts, txn, out, &parser.EvalContext{})
if err != nil {
t.Fatal(err)
}
tr.Run(nil)
if out.err != nil {
t.Fatal(out.err)
}
if !out.closed {
t.Fatalf("output RowReceiver not closed")
}
if result := out.rows.String(); result != c.expected {
t.Errorf("invalid results: %s, expected %s'", result, c.expected)
}
}
}
RangeTreeRoot = roachpb.Key(makeKey(SystemPrefix, roachpb.RKey("range-tree-root")))
// StatusPrefix specifies the key prefix to store all status details.
StatusPrefix = roachpb.Key(makeKey(SystemPrefix, roachpb.RKey("status-")))
// StatusNodePrefix stores all status info for nodes.
StatusNodePrefix = roachpb.Key(makeKey(StatusPrefix, roachpb.RKey("node-")))
// TimeseriesPrefix is the key prefix for all timeseries data.
TimeseriesPrefix = roachpb.Key(makeKey(SystemPrefix, roachpb.RKey("tsd")))
// UpdateCheckPrefix is the key prefix for all update check times.
UpdateCheckPrefix = roachpb.Key(makeKey(SystemPrefix, roachpb.RKey("update-")))
UpdateCheckCluster = roachpb.Key(makeKey(UpdateCheckPrefix, roachpb.RKey("cluster")))
// TableDataMin is the start of the range of table data keys.
TableDataMin = roachpb.Key(encoding.EncodeVarintAscending(nil, math.MinInt64))
// TableDataMin is the end of the range of table data keys.
TableDataMax = roachpb.Key(encoding.EncodeVarintAscending(nil, math.MaxInt64))
// SystemConfigTableDataMax is the end key of system config structured data.
SystemConfigTableDataMax = roachpb.Key(MakeTablePrefix(MaxSystemConfigDescID + 1))
// UserTableDataMin is the start key of user structured data.
UserTableDataMin = roachpb.Key(MakeTablePrefix(MaxReservedDescID + 1))
// MaxKey is the infinity marker which is larger than any other key.
MaxKey = roachpb.KeyMax
// MinKey is a minimum key value which sorts before all other keys.
MinKey = roachpb.KeyMin
)
func TestTableReader(t *testing.T) {
defer leaktest.AfterTest(t)()
_, sqlDB, kvDB, cleanup := sqlutils.SetupServer(t)
defer cleanup()
if _, err := sqlDB.Exec(`
CREATE DATABASE test;
CREATE TABLE test.t (a INT PRIMARY KEY, b INT, c INT, d INT, INDEX bc (b, c));
INSERT INTO test.t VALUES (1, 10, 11, 12), (2, 20, 21, 22), (3, 30, 31, 32);
INSERT INTO test.t VALUES (4, 60, 61, 62), (5, 50, 51, 52), (6, 40, 41, 42);
`); err != nil {
t.Fatal(err)
}
td := sqlbase.GetTableDescriptor(kvDB, "test", "t")
ts := TableReaderSpec{
Table: *td,
IndexIdx: 0,
Reverse: false,
Spans: nil,
Filter: Expression{Expr: "$2 != 21"}, // c != 21
OutputColumns: []uint32{0, 3}, // a, d
}
txn := client.NewTxn(context.Background(), *kvDB)
out := &testingReceiver{}
tr, err := newTableReader(&ts, txn, out, parser.EvalContext{})
if err != nil {
t.Fatal(err)
}
tr.run()
if out.err != nil {
t.Fatal(out.err)
}
if !out.closed {
t.Fatalf("output rowReceiver not closed")
}
expected := "[[1 12] [3 32] [4 62] [5 52] [6 42]]"
if fmt.Sprintf("%s", out.rows) != expected {
t.Errorf("invalid results: %s, expected %s'", out.rows, expected)
}
// Read using the bc index
var span roachpb.Span
span.Key = roachpb.Key(sqlbase.MakeIndexKeyPrefix(td.ID, td.Indexes[0].ID))
span.EndKey = append(span.Key, encoding.EncodeVarintAscending(nil, 50)...)
ts = TableReaderSpec{
Table: *td,
IndexIdx: 1,
Reverse: true,
Spans: []TableReaderSpan{{Span: span}},
Filter: Expression{Expr: "$1 != 30"}, // b != 30
OutputColumns: []uint32{0, 2}, // a, c
}
out = &testingReceiver{}
tr, err = newTableReader(&ts, txn, out, parser.EvalContext{})
if err != nil {
t.Fatal(err)
}
tr.run()
if out.err != nil {
t.Fatal(out.err)
}
if !out.closed {
t.Fatalf("output rowReceiver not closed")
}
expected = "[[6 41] [2 21] [1 11]]"
if fmt.Sprintf("%s", out.rows) != expected {
t.Errorf("invalid results: %s, expected %s'", out.rows, expected)
}
}
// EncodeTableKey encodes `val` into `b` and returns the new buffer.
func EncodeTableKey(b []byte, val parser.Datum, dir encoding.Direction) ([]byte, error) {
if (dir != encoding.Ascending) && (dir != encoding.Descending) {
return nil, util.Errorf("invalid direction: %d", dir)
}
if val == parser.DNull {
if dir == encoding.Ascending {
return encoding.EncodeNullAscending(b), nil
}
return encoding.EncodeNullDescending(b), nil
}
switch t := val.(type) {
case *parser.DBool:
var x int64
if *t {
x = 1
} else {
x = 0
}
if dir == encoding.Ascending {
return encoding.EncodeVarintAscending(b, x), nil
}
return encoding.EncodeVarintDescending(b, x), nil
case *parser.DInt:
if dir == encoding.Ascending {
return encoding.EncodeVarintAscending(b, int64(*t)), nil
}
return encoding.EncodeVarintDescending(b, int64(*t)), nil
case *parser.DFloat:
if dir == encoding.Ascending {
return encoding.EncodeFloatAscending(b, float64(*t)), nil
}
return encoding.EncodeFloatDescending(b, float64(*t)), nil
case *parser.DDecimal:
if dir == encoding.Ascending {
return encoding.EncodeDecimalAscending(b, &t.Dec), nil
}
return encoding.EncodeDecimalDescending(b, &t.Dec), nil
case *parser.DString:
if dir == encoding.Ascending {
return encoding.EncodeStringAscending(b, string(*t)), nil
}
return encoding.EncodeStringDescending(b, string(*t)), nil
case *parser.DBytes:
if dir == encoding.Ascending {
return encoding.EncodeStringAscending(b, string(*t)), nil
}
return encoding.EncodeStringDescending(b, string(*t)), nil
case *parser.DDate:
if dir == encoding.Ascending {
return encoding.EncodeVarintAscending(b, int64(*t)), nil
}
return encoding.EncodeVarintDescending(b, int64(*t)), nil
case *parser.DTimestamp:
if dir == encoding.Ascending {
return encoding.EncodeTimeAscending(b, t.Time), nil
}
return encoding.EncodeTimeDescending(b, t.Time), nil
case *parser.DTimestampTZ:
if dir == encoding.Ascending {
return encoding.EncodeTimeAscending(b, t.Time), nil
}
return encoding.EncodeTimeDescending(b, t.Time), nil
case *parser.DInterval:
if dir == encoding.Ascending {
return encoding.EncodeDurationAscending(b, t.Duration)
}
return encoding.EncodeDurationDescending(b, t.Duration)
case *parser.DTuple:
for _, datum := range *t {
var err error
b, err = EncodeTableKey(b, datum, dir)
if err != nil {
return nil, err
}
}
return b, nil
}
return nil, util.Errorf("unable to encode table key: %T", val)
}
// Encodes `val` into `b` and returns the new buffer.
func encodeTableKey(b []byte, val parser.Datum, dir encoding.Direction) ([]byte, *roachpb.Error) {
if (dir != encoding.Ascending) && (dir != encoding.Descending) {
return nil, roachpb.NewErrorf("invalid direction: %d", dir)
}
if val == parser.DNull {
if dir == encoding.Ascending {
return encoding.EncodeNullAscending(b), nil
}
return encoding.EncodeNullDescending(b), nil
}
switch t := val.(type) {
case parser.DBool:
var x int64
if t {
x = 1
} else {
x = 0
}
if dir == encoding.Ascending {
return encoding.EncodeVarintAscending(b, x), nil
}
return encoding.EncodeVarintDescending(b, x), nil
case parser.DInt:
if dir == encoding.Ascending {
return encoding.EncodeVarintAscending(b, int64(t)), nil
}
return encoding.EncodeVarintDescending(b, int64(t)), nil
case parser.DFloat:
if dir == encoding.Ascending {
return encoding.EncodeFloatAscending(b, float64(t)), nil
}
return encoding.EncodeFloatDescending(b, float64(t)), nil
case parser.DDecimal:
if dir == encoding.Ascending {
return encoding.EncodeDecimalAscending(b, t.Decimal), nil
}
return encoding.EncodeDecimalDescending(b, t.Decimal), nil
case parser.DString:
if dir == encoding.Ascending {
return encoding.EncodeStringAscending(b, string(t)), nil
}
return encoding.EncodeStringDescending(b, string(t)), nil
case parser.DBytes:
if dir == encoding.Ascending {
return encoding.EncodeStringAscending(b, string(t)), nil
}
return encoding.EncodeStringDescending(b, string(t)), nil
case parser.DDate:
if dir == encoding.Ascending {
return encoding.EncodeVarintAscending(b, int64(t)), nil
}
return encoding.EncodeVarintDescending(b, int64(t)), nil
case parser.DTimestamp:
if dir == encoding.Ascending {
return encoding.EncodeTimeAscending(b, t.Time), nil
}
return encoding.EncodeTimeDescending(b, t.Time), nil
case parser.DInterval:
if dir == encoding.Ascending {
return encoding.EncodeVarintAscending(b, int64(t.Duration)), nil
}
return encoding.EncodeVarintDescending(b, int64(t.Duration)), nil
}
return nil, roachpb.NewUErrorf("unable to encode table key: %T", val)
}
func TestPrettyPrint(t *testing.T) {
tm, _ := time.Parse(time.RFC3339Nano, "2016-03-30T13:40:35.053725008Z")
duration := duration.Duration{Months: 1, Days: 1, Nanos: 1 * time.Second.Nanoseconds()}
durationAsc, _ := encoding.EncodeDurationAscending(nil, duration)
durationDesc, _ := encoding.EncodeDurationDescending(nil, duration)
txnID := uuid.NewV4()
// The following test cases encode keys with a mixture of ascending and descending direction,
// but always decode keys in the ascending direction. This is why some of the decoded values
// seem bizarre.
testCases := []struct {
key roachpb.Key
exp string
}{
// local
{StoreIdentKey(), "/Local/Store/storeIdent"},
{StoreGossipKey(), "/Local/Store/gossipBootstrap"},
{AbortCacheKey(roachpb.RangeID(1000001), txnID), fmt.Sprintf(`/Local/RangeID/1000001/r/AbortCache/%q`, txnID)},
{RaftTombstoneKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftTombstone"},
{RaftAppliedIndexKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftAppliedIndex"},
{LeaseAppliedIndexKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/LeaseAppliedIndex"},
{RaftTruncatedStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftTruncatedState"},
{RangeLeaderLeaseKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeLeaderLease"},
{RangeStatsKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeStats"},
{RangeFrozenStatusKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeFrozenStatus"},
{RangeLastGCKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeLastGC"},
{RaftHardStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RaftHardState"},
{RaftLastIndexKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RaftLastIndex"},
{RaftLogKey(roachpb.RangeID(1000001), uint64(200001)), "/Local/RangeID/1000001/u/RaftLog/logIndex:200001"},
{RangeLastReplicaGCTimestampKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RangeLastReplicaGCTimestamp"},
{RangeLastVerificationTimestampKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RangeLastVerificationTimestamp"},
{MakeRangeKeyPrefix(roachpb.RKey("ok")), `/Local/Range/"ok"`},
{RangeDescriptorKey(roachpb.RKey("111")), `/Local/Range/"111"/RangeDescriptor`},
{RangeTreeNodeKey(roachpb.RKey("111")), `/Local/Range/"111"/RangeTreeNode`},
{TransactionKey(roachpb.Key("111"), txnID), fmt.Sprintf(`/Local/Range/"111"/Transaction/addrKey:/id:%q`, txnID)},
{LocalMax, `/Meta1/""`}, // LocalMax == Meta1Prefix
// system
{makeKey(Meta2Prefix, roachpb.Key("foo")), `/Meta2/"foo"`},
{makeKey(Meta1Prefix, roachpb.Key("foo")), `/Meta1/"foo"`},
{RangeMetaKey(roachpb.RKey("f")), `/Meta2/"f"`},
{NodeStatusKey(1111), "/System/StatusNode/1111"},
{SystemMax, "/System/Max"},
// key of key
{RangeMetaKey(roachpb.RKey(MakeRangeKeyPrefix(roachpb.RKey("ok")))), `/Meta2/Local/Range/"ok"`},
{RangeMetaKey(roachpb.RKey(makeKey(MakeTablePrefix(42), roachpb.RKey("foo")))), `/Meta2/Table/42/"foo"`},
{RangeMetaKey(roachpb.RKey(makeKey(Meta2Prefix, roachpb.Key("foo")))), `/Meta1/"foo"`},
// table
{UserTableDataMin, "/Table/50"},
{MakeTablePrefix(111), "/Table/111"},
{makeKey(MakeTablePrefix(42), roachpb.RKey("foo")), `/Table/42/"foo"`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeFloatAscending(nil, float64(233.221112)))),
"/Table/42/233.221112"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeFloatDescending(nil, float64(-233.221112)))),
"/Table/42/233.221112"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeFloatAscending(nil, math.Inf(1)))),
"/Table/42/+Inf"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeFloatAscending(nil, math.NaN()))),
"/Table/42/NaN"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeVarintAscending(nil, 1222)),
roachpb.RKey(encoding.EncodeStringAscending(nil, "handsome man"))),
`/Table/42/1222/"handsome man"`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeVarintAscending(nil, 1222))),
`/Table/42/1222`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeVarintDescending(nil, 1222))),
`/Table/42/-1223`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeBytesAscending(nil, []byte{1, 2, 8, 255}))),
`/Table/42/"\x01\x02\b\xff"`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeBytesAscending(nil, []byte{1, 2, 8, 255})),
roachpb.RKey("bar")), `/Table/42/"\x01\x02\b\xff"/"bar"`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeBytesDescending(nil, []byte{1, 2, 8, 255})),
roachpb.RKey("bar")), `/Table/42/"\x01\x02\b\xff"/"bar"`},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeNullAscending(nil))), "/Table/42/NULL"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeNotNullAscending(nil))), "/Table/42/#"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeTimeAscending(nil, tm))),
"/Table/42/2016-03-30T13:40:35.053725008Z"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeTimeDescending(nil, tm))),
"/Table/42/1923-10-04T10:19:23.946274991Z"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeDecimalAscending(nil, inf.NewDec(1234, 2)))),
"/Table/42/12.34"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(encoding.EncodeDecimalDescending(nil, inf.NewDec(1234, 2)))),
"/Table/42/-12.34"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(durationAsc)),
"/Table/42/1m1d1s"},
{makeKey(MakeTablePrefix(42),
roachpb.RKey(durationDesc)),
"/Table/42/-2m-2d743h59m58.999999999s"},
// others
{makeKey([]byte("")), "/Min"},
{Meta1KeyMax, "/Meta1/Max"},
{Meta2KeyMax, "/Meta2/Max"},
{makeKey(MakeTablePrefix(42), roachpb.RKey([]byte{0x12, 'a', 0x00, 0x02})), "/Table/42/<unknown escape sequence: 0x0 0x2>"},
}
for i, test := range testCases {
keyInfo := MassagePrettyPrintedSpanForTest(PrettyPrint(test.key), nil)
exp := MassagePrettyPrintedSpanForTest(test.exp, nil)
if exp != keyInfo {
t.Errorf("%d: expected %s, got %s", i, exp, keyInfo)
}
if exp != MassagePrettyPrintedSpanForTest(test.key.String(), nil) {
t.Errorf("%d: expected %s, got %s", i, exp, test.key.String())
}
parsed, err := UglyPrint(keyInfo)
if err != nil {
if _, ok := err.(*errUglifyUnsupported); !ok {
t.Errorf("%d: %s: %s", i, keyInfo, err)
} else {
t.Logf("%d: skipping parsing of %s; key is unsupported: %v", i, keyInfo, err)
}
} else if exp, act := test.key, parsed; !bytes.Equal(exp, act) {
t.Errorf("%d: expected %q, got %q", i, exp, act)
}
if t.Failed() {
return
}
}
}
func TestClusterFlow(t *testing.T) {
defer leaktest.AfterTest(t)()
const numRows = 100
args := base.TestClusterArgs{ReplicationMode: base.ReplicationManual}
tc := serverutils.StartTestCluster(t, 3, args)
defer tc.Stopper().Stop()
sumDigitsFn := func(row int) parser.Datum {
sum := 0
for row > 0 {
sum += row % 10
row /= 10
}
return parser.NewDInt(parser.DInt(sum))
}
sqlutils.CreateTable(t, tc.ServerConn(0), "t",
"num INT PRIMARY KEY, digitsum INT, numstr STRING, INDEX s (digitsum)",
numRows,
sqlutils.ToRowFn(sqlutils.RowIdxFn, sumDigitsFn, sqlutils.RowEnglishFn))
kvDB := tc.Server(0).KVClient().(*client.DB)
desc := sqlbase.GetTableDescriptor(kvDB, "test", "t")
makeIndexSpan := func(start, end int) TableReaderSpan {
var span roachpb.Span
prefix := roachpb.Key(sqlbase.MakeIndexKeyPrefix(desc, desc.Indexes[0].ID))
span.Key = append(prefix, encoding.EncodeVarintAscending(nil, int64(start))...)
span.EndKey = append(span.EndKey, prefix...)
span.EndKey = append(span.EndKey, encoding.EncodeVarintAscending(nil, int64(end))...)
return TableReaderSpan{Span: span}
}
// Set up table readers on three hosts feeding data into a join reader on
// the third host. This is a basic test for the distributed flow
// infrastructure, including local and remote streams.
//
// Note that the ranges won't necessarily be local to the table readers, but
// that doesn't matter for the purposes of this test.
tr1 := TableReaderSpec{
Table: *desc,
IndexIdx: 1,
OutputColumns: []uint32{0, 1},
Spans: []TableReaderSpan{makeIndexSpan(0, 8)},
}
tr2 := TableReaderSpec{
Table: *desc,
IndexIdx: 1,
OutputColumns: []uint32{0, 1},
Spans: []TableReaderSpan{makeIndexSpan(8, 12)},
}
tr3 := TableReaderSpec{
Table: *desc,
IndexIdx: 1,
OutputColumns: []uint32{0, 1},
Spans: []TableReaderSpan{makeIndexSpan(12, 100)},
}
jr := JoinReaderSpec{
Table: *desc,
OutputColumns: []uint32{2},
}
txn := client.NewTxn(context.Background(), *kvDB)
fid := FlowID{uuid.MakeV4()}
req1 := &SetupFlowRequest{Txn: txn.Proto}
req1.Flow = FlowSpec{
FlowID: fid,
Processors: []ProcessorSpec{{
Core: ProcessorCoreUnion{TableReader: &tr1},
Output: []OutputRouterSpec{{
Type: OutputRouterSpec_MIRROR,
Streams: []StreamEndpointSpec{
{Mailbox: &MailboxSpec{StreamID: 0, TargetAddr: tc.Server(2).ServingAddr()}},
},
}},
}},
}
req2 := &SetupFlowRequest{Txn: txn.Proto}
req2.Flow = FlowSpec{
FlowID: fid,
Processors: []ProcessorSpec{{
Core: ProcessorCoreUnion{TableReader: &tr2},
Output: []OutputRouterSpec{{
Type: OutputRouterSpec_MIRROR,
Streams: []StreamEndpointSpec{
{Mailbox: &MailboxSpec{StreamID: 1, TargetAddr: tc.Server(2).ServingAddr()}},
},
}},
}},
}
req3 := &SetupFlowRequest{Txn: txn.Proto}
req3.Flow = FlowSpec{
FlowID: fid,
Processors: []ProcessorSpec{
{
Core: ProcessorCoreUnion{TableReader: &tr3},
Output: []OutputRouterSpec{{
Type: OutputRouterSpec_MIRROR,
Streams: []StreamEndpointSpec{
{LocalStreamID: LocalStreamID(0)},
},
}},
},
{
Input: []InputSyncSpec{{
Type: InputSyncSpec_ORDERED,
Ordering: Ordering{Columns: []Ordering_Column{{1, Ordering_Column_ASC}}},
Streams: []StreamEndpointSpec{
{Mailbox: &MailboxSpec{StreamID: 0}},
{Mailbox: &MailboxSpec{StreamID: 1}},
{LocalStreamID: LocalStreamID(0)},
},
}},
Core: ProcessorCoreUnion{JoinReader: &jr},
Output: []OutputRouterSpec{{
Type: OutputRouterSpec_MIRROR,
Streams: []StreamEndpointSpec{{Mailbox: &MailboxSpec{SimpleResponse: true}}},
}}},
},
}
var clients []DistSQLClient
for i := 0; i < 3; i++ {
s := tc.Server(i)
conn, err := s.RPCContext().GRPCDial(s.ServingAddr())
if err != nil {
t.Fatal(err)
}
clients = append(clients, NewDistSQLClient(conn))
}
ctx := context.Background()
if log.V(1) {
log.Infof(ctx, "Setting up flow on 0")
}
if resp, err := clients[0].SetupFlow(context.Background(), req1); err != nil {
t.Fatal(err)
} else if resp.Error != nil {
t.Fatal(resp.Error)
}
if log.V(1) {
log.Infof(ctx, "Setting up flow on 1")
}
if resp, err := clients[1].SetupFlow(context.Background(), req2); err != nil {
t.Fatal(err)
} else if resp.Error != nil {
t.Fatal(resp.Error)
}
if log.V(1) {
log.Infof(ctx, "Running flow on 2")
}
stream, err := clients[2].RunSimpleFlow(context.Background(), req3)
if err != nil {
t.Fatal(err)
}
var decoder StreamDecoder
var rows sqlbase.EncDatumRows
for {
msg, err := stream.Recv()
if err != nil {
if err == io.EOF {
break
}
t.Fatal(err)
}
err = decoder.AddMessage(msg)
if err != nil {
t.Fatal(err)
}
rows = testGetDecodedRows(t, &decoder, rows)
}
if done, trailerErr := decoder.IsDone(); !done {
t.Fatal("stream not done")
} else if trailerErr != nil {
t.Fatal("error in the stream trailer:", trailerErr)
}
// The result should be all the numbers in string form, ordered by the
// digit sum (and then by number).
var results []string
for sum := 1; sum <= 50; sum++ {
for i := 1; i <= numRows; i++ {
if int(*sumDigitsFn(i).(*parser.DInt)) == sum {
results = append(results, fmt.Sprintf("['%s']", sqlutils.IntToEnglish(i)))
}
}
}
expected := strings.Join(results, " ")
expected = "[" + expected + "]"
if rowStr := rows.String(); rowStr != expected {
t.Errorf("Result: %s\n Expected: %s\n", rowStr, expected)
}
}
func TestTableReader(t *testing.T) {
defer leaktest.AfterTest(t)()
ctx, _ := createTestServerContext()
server, sqlDB, kvDB := setupWithContext(t, ctx)
defer cleanup(server, sqlDB)
if _, err := sqlDB.Exec(`
CREATE DATABASE test;
CREATE TABLE test.t (a INT PRIMARY KEY, b INT, c INT, d INT, INDEX bc (b, c));
INSERT INTO test.t VALUES (1, 10, 11, 12), (2, 20, 21, 22), (3, 30, 31, 32);
INSERT INTO test.t VALUES (4, 60, 61, 62), (5, 50, 51, 52), (6, 40, 41, 42);
`); err != nil {
t.Fatal(err)
}
td := getTableDescriptor(kvDB, "test", "t")
ts := sql.TableReaderSpec{
Table: *td,
IndexIdx: 0,
Reverse: false,
Spans: nil,
Filter: sql.Expression{Expr: "$2 != 21"}, // c != 21
OutputColumns: []uint32{0, 3}, // a, d
}
txn := client.NewTxn(context.Background(), *kvDB)
tr, err := sql.NewTableReader(&ts, txn, parser.EvalContext{})
if err != nil {
t.Fatal(err)
}
pErr := tr.Run()
if pErr != nil {
t.Fatal(pErr)
}
// TODO(radu): currently the table reader just prints out stuff; when it
// will output results we will be able to verify them.
// Expected output:
// RESULT: 1 <skipped> 11 12
// RESULT: 3 <skipped> 31 32
// RESULT: 4 <skipped> 61 62
// RESULT: 5 <skipped> 51 52
// RESULT: 6 <skipped> 41 42
// Read using the bc index
var span roachpb.Span
span.Key = roachpb.Key(sqlbase.MakeIndexKeyPrefix(td.ID, td.Indexes[0].ID))
span.EndKey = append(span.Key, encoding.EncodeVarintAscending(nil, 50)...)
ts = sql.TableReaderSpec{
Table: *td,
IndexIdx: 1,
Reverse: true,
Spans: []sql.TableReaderSpan{{Span: span}},
Filter: sql.Expression{Expr: "$1 != 30"}, // b != 30
OutputColumns: []uint32{0, 1}, // a, c
}
tr, err = sql.NewTableReader(&ts, txn, parser.EvalContext{})
if err != nil {
t.Fatal(err)
}
pErr = tr.Run()
if pErr != nil {
t.Fatal(pErr)
}
// Expected output:
// RESULT: 6 40 41 <skipped>
// RESULT: 2 20 21 <skipped>
// RESULT: 1 10 11 <skipped>
}