// MakeZoneKey returns the key for 'id's entry in the system.zones table.
func MakeZoneKey(id ID) roachpb.Key {
k := keys.MakeTablePrefix(uint32(ZonesTable.ID))
k = encoding.EncodeUvarint(k, uint64(ZonesTable.PrimaryIndex.ID))
k = encoding.EncodeUvarint(k, uint64(id))
k = encoding.EncodeUvarint(k, uint64(ZonesTable.Columns[1].ID))
return k
}
func TestMakeTableIndexKey(t *testing.T) {
defer leaktest.AfterTest(t)
key := MakeTableIndexKey(12, 345, []byte("foo"), []byte("bar"))
expKey := MakeKey(TableDataPrefix, encoding.EncodeUvarint(nil, 12), encoding.EncodeUvarint(nil, 345), encoding.EncodeBytes(nil, []byte("foo")), encoding.EncodeBytes(nil, []byte("bar")))
if !key.Equal(expKey) {
t.Errorf("key %q doesn't match expected %q", key, expKey)
}
// Check that keys are ordered
keys := []proto.Key{
MakeTableIndexKey(0, 0, []byte("foo")),
MakeTableIndexKey(0, 0, []byte("fooo")),
MakeTableIndexKey(0, 1, []byte("bar")),
MakeTableIndexKey(1, 0, []byte("bar")),
MakeTableIndexKey(1, 0, []byte("bar"), []byte("foo")),
MakeTableIndexKey(1, 1, []byte("bar"), []byte("fo")),
MakeTableIndexKey(1, 1, []byte("bar"), []byte("foo")),
MakeTableIndexKey(1, 2, []byte("bar")),
MakeTableIndexKey(2, 2, []byte("ba")),
}
for i := 1; i < len(keys); i++ {
if bytes.Compare(keys[i-1], keys[i]) >= 0 {
t.Errorf("key %d >= key %d", i-1, i)
}
}
}
func newRangeDataIterator(d *proto.RangeDescriptor, e engine.Engine) *rangeDataIterator {
// The first range in the keyspace starts at KeyMin, which includes the node-local
// space. We need the original StartKey to find the range metadata, but the
// actual data starts at LocalMax.
dataStartKey := d.StartKey
if d.StartKey.Equal(proto.KeyMin) {
dataStartKey = keys.LocalMax
}
ri := &rangeDataIterator{
ranges: []keyRange{
{
start: engine.MVCCEncodeKey(keys.MakeKey(keys.LocalRangeIDPrefix, encoding.EncodeUvarint(nil, uint64(d.RangeID)))),
end: engine.MVCCEncodeKey(keys.MakeKey(keys.LocalRangeIDPrefix, encoding.EncodeUvarint(nil, uint64(d.RangeID+1)))),
},
{
start: engine.MVCCEncodeKey(keys.MakeKey(keys.LocalRangePrefix, encoding.EncodeBytes(nil, d.StartKey))),
end: engine.MVCCEncodeKey(keys.MakeKey(keys.LocalRangePrefix, encoding.EncodeBytes(nil, d.EndKey))),
},
{
start: engine.MVCCEncodeKey(dataStartKey),
end: engine.MVCCEncodeKey(d.EndKey),
},
},
iter: e.NewIterator(),
}
ri.iter.Seek(ri.ranges[ri.curIndex].start)
ri.advance()
return ri
}
func sqlKV(tableID uint32, indexID, descriptorID uint64) roachpb.KeyValue {
k := keys.MakeTablePrefix(tableID)
k = encoding.EncodeUvarint(k, indexID)
k = encoding.EncodeUvarint(k, descriptorID)
k = encoding.EncodeUvarint(k, 12345) // Column ID, but could be anything.
return kv(k, nil)
}
// runClientScan first creates test data (and resets the benchmarking
// timer). It then performs b.N client scans in increments of numRows
// keys over all of the data, restarting at the beginning of the
// keyspace, as many times as necessary.
func runClientScan(useSSL bool, numRows, numVersions int, b *testing.B) {
const numKeys = 100000
s, db := setupClientBenchData(useSSL, numVersions, numKeys, b)
defer s.Stop()
b.SetBytes(int64(numRows * valueSize))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
startKeyBuf := append(make([]byte, 0, 64), []byte("key-")...)
endKeyBuf := append(make([]byte, 0, 64), []byte("key-")...)
for pb.Next() {
// Choose a random key to start scan.
keyIdx := rand.Int31n(int32(numKeys - numRows))
startKey := roachpb.Key(encoding.EncodeUvarint(startKeyBuf, uint64(keyIdx)))
endKey := roachpb.Key(encoding.EncodeUvarint(endKeyBuf, uint64(keyIdx)+uint64(numRows)))
rows, err := db.Scan(startKey, endKey, int64(numRows))
if err != nil {
b.Fatalf("failed scan: %s", err)
}
if len(rows) != numRows {
b.Fatalf("failed to scan: %d != %d", len(rows), numRows)
}
}
})
b.StopTimer()
}
// MakeDescMetadataKey returns the key for the descriptor.
func MakeDescMetadataKey(descID ID) proto.Key {
k := MakeTablePrefix(DescriptorTable.ID)
k = encoding.EncodeUvarint(k, uint64(DescriptorTable.PrimaryIndex.ID))
k = encoding.EncodeUvarint(k, uint64(descID))
k = encoding.EncodeUvarint(k, uint64(DescriptorTable.Columns[1].ID))
return k
}
func encodeIndexKeyPrefix(tableID, indexID uint32) []byte {
var key []byte
key = append(key, keys.TableDataPrefix...)
key = encoding.EncodeUvarint(key, uint64(tableID))
key = encoding.EncodeUvarint(key, uint64(indexID))
return key
}
func newRangeDataIterator(r *Range, e engine.Engine) *rangeDataIterator {
r.RLock()
startKey := r.Desc().StartKey
if startKey.Equal(engine.KeyMin) {
startKey = engine.KeyLocalMax
}
endKey := r.Desc().EndKey
r.RUnlock()
ri := &rangeDataIterator{
ranges: []keyRange{
{
start: engine.MVCCEncodeKey(engine.MakeKey(engine.KeyLocalRangeIDPrefix, encoding.EncodeUvarint(nil, uint64(r.Desc().RaftID)))),
end: engine.MVCCEncodeKey(engine.MakeKey(engine.KeyLocalRangeIDPrefix, encoding.EncodeUvarint(nil, uint64(r.Desc().RaftID+1)))),
},
{
start: engine.MVCCEncodeKey(engine.MakeKey(engine.KeyLocalRangeKeyPrefix, encoding.EncodeBytes(nil, startKey))),
end: engine.MVCCEncodeKey(engine.MakeKey(engine.KeyLocalRangeKeyPrefix, encoding.EncodeBytes(nil, endKey))),
},
{
start: engine.MVCCEncodeKey(startKey),
end: engine.MVCCEncodeKey(endKey),
},
},
iter: e.NewIterator(),
}
ri.iter.Seek(ri.ranges[ri.curIndex].start)
ri.advance()
return ri
}
// MakeDescMetadataKey returns the key for the descriptor.
func MakeDescMetadataKey(descID ID) roachpb.Key {
k := keys.MakeTablePrefix(uint32(DescriptorTable.ID))
k = encoding.EncodeUvarint(k, uint64(DescriptorTable.PrimaryIndex.ID))
k = encoding.EncodeUvarint(k, uint64(descID))
k = encoding.EncodeUvarint(k, uint64(DescriptorTable.Columns[1].ID))
return k
}
// populateTableIndexKey populates the key passed in with the
// order encoded values forming the index key.
func populateTableIndexKey(key []byte, tableID, indexID uint32, columnValues ...[]byte) []byte {
key = append(key, TableDataPrefix...)
key = encoding.EncodeUvarint(key, uint64(tableID))
key = encoding.EncodeUvarint(key, uint64(indexID))
for _, value := range columnValues {
key = encoding.EncodeBytes(key, value)
}
return key
}
// MakeColumnKey returns the key for the column in the given row.
func MakeColumnKey(rowKey []byte, colID uint32) []byte {
key := append([]byte(nil), rowKey...)
size := len(key)
key = encoding.EncodeUvarint(key, uint64(colID))
// Note that we assume that `len(key)-size` will always be encoded to a
// single byte by EncodeUvarint. This is currently always true because the
// varint encoding will encode 1-9 bytes.
return encoding.EncodeUvarint(key, uint64(len(key)-size))
}
// MakeNameMetadataKey returns the key for the name. Pass name == "" in order
// to generate the prefix key to use to scan over all of the names for the
// specified parentID.
func MakeNameMetadataKey(parentID ID, name string) roachpb.Key {
name = normalizeName(name)
k := keys.MakeTablePrefix(uint32(NamespaceTable.ID))
k = encoding.EncodeUvarint(k, uint64(NamespaceTable.PrimaryIndex.ID))
k = encoding.EncodeUvarint(k, uint64(parentID))
if name != "" {
k = encoding.EncodeBytes(k, []byte(name))
k = keys.MakeColumnKey(k, uint32(NamespaceTable.Columns[2].ID))
}
return k
}
// runMVCCScan first creates test data (and resets the benchmarking
// timer). It then performs b.N MVCCScans in increments of numRows
// keys over all of the data in the rocksdb instance, restarting at
// the beginning of the keyspace, as many times as necessary.
func runMVCCScan(numRows, numVersions int, b *testing.B) {
// Use the same number of keys for all of the mvcc scan
// benchmarks. Using a different number of keys per test gives
// preferential treatment to tests with fewer keys. Note that the
// datasets all fit in cache and the cache is pre-warmed.
const numKeys = 100000
rocksdb := setupMVCCScanData(numVersions, numKeys, b)
defer rocksdb.Stop()
prewarmCache(rocksdb)
b.SetBytes(int64(numRows * 1024))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
// Choose a random key to start scan.
keyIdx := rand.Int31n(int32(numKeys - numRows))
startKey := proto.Key(encoding.EncodeUvarint([]byte("key-"), uint64(keyIdx)))
walltime := int64(5 * (rand.Int31n(int32(numVersions)) + 1))
ts := makeTS(walltime, 0)
kvs, err := MVCCScan(rocksdb, startKey, KeyMax, int64(numRows), ts, nil)
if err != nil {
b.Fatalf("failed scan: %s", err)
}
if len(kvs) != numRows {
b.Fatalf("failed to scan: %d != %d", len(kvs), numRows)
}
}
})
b.StopTimer()
}
// MakeNameMetadataKey returns the key for the namespace.
func MakeNameMetadataKey(parentID uint32, name string) proto.Key {
k := make([]byte, 0, len(NameMetadataPrefix)+encoding.MaxUvarintSize+len(name))
k = append(k, NameMetadataPrefix...)
k = encoding.EncodeUvarint(k, uint64(parentID))
k = append(k, name...)
return k
}
// encodeTableKey encodes a single element of a table key, appending the
// encoded value to b.
func encodeTableKey(b []byte, v reflect.Value) ([]byte, error) {
switch t := v.Interface().(type) {
case []byte:
return roachencoding.EncodeBytes(b, t), nil
case string:
return roachencoding.EncodeBytes(b, []byte(t)), nil
}
switch v.Kind() {
case reflect.Bool:
if v.Bool() {
return roachencoding.EncodeVarint(b, 1), nil
}
return roachencoding.EncodeVarint(b, 0), nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return roachencoding.EncodeVarint(b, v.Int()), nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return roachencoding.EncodeUvarint(b, v.Uint()), nil
case reflect.Float32, reflect.Float64:
return roachencoding.EncodeNumericFloat(b, v.Float()), nil
case reflect.String:
return roachencoding.EncodeBytes(b, []byte(v.String())), nil
}
return nil, fmt.Errorf("unable to encode key: %s", v)
}
func TestMakeTableDataKey(t *testing.T) {
defer leaktest.AfterTest(t)
key := MakeTableDataKey(12, 345, 6, []byte("foo"), []byte("bar"))
// Expected key is the TableIndexKey + ColumnID
expKey := MakeKey(MakeTableIndexKey(12, 345, []byte("foo"), []byte("bar")), encoding.EncodeUvarint(nil, 6))
if !key.Equal(expKey) {
t.Errorf("key %q doesn't match expected %q", key, expKey)
}
// Check that keys are ordered
keys := []proto.Key{
// Data-key follows Index key order
MakeTableDataKey(0, 0, 0, []byte("foo")),
MakeTableDataKey(0, 0, 8, []byte("fooo")),
MakeTableDataKey(0, 1, 10, []byte("bar")),
MakeTableDataKey(1, 0, 3, []byte("bar")),
MakeTableDataKey(1, 0, 5, []byte("bar"), []byte("foo")),
MakeTableDataKey(1, 1, 7, []byte("bar"), []byte("fo")),
MakeTableDataKey(1, 1, 4, []byte("bar"), []byte("foo")),
MakeTableDataKey(1, 2, 89, []byte("bar")),
MakeTableDataKey(2, 2, 23, []byte("ba")),
// For the same Index key, Data-key follows column ID order.
MakeTableDataKey(2, 2, 0, []byte("bar"), []byte("foo")),
MakeTableDataKey(2, 2, 7, []byte("bar"), []byte("foo")),
MakeTableDataKey(2, 2, 23, []byte("bar"), []byte("foo")),
MakeTableDataKey(2, 2, 45, []byte("bar"), []byte("foo")),
}
for i := 1; i < len(keys); i++ {
if bytes.Compare(keys[i-1], keys[i]) >= 0 {
t.Errorf("key %d >= key %d", i-1, i)
}
}
}
func runMVCCBatchPut(valueSize, batchSize int, b *testing.B) {
rng := util.NewPseudoRand()
value := proto.Value{Bytes: util.RandBytes(rng, valueSize)}
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
rocksdb := NewInMem(proto.Attributes{Attrs: []string{"ssd"}}, testCacheSize)
defer rocksdb.Stop()
b.SetBytes(int64(valueSize))
b.ResetTimer()
for i := 0; i < b.N; i += batchSize {
end := i + batchSize
if end > b.N {
end = b.N
}
batch := rocksdb.NewBatch()
for j := i; j < end; j++ {
key := proto.Key(encoding.EncodeUvarint(keyBuf[0:4], uint64(j)))
ts := makeTS(time.Now().UnixNano(), 0)
if err := MVCCPut(batch, nil, key, ts, value, nil); err != nil {
b.Fatalf("failed put: %s", err)
}
}
if err := batch.Commit(); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
}
// runMVCCGet first creates test data (and resets the benchmarking
// timer). It then performs b.N MVCCGets.
func runMVCCGet(numVersions, valueSize int, b *testing.B) {
const overhead = 48 // Per key/value overhead (empirically determined)
const targetSize = 512 << 20 // 512 MB
// Adjust the number of keys so that each test has approximately the same
// amount of data.
numKeys := targetSize / ((overhead + valueSize) * (1 + (numVersions-1)/2))
rocksdb, stopper := setupMVCCData(numVersions, numKeys, valueSize, b)
defer stopper.Stop()
b.SetBytes(int64(valueSize))
b.ResetTimer()
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
for i := 0; i < b.N; i++ {
// Choose a random key to retrieve.
keyIdx := rand.Int31n(int32(numKeys))
key := roachpb.Key(encoding.EncodeUvarint(keyBuf[:4], uint64(keyIdx)))
walltime := int64(5 * (rand.Int31n(int32(numVersions)) + 1))
ts := makeTS(walltime, 0)
if v, _, err := MVCCGet(rocksdb, key, ts, true, nil); err != nil {
b.Fatalf("failed get: %s", err)
} else if v == nil {
b.Fatalf("failed get (key not found): %d@%d", keyIdx, walltime)
} else if valueBytes, err := v.GetBytes(); err != nil {
b.Fatal(err)
} else if len(valueBytes) != valueSize {
b.Fatalf("unexpected value size: %d", len(valueBytes))
}
}
b.StopTimer()
}
// runMVCCGet first creates test data (and resets the benchmarking
// timer). It then performs b.N MVCCGets.
func runMVCCGet(numVersions int, b *testing.B) {
// Use the same number of keys for all of the mvcc get
// benchmarks. Using a different number of keys per test gives
// preferential treatment to tests with fewer keys. Note that the
// datasets all fit in cache and the cache is pre-warmed.
const numKeys = 100000
rocksdb := setupMVCCScanData(numVersions, numKeys, b)
defer rocksdb.Close()
prewarmCache(rocksdb)
b.SetBytes(1024)
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
for pb.Next() {
// Choose a random key to retrieve.
keyIdx := rand.Int31n(int32(numKeys))
key := proto.Key(encoding.EncodeUvarint(keyBuf[0:4], uint64(keyIdx)))
walltime := int64(5 * (rand.Int31n(int32(numVersions)) + 1))
ts := makeTS(walltime, 0)
if v, _, err := MVCCGet(rocksdb, key, ts, true, nil); err != nil {
b.Fatalf("failed get: %s", err)
} else if len(v.Bytes) != 1024 {
b.Fatalf("unexpected value size: %d", len(v.Bytes))
}
}
})
b.StopTimer()
}
// runMVCCScan first creates test data (and resets the benchmarking
// timer). It then performs b.N MVCCScans in increments of numRows
// keys over all of the data in the rocksdb instance, restarting at
// the beginning of the keyspace, as many times as necessary.
func runMVCCScan(numRows, numVersions, valueSize int, b *testing.B) {
// Use the same number of keys for all of the mvcc scan
// benchmarks. Using a different number of keys per test gives
// preferential treatment to tests with fewer keys. Note that the
// datasets all fit in cache and the cache is pre-warmed.
const numKeys = 100000
rocksdb, stopper := setupMVCCData(numVersions, numKeys, valueSize, b)
defer stopper.Stop()
b.SetBytes(int64(numRows * valueSize))
b.ResetTimer()
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
for i := 0; i < b.N; i++ {
// Choose a random key to start scan.
keyIdx := rand.Int31n(int32(numKeys - numRows))
startKey := roachpb.Key(encoding.EncodeUvarint(keyBuf[:4], uint64(keyIdx)))
walltime := int64(5 * (rand.Int31n(int32(numVersions)) + 1))
ts := makeTS(walltime, 0)
kvs, _, err := MVCCScan(rocksdb, startKey, keyMax, int64(numRows), ts, true, nil)
if err != nil {
b.Fatalf("failed scan: %s", err)
}
if len(kvs) != numRows {
b.Fatalf("failed to scan: %d != %d", len(kvs), numRows)
}
}
b.StopTimer()
}
// MakeRangeIDKey creates a range-local key based on the range's
// Range ID, metadata key suffix, and optional detail (e.g. the
// encoded command ID for a response cache entry, etc.).
func MakeRangeIDKey(rangeID proto.RangeID, suffix, detail proto.Key) proto.Key {
if len(suffix) != LocalSuffixLength {
panic(fmt.Sprintf("suffix len(%q) != %d", suffix, LocalSuffixLength))
}
return MakeKey(LocalRangeIDPrefix,
encoding.EncodeUvarint(nil, uint64(rangeID)), suffix, detail)
}
func TestInsert(t *testing.T) {
defer leaktest.AfterTest(t)
s, sqlDB, kvDB := setup(t)
defer cleanup(s, sqlDB)
if _, err := sqlDB.Exec(`
CREATE DATABASE t;
CREATE TABLE t.kv (
k CHAR PRIMARY KEY,
v CHAR,
CONSTRAINT a INDEX (k, v),
CONSTRAINT b UNIQUE (v)
);
INSERT INTO t.kv VALUES ('c', 'e'), ('a', 'c'), ('b', 'd');
`); err != nil {
t.Fatal(err)
}
// The first `MaxReservedDescID` (plus 0) are set aside.
nameKey := sql.MakeNameMetadataKey(sql.MaxReservedDescID+1, "kv")
gr, err := kvDB.Get(nameKey)
if err != nil {
t.Fatal(err)
}
if !gr.Exists() {
t.Fatalf("TableDescriptor %q does not exist", nameKey)
}
descKey := sql.MakeDescMetadataKey(sql.ID(gr.ValueInt()))
desc := sql.TableDescriptor{}
if err := kvDB.GetProto(descKey, &desc); err != nil {
t.Fatal(err)
}
var tablePrefix []byte
tablePrefix = append(tablePrefix, keys.TableDataPrefix...)
tablePrefix = encoding.EncodeUvarint(tablePrefix, uint64(desc.ID))
tableStartKey := proto.Key(tablePrefix)
tableEndKey := tableStartKey.PrefixEnd()
if kvs, err := kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil {
t.Fatal(err)
} else if l := 12; len(kvs) != l {
t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs))
}
// Insert that will fail after the index `a` is written.
if _, err := sqlDB.Exec(`INSERT INTO t.kv VALUES ('d', 'd')`); !testutils.IsError(err, "duplicate key value .* violates unique constraint") {
t.Fatalf("unexpected error %s", err)
}
// Verify nothing was written.
if kvs, err := kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil {
t.Fatal(err)
} else if l := 12; len(kvs) != l {
t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs))
}
}
// MakeNameMetadataKey returns the key for the name.
func MakeNameMetadataKey(parentID ID, name string) proto.Key {
name = strings.ToLower(name)
k := make([]byte, 0, len(keys.NameMetadataPrefix)+encoding.MaxUvarintSize+len(name))
k = append(k, keys.NameMetadataPrefix...)
k = encoding.EncodeUvarint(k, uint64(parentID))
k = append(k, name...)
return k
}
// ResponseCacheKey returns a range-local key by Raft ID for a
// response cache entry, with detail specified by encoding the
// supplied client command ID.
func ResponseCacheKey(raftID int64, cmdID *proto.ClientCmdID) proto.Key {
detail := proto.Key{}
if cmdID != nil {
detail = encoding.EncodeUvarint(nil, uint64(cmdID.WallTime)) // wall time helps sort for locality
detail = encoding.EncodeUint64(detail, uint64(cmdID.Random))
}
return MakeRangeIDKey(raftID, KeyLocalResponseCacheSuffix, detail)
}
// ResponseCacheKey returns a range-local key by Range ID for a
// response cache entry, with detail specified by encoding the
// supplied client command ID.
func ResponseCacheKey(rangeID roachpb.RangeID, cmdID *roachpb.ClientCmdID) roachpb.Key {
detail := roachpb.RKey{}
if cmdID != nil {
// Wall time helps sort for locality.
detail = encoding.EncodeUvarint(nil, uint64(cmdID.WallTime))
detail = encoding.EncodeUint64(detail, uint64(cmdID.Random))
}
return MakeRangeIDKey(rangeID, LocalResponseCacheSuffix, detail)
}
func TestDelete(t *testing.T) {
defer leaktest.AfterTest(t)
s, sqlDB, kvDB := setup(t)
defer cleanup(s, sqlDB)
if _, err := sqlDB.Exec(`
CREATE DATABASE t;
CREATE TABLE t.kv (
k CHAR PRIMARY KEY,
v CHAR,
CONSTRAINT a UNIQUE (v)
);
INSERT INTO t.kv VALUES ('c', 'e'), ('a', 'c'), ('b', 'd');
`); err != nil {
t.Fatal(err)
}
// The first `MaxReservedDescID` (plus 0) are set aside.
nameKey := sql.MakeNameMetadataKey(sql.MaxReservedDescID+1, "kv")
gr, err := kvDB.Get(nameKey)
if err != nil {
t.Fatal(err)
}
if !gr.Exists() {
t.Fatalf("TableDescriptor %q does not exist", nameKey)
}
descKey := sql.MakeDescMetadataKey(sql.ID(gr.ValueInt()))
desc := sql.TableDescriptor{}
if err := kvDB.GetProto(descKey, &desc); err != nil {
t.Fatal(err)
}
var tablePrefix []byte
tablePrefix = append(tablePrefix, keys.TableDataPrefix...)
tablePrefix = encoding.EncodeUvarint(tablePrefix, uint64(desc.ID))
tableStartKey := proto.Key(tablePrefix)
tableEndKey := tableStartKey.PrefixEnd()
if kvs, err := kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil {
t.Fatal(err)
} else if l := 9; len(kvs) != l {
t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs))
}
if _, err := sqlDB.Exec(`DELETE FROM t.kv`); err != nil {
t.Fatal(err)
}
if kvs, err := kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil {
t.Fatal(err)
} else if l := 0; len(kvs) != l {
t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs))
}
}
func TestMakeSplitKey(t *testing.T) {
e := func(vals ...uint64) roachpb.Key {
var k roachpb.Key
for _, v := range vals {
k = encoding.EncodeUvarint(k, v)
}
return k
}
goodData := []struct {
in roachpb.Key
expected roachpb.Key
}{
{e(1, 2, 0), e(1, 2)}, // /Table/1/2/0 -> /Table/1/2
{e(1, 2, 1), e(1)}, // /Table/1/2/1 -> /Table/1
{e(1, 2, 2), e()}, // /Table/1/2/2 -> /Table
{e(1, 2, 3, 0), e(1, 2, 3)}, // /Table/1/2/3/0 -> /Table/1/2/3
{e(1, 2, 3, 1), e(1, 2)}, // /Table/1/2/3/1 -> /Table/1/2
{e(1, 2, 200, 2), e(1, 2)}, // /Table/1/2/200/2 -> /Table/1/2
{e(1, 2, 3, 4, 1), e(1, 2, 3)}, // /Table/1/2/3/4/1 -> /Table/1/2/3
}
for i, d := range goodData {
out, err := MakeSplitKey(d.in)
if err != nil {
t.Fatalf("%d: %s: unexpected error: %v", i, d.in, err)
}
if !d.expected.Equal(out) {
t.Fatalf("%d: %s: expected %s, but got %s", i, d.in, d.expected, out)
}
}
errorData := []struct {
in roachpb.Key
err string
}{
// Column ID suffix size is too large.
{e(1), "malformed table key"},
{e(1, 2), "malformed table key"},
// The table ID is invalid.
{e(200)[:1], "insufficient bytes to decode uvarint value"},
// The index ID is invalid.
{e(1, 200)[:2], "insufficient bytes to decode uvarint value"},
// The column ID suffix is invalid.
{e(1, 2, 200)[:3], "insufficient bytes to decode uvarint value"},
}
for i, d := range errorData {
_, err := MakeSplitKey(d.in)
if !testutils.IsError(err, d.err) {
t.Fatalf("%d: %s: expected %s, but got %v", i, d.in, d.err, err)
}
}
}
// setupMVCCData writes up to numVersions values at each of numKeys
// keys. The number of versions written for each key is chosen
// randomly according to a uniform distribution. Each successive
// version is written starting at 5ns and then in 5ns increments. This
// allows scans at various times, starting at t=5ns, and continuing to
// t=5ns*(numVersions+1). A version for each key will be read on every
// such scan, but the dynamics of the scan will change depending on
// the historical timestamp. Earlier timestamps mean scans which must
// skip more historical versions; later timestamps mean scans which
// skip fewer.
//
// The creation of the rocksdb database is time consuming, especially
// for larger numbers of versions. The database is persisted between
// runs and stored in the current directory as
// "mvcc_scan_<versions>_<keys>_<valueBytes>".
func setupMVCCScanData(numVersions, numKeys, valueBytes int, b *testing.B) (*RocksDB, *stop.Stopper) {
loc := fmt.Sprintf("mvcc_scan_%d_%d_%d", numVersions, numKeys, valueBytes)
exists := true
if _, err := os.Stat(loc); os.IsNotExist(err) {
exists = false
}
const cacheSize = 8 << 30 // 8 GB
stopper := stop.NewStopper()
rocksdb := NewRocksDB(roachpb.Attributes{}, loc, cacheSize, stopper)
if err := rocksdb.Open(); err != nil {
b.Fatalf("could not create new rocksdb db instance at %s: %v", loc, err)
}
if exists {
return rocksdb, stopper
}
log.Infof("creating mvcc data: %s", loc)
rng, _ := randutil.NewPseudoRand()
keys := make([]roachpb.Key, numKeys)
nvs := make([]int, numKeys)
for t := 1; t <= numVersions; t++ {
walltime := int64(5 * t)
ts := makeTS(walltime, 0)
batch := rocksdb.NewBatch()
for i := 0; i < numKeys; i++ {
if t == 1 {
keys[i] = roachpb.Key(encoding.EncodeUvarint([]byte("key-"), uint64(i)))
nvs[i] = rand.Intn(numVersions) + 1
}
// Only write values if this iteration is less than the random
// number of versions chosen for this key.
if t <= nvs[i] {
value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueBytes))
value.InitChecksum(keys[i])
if err := MVCCPut(batch, nil, keys[i], ts, value, nil); err != nil {
b.Fatal(err)
}
}
}
if err := batch.Commit(); err != nil {
b.Fatal(err)
}
batch.Close()
}
rocksdb.CompactRange(nil, nil)
return rocksdb, stopper
}
// checkTableSize checks that the number of key:value pairs stored
// in the table equals e.
func (mt mutationTest) checkTableSize(e int) {
// Check that there are no hidden values
var tablePrefix []byte
tablePrefix = append(tablePrefix, keys.TableDataPrefix...)
tableDesc := mt.desc.GetTable()
tablePrefix = encoding.EncodeUvarint(tablePrefix, uint64(tableDesc.ID))
tableStartKey := roachpb.Key(tablePrefix)
tableEndKey := tableStartKey.PrefixEnd()
if kvs, err := mt.kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil {
mt.Fatal(err)
} else if len(kvs) != e {
mt.Fatalf("expected %d key value pairs, but got %d", e, len(kvs))
}
}
// encodePrimaryKey encodes a primary key for the table using the model object
// v. It returns the encoded primary key.
func (m *model) encodePrimaryKey(v reflect.Value) ([]byte, error) {
var key []byte
key = append(key, keys.TableDataPrefix...)
key = roachencoding.EncodeUvarint(key, uint64(m.desc.ID))
for _, col := range m.primaryKey {
var err error
key, err = encodeTableKey(key, v.FieldByIndex(col.field.Index))
if err != nil {
return nil, err
}
}
return key, nil
}