func runMVCCConditionalPut(emk engineMaker, valueSize int, createFirst bool, b *testing.B) {
rng, _ := randutil.NewPseudoRand()
value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueSize))
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
eng := emk(b, fmt.Sprintf("cput_%d", valueSize))
defer eng.Close()
b.SetBytes(int64(valueSize))
var expected *roachpb.Value
if createFirst {
for i := 0; i < b.N; i++ {
key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(i)))
ts := makeTS(timeutil.Now().UnixNano(), 0)
if err := MVCCPut(context.Background(), eng, nil, key, ts, value, nil); err != nil {
b.Fatalf("failed put: %s", err)
}
}
expected = &value
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(i)))
ts := makeTS(timeutil.Now().UnixNano(), 0)
if err := MVCCConditionalPut(context.Background(), eng, nil, key, ts, value, expected, nil); err != nil {
b.Fatalf("failed put: %s", err)
}
}
b.StopTimer()
}
func sqlKV(tableID uint32, indexID, descriptorID uint64) roachpb.KeyValue {
k := keys.MakeTablePrefix(tableID)
k = encoding.EncodeUvarintAscending(k, indexID)
k = encoding.EncodeUvarintAscending(k, descriptorID)
k = encoding.EncodeUvarintAscending(k, 12345) // Column ID, but could be anything.
return kv(k, nil)
}
// MakeFamilyKey returns the key for the family in the given row by appending to
// the passed key. If SentinelFamilyID is passed, a sentinel key (which is the
// first key in a sql table row) is returned.
func MakeFamilyKey(key []byte, famID uint32) []byte {
if famID == SentinelFamilyID {
return encoding.EncodeUvarintAscending(key, 0)
}
size := len(key)
key = encoding.EncodeUvarintAscending(key, uint64(famID))
// Note that we assume that `len(key)-size` will always be encoded to a
// single byte by EncodeUvarint. This is currently always true because the
// varint encoding will encode 1-9 bytes.
return encoding.EncodeUvarintAscending(key, uint64(len(key)-size))
}
// MakeIndexKeyPrefix returns the key prefix used for the index's data.
func MakeIndexKeyPrefix(desc *TableDescriptor, indexID IndexID) []byte {
var key []byte
if i, err := desc.FindIndexByID(indexID); err == nil && len(i.Interleave.Ancestors) > 0 {
key = encoding.EncodeUvarintAscending(key, uint64(i.Interleave.Ancestors[0].TableID))
key = encoding.EncodeUvarintAscending(key, uint64(i.Interleave.Ancestors[0].IndexID))
return key
}
key = encoding.EncodeUvarintAscending(key, uint64(desc.ID))
key = encoding.EncodeUvarintAscending(key, uint64(indexID))
return key
}
// MakeNameMetadataKey returns the key for the name. Pass name == "" in order
// to generate the prefix key to use to scan over all of the names for the
// specified parentID.
func MakeNameMetadataKey(parentID ID, name string) roachpb.Key {
normName := parser.ReNormalizeName(name)
k := keys.MakeTablePrefix(uint32(NamespaceTable.ID))
k = encoding.EncodeUvarintAscending(k, uint64(NamespaceTable.PrimaryIndex.ID))
k = encoding.EncodeUvarintAscending(k, uint64(parentID))
if name != "" {
k = encoding.EncodeBytesAscending(k, []byte(normName))
k = keys.MakeFamilyKey(k, uint32(NamespaceTable.Columns[2].ID))
}
return k
}
// runMVCCScan first creates test data (and resets the benchmarking
// timer). It then performs b.N MVCCScans in increments of numRows
// keys over all of the data in the Engine instance, restarting at
// the beginning of the keyspace, as many times as necessary.
func runMVCCScan(emk engineMaker, numRows, numVersions, valueSize int, b *testing.B) {
// Use the same number of keys for all of the mvcc scan
// benchmarks. Using a different number of keys per test gives
// preferential treatment to tests with fewer keys. Note that the
// datasets all fit in cache and the cache is pre-warmed.
const numKeys = 100000
eng, _ := setupMVCCData(emk, numVersions, numKeys, valueSize, b)
defer eng.Close()
b.SetBytes(int64(numRows * valueSize))
b.ResetTimer()
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
for i := 0; i < b.N; i++ {
// Choose a random key to start scan.
keyIdx := rand.Int31n(int32(numKeys - numRows))
startKey := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(keyIdx)))
walltime := int64(5 * (rand.Int31n(int32(numVersions)) + 1))
ts := makeTS(walltime, 0)
kvs, _, _, err := MVCCScan(context.Background(), eng, startKey, keyMax, int64(numRows), ts, true, nil)
if err != nil {
b.Fatalf("failed scan: %s", err)
}
if len(kvs) != numRows {
b.Fatalf("failed to scan: %d != %d", len(kvs), numRows)
}
}
b.StopTimer()
}
// runMVCCGet first creates test data (and resets the benchmarking
// timer). It then performs b.N MVCCGets.
func runMVCCGet(emk engineMaker, numVersions, valueSize int, b *testing.B) {
const overhead = 48 // Per key/value overhead (empirically determined)
const targetSize = 512 << 20 // 512 MB
// Adjust the number of keys so that each test has approximately the same
// amount of data.
numKeys := targetSize / ((overhead + valueSize) * (1 + (numVersions-1)/2))
eng, _ := setupMVCCData(emk, numVersions, numKeys, valueSize, b)
defer eng.Close()
b.SetBytes(int64(valueSize))
b.ResetTimer()
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
for i := 0; i < b.N; i++ {
// Choose a random key to retrieve.
keyIdx := rand.Int31n(int32(numKeys))
key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(keyIdx)))
walltime := int64(5 * (rand.Int31n(int32(numVersions)) + 1))
ts := makeTS(walltime, 0)
if v, _, err := MVCCGet(context.Background(), eng, key, ts, true, nil); err != nil {
b.Fatalf("failed get: %s", err)
} else if v == nil {
b.Fatalf("failed get (key not found): %d@%d", keyIdx, walltime)
} else if valueBytes, err := v.GetBytes(); err != nil {
b.Fatal(err)
} else if len(valueBytes) != valueSize {
b.Fatalf("unexpected value size: %d", len(valueBytes))
}
}
b.StopTimer()
}
func (td *tableDeleter) deleteAllRowsFast(
ctx context.Context, resume roachpb.Span, limit int64,
) (roachpb.Span, error) {
if resume.Key == nil {
tablePrefix := roachpb.Key(
encoding.EncodeUvarintAscending(nil, uint64(td.rd.helper.tableDesc.ID)),
)
// Delete rows and indexes starting with the table's prefix.
resume = roachpb.Span{
Key: tablePrefix,
EndKey: tablePrefix.PrefixEnd(),
}
}
if log.V(2) {
log.Infof(ctx, "DelRange %s - %s", resume.Key, resume.EndKey)
}
td.b.DelRange(resume.Key, resume.EndKey, false /* returnKeys */)
td.b.Header.MaxSpanRequestKeys = limit
if err := td.finalize(ctx); err != nil {
return resume, err
}
if l := len(td.b.Results); l != 1 {
panic(fmt.Sprintf("%d results returned", l))
}
return td.b.Results[0].ResumeSpan, nil
}
func BenchmarkBatchBuilderPut(b *testing.B) {
value := make([]byte, 10)
for i := range value {
value[i] = byte(i)
}
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
b.ResetTimer()
const batchSize = 1000
batch := &rocksDBBatchBuilder{}
for i := 0; i < b.N; i += batchSize {
end := i + batchSize
if end > b.N {
end = b.N
}
for j := i; j < end; j++ {
key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(j)))
ts := hlc.Timestamp{WallTime: int64(j)}
batch.Put(MVCCKey{key, ts}, value)
}
batch.Finish()
}
b.StopTimer()
}
func makePrefixWithRangeID(prefix []byte, rangeID roachpb.RangeID, infix roachpb.RKey) roachpb.Key {
// Size the key buffer so that it is large enough for most callers.
key := make(roachpb.Key, 0, 32)
key = append(key, prefix...)
key = encoding.EncodeUvarintAscending(key, uint64(rangeID))
key = append(key, infix...)
return key
}
// MakeKeyFromEncDatums creates a key by concatenating keyPrefix with the
// encodings of the given EncDatum values. The values correspond to
// index.ColumnIDs.
//
// If a table or index is interleaved, `encoding.encodedNullDesc` is used in
// place of the family id (a varint) to signal the next component of the key.
// An example of one level of interleaving (a parent):
// /<parent_table_id>/<parent_index_id>/<field_1>/<field_2>/NullDesc/<table_id>/<index_id>/<field_3>/<family>
//
// Note that ImplicitColumnIDs are not encoded, so the result isn't always a
// full index key.
func MakeKeyFromEncDatums(
values EncDatumRow,
tableDesc *TableDescriptor,
index *IndexDescriptor,
keyPrefix []byte,
alloc *DatumAlloc,
) (roachpb.Key, error) {
dirs := index.ColumnDirections
if len(values) != len(dirs) {
return nil, errors.Errorf("%d values, %d directions", len(values), len(dirs))
}
// We know we will append to the key which will cause the capacity to grow
// so make it bigger from the get-go.
key := make(roachpb.Key, len(keyPrefix), len(keyPrefix)*2)
copy(key, keyPrefix)
if len(index.Interleave.Ancestors) > 0 {
for i, ancestor := range index.Interleave.Ancestors {
// The first ancestor is assumed to already be encoded in keyPrefix.
if i != 0 {
key = encoding.EncodeUvarintAscending(key, uint64(ancestor.TableID))
key = encoding.EncodeUvarintAscending(key, uint64(ancestor.IndexID))
}
length := int(ancestor.SharedPrefixLen)
var err error
key, err = appendEncDatumsToKey(key, values[:length], dirs[:length], alloc)
if err != nil {
return nil, err
}
values, dirs = values[length:], dirs[length:]
// We reuse NotNullDescending (0xfe) as the interleave sentinel.
key = encoding.EncodeNotNullDescending(key)
}
key = encoding.EncodeUvarintAscending(key, uint64(tableDesc.ID))
key = encoding.EncodeUvarintAscending(key, uint64(index.ID))
}
return appendEncDatumsToKey(key, values, dirs, alloc)
}
// EncodeIndexKey creates a key by concatenating keyPrefix with the encodings of
// the columns in the index.
//
// If a table or index is interleaved, `encoding.encodedNullDesc` is used in
// place of the family id (a varint) to signal the next component of the key.
// An example of one level of interleaving (a parent):
// /<parent_table_id>/<parent_index_id>/<field_1>/<field_2>/NullDesc/<table_id>/<index_id>/<field_3>/<family>
//
// Returns the key and whether any of the encoded values were NULLs.
//
// Note that ImplicitColumnIDs are not encoded, so the result isn't always a
// full index key.
func EncodeIndexKey(
tableDesc *TableDescriptor,
index *IndexDescriptor,
colMap map[ColumnID]int,
values []parser.Datum,
keyPrefix []byte,
) (key []byte, containsNull bool, err error) {
key = keyPrefix
colIDs := index.ColumnIDs
dirs := directions(index.ColumnDirections)
if len(index.Interleave.Ancestors) > 0 {
for i, ancestor := range index.Interleave.Ancestors {
// The first ancestor is assumed to already be encoded in keyPrefix.
if i != 0 {
key = encoding.EncodeUvarintAscending(key, uint64(ancestor.TableID))
key = encoding.EncodeUvarintAscending(key, uint64(ancestor.IndexID))
}
length := int(ancestor.SharedPrefixLen)
var n bool
key, n, err = EncodeColumns(colIDs[:length], dirs[:length], colMap, values, key)
if err != nil {
return key, containsNull, err
}
colIDs, dirs = colIDs[length:], dirs[length:]
containsNull = containsNull || n
// We reuse NotNullDescending (0xfe) as the interleave sentinel.
key = encoding.EncodeNotNullDescending(key)
}
key = encoding.EncodeUvarintAscending(key, uint64(tableDesc.ID))
key = encoding.EncodeUvarintAscending(key, uint64(index.ID))
}
var n bool
key, n, err = EncodeColumns(colIDs, dirs, colMap, values, key)
containsNull = containsNull || n
return key, containsNull, err
}
func TestEnsureSafeSplitKey(t *testing.T) {
e := func(vals ...uint64) roachpb.Key {
var k roachpb.Key
for _, v := range vals {
k = encoding.EncodeUvarintAscending(k, v)
}
return k
}
goodData := []struct {
in roachpb.Key
expected roachpb.Key
}{
{e(1, 2, 0), e(1, 2)}, // /Table/1/2/0 -> /Table/1/2
{e(1, 2, 1), e(1)}, // /Table/1/2/1 -> /Table/1
{e(1, 2, 2), e()}, // /Table/1/2/2 -> /Table
{e(1, 2, 3, 0), e(1, 2, 3)}, // /Table/1/2/3/0 -> /Table/1/2/3
{e(1, 2, 3, 1), e(1, 2)}, // /Table/1/2/3/1 -> /Table/1/2
{e(1, 2, 200, 2), e(1, 2)}, // /Table/1/2/200/2 -> /Table/1/2
{e(1, 2, 3, 4, 1), e(1, 2, 3)}, // /Table/1/2/3/4/1 -> /Table/1/2/3
}
for i, d := range goodData {
out, err := EnsureSafeSplitKey(d.in)
if err != nil {
t.Fatalf("%d: %s: unexpected error: %v", i, d.in, err)
}
if !d.expected.Equal(out) {
t.Fatalf("%d: %s: expected %s, but got %s", i, d.in, d.expected, out)
}
}
errorData := []struct {
in roachpb.Key
err string
}{
// Column ID suffix size is too large.
{e(1), "malformed table key"},
{e(1, 2), "malformed table key"},
// The table ID is invalid.
{e(200)[:1], "insufficient bytes to decode uvarint value"},
// The index ID is invalid.
{e(1, 200)[:2], "insufficient bytes to decode uvarint value"},
// The column ID suffix is invalid.
{e(1, 2, 200)[:3], "insufficient bytes to decode uvarint value"},
}
for i, d := range errorData {
_, err := EnsureSafeSplitKey(d.in)
if !testutils.IsError(err, d.err) {
t.Fatalf("%d: %s: expected %s, but got %v", i, d.in, d.err, err)
}
}
}
func runBatchApplyBatchRepr(
emk engineMaker, writeOnly bool, valueSize, batchSize int, b *testing.B,
) {
rng, _ := randutil.NewPseudoRand()
value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueSize))
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
eng := emk(b, fmt.Sprintf("batch_apply_batch_repr_%d_%d", valueSize, batchSize))
defer eng.Close()
var repr []byte
{
batch := eng.NewBatch()
for i := 0; i < batchSize; i++ {
key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(i)))
ts := makeTS(timeutil.Now().UnixNano(), 0)
if err := MVCCPut(context.Background(), batch, nil, key, ts, value, nil); err != nil {
b.Fatal(err)
}
}
repr = batch.Repr()
batch.Close()
}
b.SetBytes(int64(len(repr)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var batch Batch
if writeOnly {
batch = eng.NewWriteOnlyBatch()
} else {
batch = eng.NewBatch()
}
if err := batch.ApplyBatchRepr(repr); err != nil {
b.Fatal(err)
}
batch.Close()
}
b.StopTimer()
}
// MakeRekeyMVCCKeyValFunc takes an iterator function for MVCCKeyValues and
// returns a new iterator function where the keys are rewritten inline to the
// have the given table ID.
func MakeRekeyMVCCKeyValFunc(
newTableID sqlbase.ID, f func(kv engine.MVCCKeyValue) (bool, error),
) func(engine.MVCCKeyValue) (bool, error) {
encodedNewTableID := encoding.EncodeUvarintAscending(nil, uint64(newTableID))
return func(kv engine.MVCCKeyValue) (bool, error) {
if encoding.PeekType(kv.Key.Key) != encoding.Int {
return false, errors.Errorf("unable to decode table key: %s", kv.Key.Key)
}
existingTableIDLen, err := encoding.PeekLength(kv.Key.Key)
if err != nil {
return false, err
}
if existingTableIDLen == len(encodedNewTableID) {
copy(kv.Key.Key, encodedNewTableID)
} else {
kv.Key.Key = append(encodedNewTableID, kv.Key.Key[existingTableIDLen:]...)
}
return f(kv)
}
}
func runMVCCBatchPut(emk engineMaker, valueSize, batchSize int, b *testing.B) {
rng, _ := randutil.NewPseudoRand()
value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueSize))
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
eng := emk(b, fmt.Sprintf("batch_put_%d_%d", valueSize, batchSize))
defer eng.Close()
b.SetBytes(int64(valueSize))
b.ResetTimer()
for i := 0; i < b.N; i += batchSize {
end := i + batchSize
if end > b.N {
end = b.N
}
batch := eng.NewBatch()
for j := i; j < end; j++ {
key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(j)))
ts := makeTS(timeutil.Now().UnixNano(), 0)
if err := MVCCPut(context.Background(), batch, nil, key, ts, value, nil); err != nil {
b.Fatalf("failed put: %s", err)
}
}
if err := batch.Commit(); err != nil {
b.Fatal(err)
}
batch.Close()
}
b.StopTimer()
}
func BenchmarkSstRekey(b *testing.B) {
// TODO(dan): DRY this with BenchmarkRocksDBSstFileReader.
dir, cleanupFn := testutils.TempDir(b, 1)
defer cleanupFn()
sstPath := filepath.Join(dir, "sst")
{
const maxEntries = 100000
const keyLen = 10
const valLen = 100
b.SetBytes(keyLen + valLen)
ts := hlc.Timestamp{WallTime: timeutil.Now().UnixNano()}
kv := engine.MVCCKeyValue{
Key: engine.MVCCKey{Key: roachpb.Key(make([]byte, keyLen)), Timestamp: ts},
Value: make([]byte, valLen),
}
sst := engine.MakeRocksDBSstFileWriter()
if err := sst.Open(sstPath); err != nil {
b.Fatal(sst)
}
var entries = b.N
if entries > maxEntries {
entries = maxEntries
}
for i := 0; i < entries; i++ {
payload := []byte(fmt.Sprintf("%09d", i))
kv.Key.Key = kv.Key.Key[:0]
kv.Key.Key = encoding.EncodeUvarintAscending(kv.Key.Key, uint64(i)) // tableID
kv.Key.Key = encoding.EncodeUvarintAscending(kv.Key.Key, 0) // indexID
kv.Key.Key = encoding.EncodeBytesAscending(kv.Key.Key, payload)
kv.Key.Key = keys.MakeRowSentinelKey(kv.Key.Key)
copy(kv.Value, payload)
if err := sst.Add(kv); err != nil {
b.Fatal(err)
}
}
if err := sst.Close(); err != nil {
b.Fatal(err)
}
}
const newTableID = 100
b.ResetTimer()
sst, err := engine.MakeRocksDBSstFileReader()
if err != nil {
b.Fatal(err)
}
if err := sst.AddFile(sstPath); err != nil {
b.Fatal(err)
}
defer sst.Close()
count := 0
iterateFn := sql.MakeRekeyMVCCKeyValFunc(newTableID, func(kv engine.MVCCKeyValue) (bool, error) {
count++
if count >= b.N {
return true, nil
}
return false, nil
})
for {
if err := sst.Iterate(engine.MVCCKey{Key: keys.MinKey}, engine.MVCCKey{Key: keys.MaxKey}, iterateFn); err != nil {
b.Fatal(err)
}
if count >= b.N {
break
}
}
}
// setupMVCCData writes up to numVersions values at each of numKeys
// keys. The number of versions written for each key is chosen
// randomly according to a uniform distribution. Each successive
// version is written starting at 5ns and then in 5ns increments. This
// allows scans at various times, starting at t=5ns, and continuing to
// t=5ns*(numVersions+1). A version for each key will be read on every
// such scan, but the dynamics of the scan will change depending on
// the historical timestamp. Earlier timestamps mean scans which must
// skip more historical versions; later timestamps mean scans which
// skip fewer.
//
// The creation of the database is time consuming, especially for larger
// numbers of versions. The database is persisted between runs and stored in
// the current directory as "mvcc_scan_<versions>_<keys>_<valueBytes>" (which
// is also returned).
func setupMVCCData(
emk engineMaker, numVersions, numKeys, valueBytes int, b *testing.B,
) (Engine, string) {
loc := fmt.Sprintf("mvcc_data_%d_%d_%d", numVersions, numKeys, valueBytes)
exists := true
if _, err := os.Stat(loc); os.IsNotExist(err) {
exists = false
}
eng := emk(b, loc)
if exists {
readAllFiles(filepath.Join(loc, "*"))
return eng, loc
}
log.Infof(context.Background(), "creating mvcc data: %s", loc)
// Generate the same data every time.
rng := rand.New(rand.NewSource(1449168817))
keys := make([]roachpb.Key, numKeys)
var order []int
for i := 0; i < numKeys; i++ {
keys[i] = roachpb.Key(encoding.EncodeUvarintAscending([]byte("key-"), uint64(i)))
keyVersions := rng.Intn(numVersions) + 1
for j := 0; j < keyVersions; j++ {
order = append(order, i)
}
}
// Randomize the order in which the keys are written.
for i, n := 0, len(order); i < n-1; i++ {
j := i + rng.Intn(n-i)
order[i], order[j] = order[j], order[i]
}
counts := make([]int, numKeys)
batch := eng.NewBatch()
for i, idx := range order {
// Output the keys in ~20 batches. If we used a single batch to output all
// of the keys rocksdb would create a single sstable. We want multiple
// sstables in order to exercise filtering of which sstables are examined
// during iterator seeking. We fix the number of batches we output so that
// optimizations which change the data size result in the same number of
// sstables.
if scaled := len(order) / 20; i > 0 && (i%scaled) == 0 {
log.Infof(context.Background(), "committing (%d/~%d)", i/scaled, 20)
if err := batch.Commit(); err != nil {
b.Fatal(err)
}
batch.Close()
batch = eng.NewBatch()
if err := eng.Flush(); err != nil {
b.Fatal(err)
}
}
key := keys[idx]
ts := makeTS(int64(counts[idx]+1)*5, 0)
counts[idx]++
value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueBytes))
value.InitChecksum(key)
if err := MVCCPut(context.Background(), batch, nil, key, ts, value, nil); err != nil {
b.Fatal(err)
}
}
if err := batch.Commit(); err != nil {
b.Fatal(err)
}
batch.Close()
if err := eng.Flush(); err != nil {
b.Fatal(err)
}
return eng, loc
}
func TestConcurrentBatch(t *testing.T) {
defer leaktest.AfterTest(t)()
dir, err := ioutil.TempDir("", "TestConcurrentBatch")
if err != nil {
t.Fatal(err)
}
defer func() {
if err := os.RemoveAll(dir); err != nil {
t.Fatal(err)
}
}()
db, err := NewRocksDB(roachpb.Attributes{}, dir, RocksDBCache{},
0, DefaultMaxOpenFiles)
if err != nil {
t.Fatalf("could not create new rocksdb db instance at %s: %v", dir, err)
}
defer db.Close()
// Prepare 16 4 MB batches containing non-overlapping contents.
var batches []Batch
for i := 0; i < 16; i++ {
batch := db.NewBatch()
for j := 0; true; j++ {
key := encoding.EncodeUvarintAscending([]byte("bar"), uint64(i))
key = encoding.EncodeUvarintAscending(key, uint64(j))
if err := batch.Put(MakeMVCCMetadataKey(key), nil); err != nil {
t.Fatal(err)
}
if len(batch.Repr()) >= 4<<20 {
break
}
}
batches = append(batches, batch)
}
errChan := make(chan error, len(batches))
// Concurrently write all the batches.
for _, batch := range batches {
go func(batch Batch) {
errChan <- batch.Commit()
}(batch)
}
// While the batch writes are in progress, try to write another key.
time.Sleep(100 * time.Millisecond)
remainingBatches := len(batches)
for i := 0; remainingBatches > 0; i++ {
select {
case err := <-errChan:
if err != nil {
t.Fatal(err)
}
remainingBatches--
default:
}
// This write can get delayed excessively if we hit the max memtable count
// or the L0 stop writes threshold.
start := timeutil.Now()
key := encoding.EncodeUvarintAscending([]byte("foo"), uint64(i))
if err := db.Put(MakeMVCCMetadataKey(key), nil); err != nil {
t.Fatal(err)
}
if elapsed := timeutil.Since(start); elapsed >= 10*time.Second {
t.Fatalf("write took %0.1fs\n", elapsed.Seconds())
}
}
}
// MakeAllDescsMetadataKey returns the key for all descriptors.
func MakeAllDescsMetadataKey() roachpb.Key {
k := keys.MakeTablePrefix(uint32(DescriptorTable.ID))
return encoding.EncodeUvarintAscending(k, uint64(DescriptorTable.PrimaryIndex.ID))
}
// MakeDescMetadataKey returns the key for the descriptor.
func MakeDescMetadataKey(descID ID) roachpb.Key {
k := MakeAllDescsMetadataKey()
k = encoding.EncodeUvarintAscending(k, uint64(descID))
return keys.MakeFamilyKey(k, uint32(DescriptorTable.Columns[1].ID))
}
// MakeZoneKey returns the key for 'id's entry in the system.zones table.
func MakeZoneKey(id ID) roachpb.Key {
k := keys.MakeTablePrefix(uint32(ZonesTable.ID))
k = encoding.EncodeUvarintAscending(k, uint64(ZonesTable.PrimaryIndex.ID))
k = encoding.EncodeUvarintAscending(k, uint64(id))
return keys.MakeFamilyKey(k, uint32(ZonesTable.Columns[1].ID))
}
// NodeLastUsageReportKey returns the key for accessing the node last update check
// time (when version check or usage reporting was done).
func NodeLastUsageReportKey(nodeID roachpb.NodeID) roachpb.Key {
prefix := append([]byte(nil), UpdateCheckPrefix...)
return encoding.EncodeUvarintAscending(prefix, uint64(nodeID))
}
// NodeStatusKey returns the key for accessing the node status for the
// specified node ID.
func NodeStatusKey(nodeID roachpb.NodeID) roachpb.Key {
key := make(roachpb.Key, 0, len(StatusNodePrefix)+9)
key = append(key, StatusNodePrefix...)
key = encoding.EncodeUvarintAscending(key, uint64(nodeID))
return key
}
// MakeTablePrefix returns the key prefix used for the table's data.
func MakeTablePrefix(tableID uint32) []byte {
return encoding.EncodeUvarintAscending(nil, uint64(tableID))
}
// NodeLivenessKey returns the key for the node liveness record.
func NodeLivenessKey(nodeID roachpb.NodeID) roachpb.Key {
key := make(roachpb.Key, 0, len(NodeLivenessPrefix)+9)
key = append(key, NodeLivenessPrefix...)
key = encoding.EncodeUvarintAscending(key, uint64(nodeID))
return key
}