// fillRange writes keys with the given prefix and associated values
// until bytes bytes have been written or the given range has split.
func fillRange(store *storage.Store, rangeID roachpb.RangeID, prefix roachpb.Key, bytes int64, t *testing.T) {
src := rand.New(rand.NewSource(0))
for {
var ms enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), store.Engine(), rangeID, &ms); err != nil {
t.Fatal(err)
}
keyBytes, valBytes := ms.KeyBytes, ms.ValBytes
if keyBytes+valBytes >= bytes {
return
}
key := append(append([]byte(nil), prefix...), randutil.RandBytes(src, 100)...)
key = keys.MakeRowSentinelKey(key)
val := randutil.RandBytes(src, int(src.Int31n(1<<8)))
pArgs := putArgs(key, val)
_, pErr := client.SendWrappedWith(store, nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs)
// When the split occurs in the background, our writes may start failing.
// We know we can stop writing when this happens.
if _, ok := pErr.GetDetail().(*roachpb.RangeKeyMismatchError); ok {
return
} else if pErr != nil {
t.Fatal(pErr)
}
}
}
func (z *zeroSum) monkey(tableID uint32, d time.Duration) {
r := newRand()
zipf := z.accountDistribution(r)
for {
time.Sleep(time.Duration(rand.Float64() * float64(d)))
key := keys.MakeTablePrefix(tableID)
key = encoding.EncodeVarintAscending(key, int64(zipf.Uint64()))
key = keys.MakeRowSentinelKey(key)
switch r.Intn(2) {
case 0:
if err := z.split(z.randNode(r.Intn), key); err != nil {
if strings.Contains(err.Error(), "range is already split at key") ||
strings.Contains(err.Error(), "conflict updating range descriptors") {
continue
}
z.maybeLogError(err)
} else {
atomic.AddUint64(&z.stats.splits, 1)
}
case 1:
if transferred, err := z.transferLease(z.randNode(r.Intn), r, key); err != nil {
z.maybeLogError(err)
} else if transferred {
atomic.AddUint64(&z.stats.transfers, 1)
}
}
}
}
func writeRandomTimeSeriesDataToRange(
t testing.TB,
store *storage.Store,
rangeID roachpb.RangeID,
keyPrefix []byte,
) (midpoint []byte) {
src := rand.New(rand.NewSource(0))
r := ts.Resolution10s
for i := 0; i < 20; i++ {
var data []tspb.TimeSeriesData
for j := int64(0); j <= src.Int63n(5); j++ {
d := tspb.TimeSeriesData{
Name: "test.random.metric",
Source: "cpu01",
}
for k := int64(0); k <= src.Int63n(10); k++ {
d.Datapoints = append(d.Datapoints, tspb.TimeSeriesDatapoint{
TimestampNanos: src.Int63n(200) * r.KeyDuration(),
Value: src.Float64(),
})
}
data = append(data, d)
}
for _, d := range data {
idatas, err := d.ToInternal(r.KeyDuration(), r.SampleDuration())
if err != nil {
t.Fatal(err)
}
for _, idata := range idatas {
var value roachpb.Value
if err := value.SetProto(&idata); err != nil {
t.Fatal(err)
}
mArgs := roachpb.MergeRequest{
Span: roachpb.Span{
Key: encoding.EncodeVarintAscending(keyPrefix, idata.StartTimestampNanos),
},
Value: value,
}
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeID,
}, &mArgs); pErr != nil {
t.Fatal(pErr)
}
}
}
}
// Return approximate midway point (100 is midway between random timestamps in range [0,200)).
midKey := append([]byte(nil), keyPrefix...)
midKey = encoding.EncodeVarintAscending(midKey, 100*r.KeyDuration())
return keys.MakeRowSentinelKey(midKey)
}
func fillTestRange(t testing.TB, rep *Replica, size int) {
src := rand.New(rand.NewSource(0))
for i := 0; i < snapSize/(keySize+valSize); i++ {
key := keys.MakeRowSentinelKey(randutil.RandBytes(src, keySize))
val := randutil.RandBytes(src, valSize)
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(rep, nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
t.Fatal(pErr)
}
}
}
// TestStoreRangeSplitAtTablePrefix verifies a range can be split at
// UserTableDataMin and still gossip the SystemConfig properly.
func TestStoreRangeSplitAtTablePrefix(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
key := keys.MakeRowSentinelKey(append([]byte(nil), keys.UserTableDataMin...))
args := adminSplitArgs(key, key)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); pErr != nil {
t.Fatalf("%q: split unexpected error: %s", key, pErr)
}
var desc sqlbase.TableDescriptor
descBytes, err := protoutil.Marshal(&desc)
if err != nil {
t.Fatal(err)
}
// Update SystemConfig to trigger gossip.
if err := store.DB().Txn(func(txn *client.Txn) error {
txn.SetSystemConfigTrigger()
// We don't care about the values, just the keys.
k := sqlbase.MakeDescMetadataKey(sqlbase.ID(keys.MaxReservedDescID + 1))
return txn.Put(k, &desc)
}); err != nil {
t.Fatal(err)
}
successChan := make(chan struct{}, 1)
store.Gossip().RegisterCallback(gossip.KeySystemConfig, func(_ string, content roachpb.Value) {
contentBytes, err := content.GetBytes()
if err != nil {
t.Fatal(err)
}
if bytes.Contains(contentBytes, descBytes) {
select {
case successChan <- struct{}{}:
default:
}
}
})
select {
case <-time.After(time.Second):
t.Errorf("expected a schema gossip containing %q, but did not see one", descBytes)
case <-successChan:
}
}
func writeRandomDataToRange(t testing.TB, store *storage.Store, rangeID roachpb.RangeID, keyPrefix []byte) {
src := rand.New(rand.NewSource(0))
for i := 0; i < 100; i++ {
key := append([]byte(nil), keyPrefix...)
key = append(key, randutil.RandBytes(src, int(src.Int31n(1<<7)))...)
key = keys.MakeRowSentinelKey(key)
val := randutil.RandBytes(src, int(src.Int31n(1<<8)))
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
t.Fatal(pErr)
}
}
}
func writeRandomDataToRange(
t testing.TB,
store *storage.Store,
rangeID roachpb.RangeID,
keyPrefix []byte,
) (midpoint []byte) {
src := rand.New(rand.NewSource(0))
for i := 0; i < 100; i++ {
key := append([]byte(nil), keyPrefix...)
key = append(key, randutil.RandBytes(src, int(src.Int31n(1<<7)))...)
key = keys.MakeRowSentinelKey(key)
val := randutil.RandBytes(src, int(src.Int31n(1<<8)))
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
t.Fatal(pErr)
}
}
// Return approximate midway point ("Z" in string "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz").
midKey := append([]byte(nil), keyPrefix...)
midKey = append(midKey, []byte("Z")...)
return keys.MakeRowSentinelKey(midKey)
}
// EncodeSecondaryIndex encodes key/values for a secondary index. colMap maps
// ColumnIDs to indices in `values`.
func EncodeSecondaryIndex(
tableDesc *TableDescriptor,
secondaryIndex *IndexDescriptor,
colMap map[ColumnID]int,
values []parser.Datum,
) (IndexEntry, error) {
secondaryIndexKeyPrefix := MakeIndexKeyPrefix(tableDesc, secondaryIndex.ID)
secondaryIndexKey, containsNull, err := EncodeIndexKey(
tableDesc, secondaryIndex, colMap, values, secondaryIndexKeyPrefix)
if err != nil {
return IndexEntry{}, err
}
// Add the implicit columns - they are encoded ascendingly which is done by
// passing nil for the encoding directions.
extraKey, _, err := EncodeColumns(secondaryIndex.ImplicitColumnIDs, nil,
colMap, values, nil)
if err != nil {
return IndexEntry{}, err
}
entry := IndexEntry{Key: secondaryIndexKey}
if !secondaryIndex.Unique || containsNull {
// If the index is not unique or it contains a NULL value, append
// extraKey to the key in order to make it unique.
entry.Key = append(entry.Key, extraKey...)
}
// Index keys are considered "sentinel" keys in that they do not have a
// column ID suffix.
entry.Key = keys.MakeRowSentinelKey(entry.Key)
if secondaryIndex.Unique {
// Note that a unique secondary index that contains a NULL column value
// will have extraKey appended to the key and stored in the value. We
// require extraKey to be appended to the key in order to make the key
// unique. We could potentially get rid of the duplication here but at
// the expense of complicating scanNode when dealing with unique
// secondary indexes.
entry.Value.SetBytes(extraKey)
} else {
// The zero value for an index-key is a 0-length bytes value.
entry.Value.SetBytes([]byte{})
}
return entry, nil
}
func fillTestRange(t testing.TB, rep *Replica, size int64) {
src := rand.New(rand.NewSource(0))
for i := int64(0); i < size/int64(keySize+valSize); i++ {
key := keys.MakeRowSentinelKey(randutil.RandBytes(src, keySize))
val := randutil.RandBytes(src, valSize)
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(rep, nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
t.Fatal(pErr)
}
}
rep.mu.Lock()
after := rep.mu.state.Stats.Total()
rep.mu.Unlock()
if after < size {
t.Fatalf("range not full after filling: wrote %d, but range at %d", size, after)
}
}
func BenchmarkReplicaSnapshot(b *testing.B) {
defer tracing.Disable()()
defer config.TestingDisableTableSplits()()
store, stopper, _ := createTestStore(b)
// We want to manually control the size of the raft log.
store.DisableRaftLogQueue(true)
defer stopper.Stop()
const rangeID = 1
const keySize = 1 << 7 // 128 B
const valSize = 1 << 10 // 1 KiB
const snapSize = 1 << 25 // 32 MiB
rep, err := store.GetReplica(rangeID)
if err != nil {
b.Fatal(err)
}
src := rand.New(rand.NewSource(0))
for i := 0; i < snapSize/(keySize+valSize); i++ {
key := keys.MakeRowSentinelKey(randutil.RandBytes(src, keySize))
val := randutil.RandBytes(src, valSize)
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(rep, nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
b.Fatal(pErr)
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := rep.GetSnapshot(); err != nil {
b.Fatal(err)
}
}
}
func TestManualReplication(t *testing.T) {
defer leaktest.AfterTest(t)()
tc := StartTestCluster(t, 3,
base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
UseDatabase: "t",
},
})
defer tc.Stopper().Stop()
s0 := sqlutils.MakeSQLRunner(t, tc.Conns[0])
s1 := sqlutils.MakeSQLRunner(t, tc.Conns[1])
s2 := sqlutils.MakeSQLRunner(t, tc.Conns[2])
s0.Exec(`CREATE DATABASE t`)
s0.Exec(`CREATE TABLE test (k INT PRIMARY KEY, v INT)`)
s0.Exec(`INSERT INTO test VALUES (5, 1), (4, 2), (1, 2)`)
if r := s1.Query(`SELECT * FROM test WHERE k = 5`); !r.Next() {
t.Fatal("no rows")
}
s2.ExecRowsAffected(3, `DELETE FROM test`)
// Split the table to a new range.
kvDB := tc.Servers[0].DB()
tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test")
tableStartKey := keys.MakeRowSentinelKey(keys.MakeTablePrefix(uint32(tableDesc.ID)))
leftRangeDesc, tableRangeDesc, err := tc.SplitRange(tableStartKey)
if err != nil {
t.Fatal(err)
}
log.Infof(context.Background(), "After split got ranges: %+v and %+v.", leftRangeDesc, tableRangeDesc)
if len(tableRangeDesc.Replicas) == 0 {
t.Fatalf(
"expected replica on node 1, got no replicas: %+v", tableRangeDesc.Replicas)
}
if tableRangeDesc.Replicas[0].NodeID != 1 {
t.Fatalf(
"expected replica on node 1, got replicas: %+v", tableRangeDesc.Replicas)
}
// Replicate the table's range to all the nodes.
tableRangeDesc, err = tc.AddReplicas(
tableRangeDesc.StartKey.AsRawKey(), tc.Target(1), tc.Target(2),
)
if err != nil {
t.Fatal(err)
}
if len(tableRangeDesc.Replicas) != 3 {
t.Fatalf("expected 3 replicas, got %+v", tableRangeDesc.Replicas)
}
for i := 0; i < 3; i++ {
if _, ok := tableRangeDesc.GetReplicaDescriptor(
tc.Servers[i].GetFirstStoreID()); !ok {
t.Fatalf("expected replica on store %d, got %+v",
tc.Servers[i].GetFirstStoreID(), tableRangeDesc.Replicas)
}
}
// Transfer the lease to node 1.
leaseHolder, err := tc.FindRangeLeaseHolder(
tableRangeDesc,
&ReplicationTarget{
NodeID: tc.Servers[0].GetNode().Descriptor.NodeID,
StoreID: tc.Servers[0].GetFirstStoreID(),
})
if err != nil {
t.Fatal(err)
}
if leaseHolder.StoreID != tc.Servers[0].GetFirstStoreID() {
t.Fatalf("expected initial lease on server idx 0, but is on node: %+v",
leaseHolder)
}
err = tc.TransferRangeLease(tableRangeDesc, tc.Target(1))
if err != nil {
t.Fatal(err)
}
// Check that the lease holder has changed. We'll use a bogus hint, which
// shouldn't matter (other than ensuring that it's not this call that moves
// the range holder, but that a holder already existed).
leaseHolder, err = tc.FindRangeLeaseHolder(
tableRangeDesc,
&ReplicationTarget{
NodeID: tc.Servers[0].GetNode().Descriptor.NodeID,
StoreID: tc.Servers[0].GetFirstStoreID(),
})
if err != nil {
t.Fatal(err)
}
if leaseHolder.StoreID != tc.Servers[1].GetFirstStoreID() {
t.Fatalf("expected lease on server idx 1 (node: %d store: %d), but is on node: %+v",
tc.Servers[1].GetNode().Descriptor.NodeID,
tc.Servers[1].GetFirstStoreID(),
leaseHolder)
}
}
func TestComputeSplits(t *testing.T) {
defer leaktest.AfterTest(t)()
const (
start = keys.MaxReservedDescID + 1
reservedStart = keys.MaxSystemConfigDescID + 1
)
schema := sqlbase.MakeMetadataSchema()
// Real SQL system tables only.
baseSql := schema.GetInitialValues()
// Real SQL system tables plus some user stuff.
userSql := append(schema.GetInitialValues(),
descriptor(start), descriptor(start+1), descriptor(start+5))
// Real SQL system with reserved non-system tables.
schema.AddTable(reservedStart+1, "CREATE TABLE system.test1 (i INT PRIMARY KEY)")
schema.AddTable(reservedStart+2, "CREATE TABLE system.test2 (i INT PRIMARY KEY)")
reservedSql := schema.GetInitialValues()
// Real SQL system with reserved non-system and user database.
allSql := append(schema.GetInitialValues(),
descriptor(start), descriptor(start+1), descriptor(start+5))
sort.Sort(roachpb.KeyValueByKey(allSql))
allUserSplits := []uint32{start, start + 1, start + 2, start + 3, start + 4, start + 5}
var allReservedSplits []uint32
for i := 0; i < schema.TableCount(); i++ {
allReservedSplits = append(allReservedSplits, reservedStart+uint32(i))
}
allSplits := append(allReservedSplits, allUserSplits...)
testCases := []struct {
values []roachpb.KeyValue
start, end roachpb.RKey
// Use ints in the testcase definitions, more readable.
splits []uint32
}{
// No data.
{nil, roachpb.RKeyMin, roachpb.RKeyMax, nil},
{nil, keys.MakeTablePrefix(start), roachpb.RKeyMax, nil},
{nil, keys.MakeTablePrefix(start), keys.MakeTablePrefix(start + 10), nil},
{nil, roachpb.RKeyMin, keys.MakeTablePrefix(start + 10), nil},
// No user data.
{baseSql, roachpb.RKeyMin, roachpb.RKeyMax, allReservedSplits},
{baseSql, keys.MakeTablePrefix(start), roachpb.RKeyMax, nil},
{baseSql, keys.MakeTablePrefix(start), keys.MakeTablePrefix(start + 10), nil},
{baseSql, roachpb.RKeyMin, keys.MakeTablePrefix(start + 10), allReservedSplits},
// User descriptors.
{userSql, keys.MakeTablePrefix(start - 1), roachpb.RKeyMax, allUserSplits},
{userSql, keys.MakeTablePrefix(start), roachpb.RKeyMax, allUserSplits[1:]},
{userSql, keys.MakeTablePrefix(start), keys.MakeTablePrefix(start + 10), allUserSplits[1:]},
{userSql, keys.MakeTablePrefix(start - 1), keys.MakeTablePrefix(start + 10), allUserSplits},
{userSql, keys.MakeTablePrefix(start + 4), keys.MakeTablePrefix(start + 10), allUserSplits[5:]},
{userSql, keys.MakeTablePrefix(start + 5), keys.MakeTablePrefix(start + 10), nil},
{userSql, keys.MakeTablePrefix(start + 6), keys.MakeTablePrefix(start + 10), nil},
{userSql, testutils.MakeKey(keys.MakeTablePrefix(start), roachpb.RKey("foo")),
keys.MakeTablePrefix(start + 10), allUserSplits[1:]},
{userSql, testutils.MakeKey(keys.MakeTablePrefix(start), roachpb.RKey("foo")),
keys.MakeTablePrefix(start + 5), allUserSplits[1:5]},
{userSql, testutils.MakeKey(keys.MakeTablePrefix(start), roachpb.RKey("foo")),
testutils.MakeKey(keys.MakeTablePrefix(start+5), roachpb.RKey("bar")), allUserSplits[1:5]},
{userSql, testutils.MakeKey(keys.MakeTablePrefix(start), roachpb.RKey("foo")),
testutils.MakeKey(keys.MakeTablePrefix(start), roachpb.RKey("morefoo")), nil},
// Reserved descriptors.
{reservedSql, roachpb.RKeyMin, roachpb.RKeyMax, allReservedSplits},
{reservedSql, keys.MakeTablePrefix(reservedStart), roachpb.RKeyMax, allReservedSplits[1:]},
{reservedSql, keys.MakeTablePrefix(start), roachpb.RKeyMax, nil},
{reservedSql, keys.MakeTablePrefix(reservedStart), keys.MakeTablePrefix(start + 10), allReservedSplits[1:]},
{reservedSql, roachpb.RKeyMin, keys.MakeTablePrefix(reservedStart + 2), allReservedSplits[:2]},
{reservedSql, roachpb.RKeyMin, keys.MakeTablePrefix(reservedStart + 10), allReservedSplits},
{reservedSql, keys.MakeTablePrefix(reservedStart), keys.MakeTablePrefix(reservedStart + 2), allReservedSplits[1:2]},
{reservedSql, testutils.MakeKey(keys.MakeTablePrefix(reservedStart), roachpb.RKey("foo")),
testutils.MakeKey(keys.MakeTablePrefix(start+10), roachpb.RKey("foo")), allReservedSplits[1:]},
// Reserved/User mix.
{allSql, roachpb.RKeyMin, roachpb.RKeyMax, allSplits},
{allSql, keys.MakeTablePrefix(reservedStart + 1), roachpb.RKeyMax, allSplits[2:]},
{allSql, keys.MakeTablePrefix(start), roachpb.RKeyMax, allUserSplits[1:]},
{allSql, keys.MakeTablePrefix(reservedStart), keys.MakeTablePrefix(start + 10), allSplits[1:]},
{allSql, roachpb.RKeyMin, keys.MakeTablePrefix(start + 2), allSplits[:6]},
{allSql, testutils.MakeKey(keys.MakeTablePrefix(reservedStart), roachpb.RKey("foo")),
testutils.MakeKey(keys.MakeTablePrefix(start+5), roachpb.RKey("foo")), allSplits[1:9]},
}
cfg := config.SystemConfig{}
for tcNum, tc := range testCases {
cfg.Values = tc.values
splits := cfg.ComputeSplitKeys(tc.start, tc.end)
if len(splits) == 0 && len(tc.splits) == 0 {
continue
}
// Convert ints to actual keys.
expected := []roachpb.RKey{}
for _, s := range tc.splits {
expected = append(expected, keys.MakeRowSentinelKey(keys.MakeTablePrefix(s)))
}
if !reflect.DeepEqual(splits, expected) {
t.Errorf("#%d: bad splits:\ngot: %v\nexpected: %v", tcNum, splits, expected)
}
}
}
// TestStoreRangeSplitStats starts by splitting the system keys from user-space
// keys and verifying that the user space side of the split (which is empty),
// has all zeros for stats. It then writes random data to the user space side,
// splits it halfway and verifies the two splits have stats exactly equaling
// the pre-split.
func TestStoreRangeSplitStats(t *testing.T) {
defer leaktest.AfterTest(t)()
defer config.TestingDisableTableSplits()()
store, stopper, manual := createTestStore(t)
defer stopper.Stop()
// Split the range after the last table data key.
keyPrefix := keys.MakeTablePrefix(keys.MaxReservedDescID + 1)
keyPrefix = keys.MakeRowSentinelKey(keyPrefix)
args := adminSplitArgs(roachpb.KeyMin, keyPrefix)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); pErr != nil {
t.Fatal(pErr)
}
// Verify empty range has empty stats.
rng := store.LookupReplica(keyPrefix, nil)
// NOTE that this value is expected to change over time, depending on what
// we store in the sys-local keyspace. Update it accordingly for this test.
if err := verifyRangeStats(store.Engine(), rng.RangeID, enginepb.MVCCStats{LastUpdateNanos: manual.UnixNano()}); err != nil {
t.Fatal(err)
}
// Write random data.
writeRandomDataToRange(t, store, rng.RangeID, keyPrefix)
// Get the range stats now that we have data.
snap := store.Engine().NewSnapshot()
defer snap.Close()
var ms enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), snap, rng.RangeID, &ms); err != nil {
t.Fatal(err)
}
if err := verifyRecomputedStats(snap, rng.Desc(), ms, manual.UnixNano()); err != nil {
t.Fatalf("failed to verify range's stats before split: %v", err)
}
if inMemMS := rng.GetMVCCStats(); inMemMS != ms {
t.Fatalf("in-memory and on-disk diverged:\n%+v\n!=\n%+v", inMemMS, ms)
}
manual.Increment(100)
// Split the range at approximate halfway point ("Z" in string "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz").
midKey := append([]byte(nil), keyPrefix...)
midKey = append(midKey, []byte("Z")...)
midKey = keys.MakeRowSentinelKey(midKey)
args = adminSplitArgs(keyPrefix, midKey)
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rng.RangeID,
}, &args); pErr != nil {
t.Fatal(pErr)
}
snap = store.Engine().NewSnapshot()
defer snap.Close()
var msLeft, msRight enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), snap, rng.RangeID, &msLeft); err != nil {
t.Fatal(err)
}
rngRight := store.LookupReplica(midKey, nil)
if err := engine.MVCCGetRangeStats(context.Background(), snap, rngRight.RangeID, &msRight); err != nil {
t.Fatal(err)
}
// The stats should be exactly equal when added.
expMS := enginepb.MVCCStats{
LiveBytes: msLeft.LiveBytes + msRight.LiveBytes,
KeyBytes: msLeft.KeyBytes + msRight.KeyBytes,
ValBytes: msLeft.ValBytes + msRight.ValBytes,
IntentBytes: msLeft.IntentBytes + msRight.IntentBytes,
LiveCount: msLeft.LiveCount + msRight.LiveCount,
KeyCount: msLeft.KeyCount + msRight.KeyCount,
ValCount: msLeft.ValCount + msRight.ValCount,
IntentCount: msLeft.IntentCount + msRight.IntentCount,
}
ms.SysBytes, ms.SysCount = 0, 0
ms.LastUpdateNanos = 0
if expMS != ms {
t.Errorf("expected left plus right ranges to equal original, but\n %+v\n+\n %+v\n!=\n %+v", msLeft, msRight, ms)
}
// Stats should both have the new timestamp.
now := manual.UnixNano()
if lTs := msLeft.LastUpdateNanos; lTs != now {
t.Errorf("expected left range stats to have new timestamp, want %d, got %d", now, lTs)
}
if rTs := msRight.LastUpdateNanos; rTs != now {
t.Errorf("expected right range stats to have new timestamp, want %d, got %d", now, rTs)
}
// Stats should agree with recomputation.
if err := verifyRecomputedStats(snap, rng.Desc(), msLeft, now); err != nil {
t.Fatalf("failed to verify left range's stats after split: %v", err)
}
if err := verifyRecomputedStats(snap, rngRight.Desc(), msRight, now); err != nil {
t.Fatalf("failed to verify right range's stats after split: %v", err)
}
}
// ComputeSplitKeys takes a start and end key and returns an array of keys
// at which to split the span [start, end).
// The only required splits are at each user table prefix.
func (s SystemConfig) ComputeSplitKeys(startKey, endKey roachpb.RKey) []roachpb.RKey {
tableStart := roachpb.RKey(keys.SystemConfigTableDataMax)
if !tableStart.Less(endKey) {
// This range is before the user tables span: no required splits.
return nil
}
startID, ok := ObjectIDForKey(startKey)
if !ok || startID <= keys.MaxSystemConfigDescID {
// The start key is either:
// - not part of the structured data span
// - part of the system span
// In either case, start looking for splits at the first ID usable
// by the user data span.
startID = keys.MaxSystemConfigDescID + 1
} else {
// The start key is either already a split key, or after the split
// key for its ID. We can skip straight to the next one.
startID++
}
// Build key prefixes for sequential table IDs until we reach endKey. Note
// that there are two disjoint sets of sequential keys: non-system reserved
// tables have sequential IDs, as do user tables, but the two ranges contain a
// gap.
var splitKeys []roachpb.RKey
var key roachpb.RKey
// appendSplitKeys generates all possible split keys between the given range
// of IDs and adds them to splitKeys.
appendSplitKeys := func(startID, endID uint32) {
// endID could be smaller than startID if we don't have user tables.
for id := startID; id <= endID; id++ {
key = keys.MakeRowSentinelKey(keys.MakeTablePrefix(id))
// Skip if this ID matches the startKey passed to ComputeSplitKeys.
if !startKey.Less(key) {
continue
}
// Handle the case where EndKey is already a table prefix.
if !key.Less(endKey) {
break
}
splitKeys = append(splitKeys, key)
}
}
// If the startKey falls within the non-system reserved range, compute those
// keys first.
if startID <= keys.MaxReservedDescID {
endID, err := s.GetLargestObjectID(keys.MaxReservedDescID)
if err != nil {
log.Errorf(context.TODO(), "unable to determine largest reserved object ID from system config: %s", err)
return nil
}
appendSplitKeys(startID, endID)
startID = keys.MaxReservedDescID + 1
}
// Append keys in the user space.
endID, err := s.GetLargestObjectID(0)
if err != nil {
log.Errorf(context.TODO(), "unable to determine largest object ID from system config: %s", err)
return nil
}
appendSplitKeys(startID, endID)
return splitKeys
}
// TestStoreRangeSplitStatsWithMerges starts by splitting the system keys from
// user-space keys and verifying that the user space side of the split (which is empty),
// has all zeros for stats. It then issues a number of Merge requests to the user
// space side, simulating TimeSeries data. Finally, the test splits the user space
// side halfway and verifies the stats on either side of the split are equal to a
// recomputation.
//
// Note that unlike TestStoreRangeSplitStats, we do not check if the two halves of the
// split's stats are equal to the pre-split stats when added, because this will not be
// true of ranges populated with Merge requests. The reason for this is that Merge
// requests' impact on MVCCStats are only estimated. See updateStatsOnMerge.
func TestStoreRangeSplitStatsWithMerges(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, manual := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
// Split the range after the last table data key.
keyPrefix := keys.MakeTablePrefix(keys.MaxReservedDescID + 1)
keyPrefix = keys.MakeRowSentinelKey(keyPrefix)
args := adminSplitArgs(roachpb.KeyMin, keyPrefix)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); pErr != nil {
t.Fatal(pErr)
}
// Verify empty range has empty stats.
rng := store.LookupReplica(keyPrefix, nil)
// NOTE that this value is expected to change over time, depending on what
// we store in the sys-local keyspace. Update it accordingly for this test.
empty := enginepb.MVCCStats{LastUpdateNanos: manual.UnixNano()}
if err := verifyRangeStats(store.Engine(), rng.RangeID, empty); err != nil {
t.Fatal(err)
}
// Write random TimeSeries data.
midKey := writeRandomTimeSeriesDataToRange(t, store, rng.RangeID, keyPrefix)
manual.Increment(100)
// Split the range at approximate halfway point.
args = adminSplitArgs(keyPrefix, midKey)
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rng.RangeID,
}, &args); pErr != nil {
t.Fatal(pErr)
}
snap := store.Engine().NewSnapshot()
defer snap.Close()
var msLeft, msRight enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), snap, rng.RangeID, &msLeft); err != nil {
t.Fatal(err)
}
rngRight := store.LookupReplica(midKey, nil)
if err := engine.MVCCGetRangeStats(context.Background(), snap, rngRight.RangeID, &msRight); err != nil {
t.Fatal(err)
}
// Stats should both have the new timestamp.
now := manual.UnixNano()
if lTs := msLeft.LastUpdateNanos; lTs != now {
t.Errorf("expected left range stats to have new timestamp, want %d, got %d", now, lTs)
}
if rTs := msRight.LastUpdateNanos; rTs != now {
t.Errorf("expected right range stats to have new timestamp, want %d, got %d", now, rTs)
}
// Stats should agree with recomputation.
if err := verifyRecomputedStats(snap, rng.Desc(), msLeft, now); err != nil {
t.Fatalf("failed to verify left range's stats after split: %v", err)
}
if err := verifyRecomputedStats(snap, rngRight.Desc(), msRight, now); err != nil {
t.Fatalf("failed to verify right range's stats after split: %v", err)
}
}