// TestRangeSplitsWithWritePressure sets the zone config max bytes for
// a range to 256K and writes data until there are five ranges.
func TestRangeSplitsWithWritePressure(t *testing.T) {
defer leaktest.AfterTest(t)()
// Override default zone config.
cfg := config.DefaultZoneConfig()
cfg.RangeMaxBytes = 1 << 18
defer config.TestingSetDefaultZoneConfig(cfg)()
dbCtx := client.DefaultDBContext()
dbCtx.TxnRetryOptions = retry.Options{
InitialBackoff: 1 * time.Millisecond,
MaxBackoff: 10 * time.Millisecond,
Multiplier: 2,
}
s, _ := createTestDBWithContext(t, dbCtx)
// This is purely to silence log spam.
config.TestingSetupZoneConfigHook(s.Stopper)
defer s.Stop()
// Start test writer write about a 32K/key so there aren't too many writes necessary to split 64K range.
done := make(chan struct{})
var wg sync.WaitGroup
wg.Add(1)
go startTestWriter(s.DB, int64(0), 1<<15, &wg, nil, nil, done, t)
// Check that we split 5 times in allotted time.
testutils.SucceedsSoon(t, func() error {
// Scan the txn records.
rows, err := s.DB.Scan(context.TODO(), keys.Meta2Prefix, keys.MetaMax, 0)
if err != nil {
return errors.Errorf("failed to scan meta2 keys: %s", err)
}
if lr := len(rows); lr < 5 {
return errors.Errorf("expected >= 5 scans; got %d", lr)
}
return nil
})
close(done)
wg.Wait()
// This write pressure test often causes splits while resolve
// intents are in flight, causing them to fail with range key
// mismatch errors. However, LocalSender should retry in these
// cases. Check here via MVCC scan that there are no dangling write
// intents. We do this using a SucceedsSoon construct to account
// for timing of finishing the test writer and a possibly-ongoing
// asynchronous split.
testutils.SucceedsSoon(t, func() error {
if _, _, _, err := engine.MVCCScan(context.Background(), s.Eng, keys.LocalMax, roachpb.KeyMax, math.MaxInt64, hlc.MaxTimestamp, true, nil); err != nil {
return errors.Errorf("failed to verify no dangling intents: %s", err)
}
return nil
})
}
// getActualData returns the actual value of all time series keys in the
// underlying engine. Data is returned as a map of strings to roachpb.Values.
func (tm *testModel) getActualData() map[string]roachpb.Value {
// Scan over all TS Keys stored in the engine
startKey := keys.TimeseriesPrefix
endKey := startKey.PrefixEnd()
keyValues, _, _, err := engine.MVCCScan(context.Background(), tm.Eng, startKey, endKey, math.MaxInt64, tm.Clock.Now(), true, nil)
if err != nil {
tm.t.Fatalf("error scanning TS data from engine: %s", err.Error())
}
kvMap := make(map[string]roachpb.Value)
for _, kv := range keyValues {
kvMap[string(kv.Key)] = kv.Value
}
return kvMap
}
// TestBootstrapCluster verifies the results of bootstrapping a
// cluster. Uses an in memory engine.
func TestBootstrapCluster(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
e := engine.NewInMem(roachpb.Attributes{}, 1<<20)
stopper.AddCloser(e)
if _, err := bootstrapCluster(
storage.StoreConfig{}, []engine.Engine{e}, kv.MakeTxnMetrics(metric.TestSampleInterval),
); err != nil {
t.Fatal(err)
}
// Scan the complete contents of the local database directly from the engine.
rows, _, _, err := engine.MVCCScan(context.Background(), e, keys.LocalMax, roachpb.KeyMax, math.MaxInt64, hlc.MaxTimestamp, true, nil)
if err != nil {
t.Fatal(err)
}
var foundKeys keySlice
for _, kv := range rows {
foundKeys = append(foundKeys, kv.Key)
}
var expectedKeys = keySlice{
testutils.MakeKey(roachpb.Key("\x02"), roachpb.KeyMax),
testutils.MakeKey(roachpb.Key("\x03"), roachpb.KeyMax),
roachpb.Key("\x04node-idgen"),
roachpb.Key("\x04store-idgen"),
}
// Add the initial keys for sql.
for _, kv := range GetBootstrapSchema().GetInitialValues() {
expectedKeys = append(expectedKeys, kv.Key)
}
// Resort the list. The sql values are not sorted.
sort.Sort(expectedKeys)
if !reflect.DeepEqual(foundKeys, expectedKeys) {
t.Errorf("expected keys mismatch:\n%s\n -- vs. -- \n\n%s",
formatKeys(foundKeys), formatKeys(expectedKeys))
}
// TODO(spencer): check values.
}
// TestRangeSplitMeta executes various splits (including at meta addressing)
// and checks that all created intents are resolved. This includes both intents
// which are resolved synchronously with EndTransaction and via RPC.
func TestRangeSplitMeta(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _ := createTestDB(t)
defer s.Stop()
splitKeys := []roachpb.RKey{roachpb.RKey("G"), mustMeta(roachpb.RKey("F")),
mustMeta(roachpb.RKey("K")), mustMeta(roachpb.RKey("H"))}
// Execute the consecutive splits.
for _, splitKey := range splitKeys {
log.Infof(context.Background(), "starting split at key %q...", splitKey)
if err := s.DB.AdminSplit(context.TODO(), roachpb.Key(splitKey)); err != nil {
t.Fatal(err)
}
log.Infof(context.Background(), "split at key %q complete", splitKey)
}
testutils.SucceedsSoon(t, func() error {
if _, _, _, err := engine.MVCCScan(context.Background(), s.Eng, keys.LocalMax, roachpb.KeyMax, math.MaxInt64, hlc.MaxTimestamp, true, nil); err != nil {
return errors.Errorf("failed to verify no dangling intents: %s", err)
}
return nil
})
}
// TestUpdateRangeAddressing verifies range addressing records are
// correctly updated on creation of new range descriptors.
func TestUpdateRangeAddressing(t *testing.T) {
defer leaktest.AfterTest(t)()
store, _, stopper := createTestStore(t)
defer stopper.Stop()
// When split is false, merging treats the right range as the merged
// range. With merging, expNewLeft indicates the addressing keys we
// expect to be removed.
testCases := []struct {
split bool
leftStart, leftEnd roachpb.RKey
rightStart, rightEnd roachpb.RKey
leftExpNew, rightExpNew [][]byte
}{
// Start out with whole range.
{false, roachpb.RKeyMin, roachpb.RKeyMax, roachpb.RKeyMin, roachpb.RKeyMax,
[][]byte{}, [][]byte{meta1Key(roachpb.RKeyMax), meta2Key(roachpb.RKeyMax)}},
// Split KeyMin-KeyMax at key "a".
{true, roachpb.RKeyMin, roachpb.RKey("a"), roachpb.RKey("a"), roachpb.RKeyMax,
[][]byte{meta1Key(roachpb.RKeyMax), meta2Key(roachpb.RKey("a"))}, [][]byte{meta2Key(roachpb.RKeyMax)}},
// Split "a"-KeyMax at key "z".
{true, roachpb.RKey("a"), roachpb.RKey("z"), roachpb.RKey("z"), roachpb.RKeyMax,
[][]byte{meta2Key(roachpb.RKey("z"))}, [][]byte{meta2Key(roachpb.RKeyMax)}},
// Split "a"-"z" at key "m".
{true, roachpb.RKey("a"), roachpb.RKey("m"), roachpb.RKey("m"), roachpb.RKey("z"),
[][]byte{meta2Key(roachpb.RKey("m"))}, [][]byte{meta2Key(roachpb.RKey("z"))}},
// Split KeyMin-"a" at meta2(m).
{true, roachpb.RKeyMin, metaKey(roachpb.RKey("m")), metaKey(roachpb.RKey("m")), roachpb.RKey("a"),
[][]byte{meta1Key(roachpb.RKey("m"))}, [][]byte{meta1Key(roachpb.RKeyMax), meta2Key(roachpb.RKey("a"))}},
// Split meta2(m)-"a" at meta2(z).
{true, metaKey(roachpb.RKey("m")), metaKey(roachpb.RKey("z")), metaKey(roachpb.RKey("z")), roachpb.RKey("a"),
[][]byte{meta1Key(roachpb.RKey("z"))}, [][]byte{meta1Key(roachpb.RKeyMax), meta2Key(roachpb.RKey("a"))}},
// Split meta2(m)-meta2(z) at meta2(r).
{true, metaKey(roachpb.RKey("m")), metaKey(roachpb.RKey("r")), metaKey(roachpb.RKey("r")), metaKey(roachpb.RKey("z")),
[][]byte{meta1Key(roachpb.RKey("r"))}, [][]byte{meta1Key(roachpb.RKey("z"))}},
// Now, merge all of our splits backwards...
// Merge meta2(m)-meta2(z).
{false, metaKey(roachpb.RKey("m")), metaKey(roachpb.RKey("r")), metaKey(roachpb.RKey("m")), metaKey(roachpb.RKey("z")),
[][]byte{meta1Key(roachpb.RKey("r"))}, [][]byte{meta1Key(roachpb.RKey("z"))}},
// Merge meta2(m)-"a".
{false, metaKey(roachpb.RKey("m")), metaKey(roachpb.RKey("z")), metaKey(roachpb.RKey("m")), roachpb.RKey("a"),
[][]byte{meta1Key(roachpb.RKey("z"))}, [][]byte{meta1Key(roachpb.RKeyMax), meta2Key(roachpb.RKey("a"))}},
// Merge KeyMin-"a".
{false, roachpb.RKeyMin, metaKey(roachpb.RKey("m")), roachpb.RKeyMin, roachpb.RKey("a"),
[][]byte{meta1Key(roachpb.RKey("m"))}, [][]byte{meta1Key(roachpb.RKeyMax), meta2Key(roachpb.RKey("a"))}},
// Merge "a"-"z".
{false, roachpb.RKey("a"), roachpb.RKey("m"), roachpb.RKey("a"), roachpb.RKey("z"),
[][]byte{meta2Key(roachpb.RKey("m"))}, [][]byte{meta2Key(roachpb.RKey("z"))}},
// Merge "a"-KeyMax.
{false, roachpb.RKey("a"), roachpb.RKey("z"), roachpb.RKey("a"), roachpb.RKeyMax,
[][]byte{meta2Key(roachpb.RKey("z"))}, [][]byte{meta2Key(roachpb.RKeyMax)}},
// Merge KeyMin-KeyMax.
{false, roachpb.RKeyMin, roachpb.RKey("a"), roachpb.RKeyMin, roachpb.RKeyMax,
[][]byte{meta2Key(roachpb.RKey("a"))}, [][]byte{meta1Key(roachpb.RKeyMax), meta2Key(roachpb.RKeyMax)}},
}
expMetas := metaSlice{}
for i, test := range testCases {
left := &roachpb.RangeDescriptor{RangeID: roachpb.RangeID(i * 2), StartKey: test.leftStart, EndKey: test.leftEnd}
right := &roachpb.RangeDescriptor{RangeID: roachpb.RangeID(i*2 + 1), StartKey: test.rightStart, EndKey: test.rightEnd}
b := &client.Batch{}
if test.split {
if err := splitRangeAddressing(b, left, right); err != nil {
t.Fatal(err)
}
} else {
if err := mergeRangeAddressing(b, left, right); err != nil {
t.Fatal(err)
}
}
if err := store.DB().Run(context.TODO(), b); err != nil {
t.Fatal(err)
}
// Scan meta keys directly from engine.
kvs, _, _, err := engine.MVCCScan(context.Background(), store.Engine(), keys.MetaMin, keys.MetaMax, math.MaxInt64, hlc.MaxTimestamp, true, nil)
if err != nil {
t.Fatal(err)
}
metas := metaSlice{}
for _, kv := range kvs {
scannedDesc := &roachpb.RangeDescriptor{}
if err := kv.Value.GetProto(scannedDesc); err != nil {
t.Fatal(err)
}
metas = append(metas, metaRecord{key: kv.Key, desc: scannedDesc})
}
// Continue to build up the expected metas slice, replacing any earlier
// version of same key.
addOrRemoveNew := func(keys [][]byte, desc *roachpb.RangeDescriptor, add bool) {
for _, n := range keys {
found := -1
for i := range expMetas {
if expMetas[i].key.Equal(roachpb.Key(n)) {
found = i
expMetas[i].desc = desc
break
}
}
if found == -1 && add {
expMetas = append(expMetas, metaRecord{key: n, desc: desc})
} else if found != -1 && !add {
expMetas = append(expMetas[:found], expMetas[found+1:]...)
}
}
}
addOrRemoveNew(test.leftExpNew, left, test.split /* on split, add; on merge, remove */)
addOrRemoveNew(test.rightExpNew, right, true)
sort.Sort(expMetas)
if test.split {
if log.V(1) {
log.Infof(context.Background(), "test case %d: split %q-%q at %q", i, left.StartKey, right.EndKey, left.EndKey)
}
} else {
if log.V(1) {
log.Infof(context.Background(), "test case %d: merge %q-%q + %q-%q", i, left.StartKey, left.EndKey, left.EndKey, right.EndKey)
}
}
for _, meta := range metas {
if log.V(1) {
log.Infof(context.Background(), "%q", meta.key)
}
}
if !reflect.DeepEqual(expMetas, metas) {
t.Errorf("expected metas don't match")
if len(expMetas) != len(metas) {
t.Errorf("len(expMetas) != len(metas); %d != %d", len(expMetas), len(metas))
} else {
for j, meta := range expMetas {
if !meta.key.Equal(metas[j].key) {
fmt.Printf("%d: expected %q vs %q\n", j, meta.key, metas[j].key)
}
if !reflect.DeepEqual(meta.desc, metas[j].desc) {
fmt.Printf("%d: expected %q vs %q and %s vs %s\n", j, meta.key, metas[j].key, meta.desc, metas[j].desc)
}
}
}
}
}
}