// TestLeaderAfterSplit verifies that a raft group created by a split
// elects a leader without waiting for an election timeout.
func TestLeaderAfterSplit(t *testing.T) {
defer leaktest.AfterTest(t)()
storeContext := storage.TestStoreContext()
storeContext.RaftElectionTimeoutTicks = 1000000
mtc := &multiTestContext{
storeContext: &storeContext,
}
mtc.Start(t, 3)
defer mtc.Stop()
mtc.replicateRange(1, 1, 2)
leftKey := roachpb.Key("a")
splitKey := roachpb.Key("m")
rightKey := roachpb.Key("z")
splitArgs := adminSplitArgs(roachpb.KeyMin, splitKey)
if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &splitArgs); pErr != nil {
t.Fatal(pErr)
}
incArgs := incrementArgs(leftKey, 1)
if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil {
t.Fatal(pErr)
}
incArgs = incrementArgs(rightKey, 2)
if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil {
t.Fatal(pErr)
}
}
// TestSplitSnapshotRace_SplitWins exercises one outcome of the
// split/snapshot race: The left side of the split propagates first,
// so the split completes before it sees a competing snapshot. This is
// the more common outcome in practice.
func TestSplitSnapshotRace_SplitWins(t *testing.T) {
defer leaktest.AfterTest(t)()
mtc, leftKey, rightKey := setupSplitSnapshotRace(t)
defer mtc.Stop()
// Bring the left range up first so that the split happens before it sees a snapshot.
for i := 1; i <= 3; i++ {
mtc.restartStore(i)
}
// Perform a write on the left range and wait for it to propagate.
incArgs := incrementArgs(leftKey, 10)
if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil {
t.Fatal(pErr)
}
mtc.waitForValues(leftKey, []int64{0, 11, 11, 11, 0, 0})
// Now wake the other stores up.
mtc.restartStore(4)
mtc.restartStore(5)
// Write to the right range.
incArgs = incrementArgs(rightKey, 20)
if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil {
t.Fatal(pErr)
}
mtc.waitForValues(rightKey, []int64{0, 0, 0, 25, 25, 25})
}
// TestMultiRangeScanWithMaxResults tests that commands which access multiple
// ranges with MaxResults parameter are carried out properly.
func TestMultiRangeScanWithMaxResults(t *testing.T) {
defer leaktest.AfterTest(t)()
testCases := []struct {
splitKeys []roachpb.Key
keys []roachpb.Key
}{
{[]roachpb.Key{roachpb.Key("m")},
[]roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}},
{[]roachpb.Key{roachpb.Key("h"), roachpb.Key("q")},
[]roachpb.Key{roachpb.Key("b"), roachpb.Key("f"), roachpb.Key("k"),
roachpb.Key("r"), roachpb.Key("w"), roachpb.Key("y")}},
}
for i, tc := range testCases {
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
ts := s.(*TestServer)
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = ts.stopper.ShouldDrain()
ds := kv.NewDistSender(&kv.DistSenderContext{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RPCRetryOptions: &retryOpts,
}, ts.Gossip())
tds := kv.NewTxnCoordSender(ds, ts.Clock(), ts.Ctx.Linearizable, tracing.NewTracer(),
ts.stopper, kv.NewTxnMetrics(metric.NewRegistry()))
for _, sk := range tc.splitKeys {
if err := ts.node.ctx.DB.AdminSplit(sk); err != nil {
t.Fatal(err)
}
}
for _, k := range tc.keys {
put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k))
if _, err := client.SendWrapped(tds, nil, put); err != nil {
t.Fatal(err)
}
}
// Try every possible ScanRequest startKey.
for start := 0; start < len(tc.keys); start++ {
// Try every possible maxResults, from 1 to beyond the size of key array.
for maxResults := 1; maxResults <= len(tc.keys)-start+1; maxResults++ {
scan := roachpb.NewScan(tc.keys[start], tc.keys[len(tc.keys)-1].Next(),
int64(maxResults))
reply, err := client.SendWrapped(tds, nil, scan)
if err != nil {
t.Fatal(err)
}
rows := reply.(*roachpb.ScanResponse).Rows
if start+maxResults <= len(tc.keys) && len(rows) != maxResults {
t.Errorf("%d: start=%s: expected %d rows, but got %d", i, tc.keys[start], maxResults, len(rows))
} else if start+maxResults == len(tc.keys)+1 && len(rows) != maxResults-1 {
t.Errorf("%d: expected %d rows, but got %d", i, maxResults-1, len(rows))
}
}
}
}
}
// TestStoreRangeMergeTwoEmptyRanges tries to merge two empty ranges
// together.
func TestStoreRangeMergeTwoEmptyRanges(t *testing.T) {
defer leaktest.AfterTest(t)()
defer config.TestingDisableTableSplits()()
store, stopper, _ := createTestStore(t)
defer stopper.Stop()
if _, _, err := createSplitRanges(store); err != nil {
t.Fatal(err)
}
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
_, err := client.SendWrapped(rg1(store), nil, &args)
if err != nil {
t.Fatal(err)
}
// Verify the merge by looking up keys from both ranges.
rangeA := store.LookupReplica([]byte("a"), nil)
rangeB := store.LookupReplica([]byte("c"), nil)
if !reflect.DeepEqual(rangeA, rangeB) {
t.Fatalf("ranges were not merged %+v=%+v", rangeA.Desc(), rangeB.Desc())
}
}
// TestStoreRangeMergeTwoEmptyRanges tries to merge two empty ranges together.
func TestStoreRangeMergeTwoEmptyRanges(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
if _, _, err := createSplitRanges(store); err != nil {
t.Fatal(err)
}
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
_, err := client.SendWrapped(rg1(store), nil, &args)
if err != nil {
t.Fatal(err)
}
// Verify the merge by looking up keys from both ranges.
replicaA := store.LookupReplica([]byte("a"), nil)
replicaB := store.LookupReplica([]byte("c"), nil)
if !reflect.DeepEqual(replicaA, replicaB) {
t.Fatalf("ranges were not merged %s!=%s", replicaA, replicaB)
}
}
// TestRejectFutureCommand verifies that lease holders reject commands that
// would cause a large time jump.
func TestRejectFutureCommand(t *testing.T) {
defer leaktest.AfterTest(t)()
const maxOffset = 100 * time.Millisecond
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(maxOffset)
mtc := multiTestContext{
clock: clock,
}
mtc.Start(t, 1)
defer mtc.Stop()
// First do a write. The first write will advance the clock by MaxOffset
// because of the read cache's low water mark.
getArgs := putArgs([]byte("b"), []byte("b"))
if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &getArgs); err != nil {
t.Fatal(err)
}
if now := clock.Now(); now.WallTime != int64(maxOffset) {
t.Fatalf("expected clock to advance to 100ms; got %s", now)
}
// The logical clock has advanced past the physical clock; increment
// the "physical" clock to catch up.
manual.Increment(int64(maxOffset))
startTime := manual.UnixNano()
// Commands with a future timestamp that is within the MaxOffset
// bound will be accepted and will cause the clock to advance.
for i := int64(0); i < 3; i++ {
incArgs := incrementArgs([]byte("a"), 5)
ts := hlc.ZeroTimestamp.Add(startTime+((i+1)*30)*int64(time.Millisecond), 0)
if _, err := client.SendWrappedWith(rg1(mtc.stores[0]), nil, roachpb.Header{Timestamp: ts}, &incArgs); err != nil {
t.Fatal(err)
}
}
if now := clock.Now(); now.WallTime != int64(190*time.Millisecond) {
t.Fatalf("expected clock to advance to 190ms; got %s", now)
}
// Once the accumulated offset reaches MaxOffset, commands will be rejected.
incArgs := incrementArgs([]byte("a"), 11)
ts := hlc.ZeroTimestamp.Add(int64((time.Duration(startTime)+maxOffset+1)*time.Millisecond), 0)
if _, err := client.SendWrappedWith(rg1(mtc.stores[0]), nil, roachpb.Header{Timestamp: ts}, &incArgs); err == nil {
t.Fatalf("expected clock offset error but got nil")
}
// The clock remained at 190ms and the final command was not executed.
if now := clock.Now(); now.WallTime != int64(190*time.Millisecond) {
t.Errorf("expected clock to advance to 190ms; got %s", now)
}
val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], roachpb.Key("a"), clock.Now(), true, nil)
if err != nil {
t.Fatal(err)
}
if v := mustGetInt(val); v != 15 {
t.Errorf("expected 15, got %v", v)
}
}
// TestSplitSnapshotRace_SnapshotWins exercises one outcome of the
// split/snapshot race: The right side of the split replicates first,
// so target node sees a raft snapshot before it has processed the split,
// so it still has a conflicting range.
func TestSplitSnapshotRace_SnapshotWins(t *testing.T) {
defer leaktest.AfterTest(t)()
mtc, leftKey, rightKey := setupSplitSnapshotRace(t)
defer mtc.Stop()
// Bring the right range up first.
for i := 3; i <= 5; i++ {
mtc.restartStore(i)
}
// Perform a write on the right range.
incArgs := incrementArgs(rightKey, 20)
if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil {
t.Fatal(pErr)
}
// It immediately propagates between nodes 4 and 5, but node 3
// remains at its old value. It can't accept the right-hand range
// because it conflicts with its not-yet-split copy of the left-hand
// range. This test is not completely deterministic: we want to make
// sure that node 3 doesn't panic when it receives the snapshot, but
// since it silently drops the message there is nothing we can wait
// for. There is a high probability that the message will have been
// received by the time that nodes 4 and 5 have processed their
// update.
mtc.waitForValues(rightKey, []int64{0, 0, 0, 2, 25, 25})
// Wake up the left-hand range. This will allow the left-hand
// range's split to complete and unblock the right-hand range.
mtc.restartStore(1)
mtc.restartStore(2)
// Perform writes on both sides. This is not strictly necessary but
// it helps wake up dormant ranges that would otherwise have to wait
// for retry timeouts.
incArgs = incrementArgs(leftKey, 10)
if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil {
t.Fatal(pErr)
}
mtc.waitForValues(leftKey, []int64{0, 11, 11, 11, 0, 0})
incArgs = incrementArgs(rightKey, 200)
if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil {
t.Fatal(pErr)
}
mtc.waitForValues(rightKey, []int64{0, 0, 0, 225, 225, 225})
}
// TestStoreRangeMergeNonCollocated attempts to merge two ranges
// that are not on the same stores.
func TestStoreRangeMergeNonCollocated(t *testing.T) {
defer leaktest.AfterTest(t)()
mtc := startMultiTestContext(t, 4)
defer mtc.Stop()
store := mtc.stores[0]
// Split into 3 ranges
argsSplit := adminSplitArgs(roachpb.KeyMin, []byte("d"))
if _, pErr := client.SendWrapped(rg1(store), nil, &argsSplit); pErr != nil {
t.Fatalf("Can't split range %s", pErr)
}
argsSplit = adminSplitArgs(roachpb.KeyMin, []byte("b"))
if _, pErr := client.SendWrapped(rg1(store), nil, &argsSplit); pErr != nil {
t.Fatalf("Can't split range %s", pErr)
}
rangeA := store.LookupReplica([]byte("a"), nil)
rangeADesc := rangeA.Desc()
rangeB := store.LookupReplica([]byte("c"), nil)
rangeBDesc := rangeB.Desc()
rangeC := store.LookupReplica([]byte("e"), nil)
rangeCDesc := rangeC.Desc()
if bytes.Equal(rangeADesc.StartKey, rangeBDesc.StartKey) {
log.Errorf(context.TODO(), "split ranges keys are equal %q!=%q", rangeADesc.StartKey, rangeBDesc.StartKey)
}
if bytes.Equal(rangeBDesc.StartKey, rangeCDesc.StartKey) {
log.Errorf(context.TODO(), "split ranges keys are equal %q!=%q", rangeBDesc.StartKey, rangeCDesc.StartKey)
}
if bytes.Equal(rangeADesc.StartKey, rangeCDesc.StartKey) {
log.Errorf(context.TODO(), "split ranges keys are equal %q!=%q", rangeADesc.StartKey, rangeCDesc.StartKey)
}
// Replicate the ranges to different sets of stores. Ranges A and C
// are collocated, but B is different.
mtc.replicateRange(rangeA.RangeID, 1, 2)
mtc.replicateRange(rangeB.RangeID, 1, 3)
mtc.replicateRange(rangeC.RangeID, 1, 2)
// Attempt to merge.
rangeADesc = rangeA.Desc()
argsMerge := adminMergeArgs(roachpb.Key(rangeADesc.StartKey))
if _, pErr := rangeA.AdminMerge(context.Background(), argsMerge, rangeADesc); !testutils.IsPError(pErr, "ranges not collocated") {
t.Fatalf("did not got expected error; got %s", pErr)
}
}
// TestStoreRangeMergeStats starts by splitting a range, then writing random data
// to both sides of the split. It then merges the ranges and verifies the merged
// range has stats consistent with recomputations.
func TestStoreRangeMergeStats(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, manual := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
// Split the range.
aDesc, bDesc, err := createSplitRanges(store)
if err != nil {
t.Fatal(err)
}
// Write some values left and right of the proposed split key.
writeRandomDataToRange(t, store, aDesc.RangeID, []byte("aaa"))
writeRandomDataToRange(t, store, bDesc.RangeID, []byte("ccc"))
// Get the range stats for both ranges now that we have data.
var msA, msB enginepb.MVCCStats
snap := store.Engine().NewSnapshot()
defer snap.Close()
if err := engine.MVCCGetRangeStats(context.Background(), snap, aDesc.RangeID, &msA); err != nil {
t.Fatal(err)
}
if err := engine.MVCCGetRangeStats(context.Background(), snap, bDesc.RangeID, &msB); err != nil {
t.Fatal(err)
}
// Stats should agree with recomputation.
if err := verifyRecomputedStats(snap, aDesc, msA, manual.UnixNano()); err != nil {
t.Fatalf("failed to verify range A's stats before split: %v", err)
}
if err := verifyRecomputedStats(snap, bDesc, msB, manual.UnixNano()); err != nil {
t.Fatalf("failed to verify range B's stats before split: %v", err)
}
manual.Increment(100)
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil {
t.Fatal(err)
}
rngMerged := store.LookupReplica(aDesc.StartKey, nil)
// Get the range stats for the merged range and verify.
snap = store.Engine().NewSnapshot()
defer snap.Close()
var msMerged enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), snap, rngMerged.RangeID, &msMerged); err != nil {
t.Fatal(err)
}
// Merged stats should agree with recomputation.
if err := verifyRecomputedStats(snap, rngMerged.Desc(), msMerged, manual.UnixNano()); err != nil {
t.Errorf("failed to verify range's stats after merge: %v", err)
}
}
// TestStoreRangeSplitAtRangeBounds verifies a range cannot be split
// at its start or end keys (would create zero-length range!). This
// sort of thing might happen in the wild if two split requests
// arrived for same key. The first one succeeds and second would try
// to split at the start of the newly split range.
func TestStoreRangeSplitAtRangeBounds(t *testing.T) {
defer leaktest.AfterTest(t)()
defer config.TestingDisableTableSplits()()
store, stopper, _ := createTestStore(t)
defer stopper.Stop()
args := adminSplitArgs(roachpb.KeyMin, []byte("a"))
if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil {
t.Fatal(err)
}
// This second split will try to split at end of first split range.
if _, err := client.SendWrapped(rg1(store), nil, &args); err == nil {
t.Fatalf("split succeeded unexpectedly")
}
// Now try to split at start of new range.
args = adminSplitArgs(roachpb.KeyMin, []byte("a"))
if _, err := client.SendWrapped(rg1(store), nil, &args); err == nil {
t.Fatalf("split succeeded unexpectedly")
}
}
// TestStoreRangeMergeLastRange verifies that merging the last range
// fails.
func TestStoreRangeMergeLastRange(t *testing.T) {
defer leaktest.AfterTest(t)()
defer config.TestingDisableTableSplits()()
store, stopper, _ := createTestStore(t)
defer stopper.Stop()
// Merge last range.
args := adminMergeArgs(roachpb.KeyMin)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); !testutils.IsPError(pErr, "cannot merge final range") {
t.Fatalf("expected 'cannot merge final range' error; got %s", pErr)
}
}
func BenchmarkStoreRangeSplit(b *testing.B) {
defer tracing.Disable()()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(b, sCtx)
defer stopper.Stop()
// Perform initial split of ranges.
sArgs := adminSplitArgs(roachpb.KeyMin, []byte("b"))
if _, err := client.SendWrapped(rg1(store), nil, &sArgs); err != nil {
b.Fatal(err)
}
// Write some values left and right of the split key.
aDesc := store.LookupReplica([]byte("a"), nil).Desc()
bDesc := store.LookupReplica([]byte("c"), nil).Desc()
writeRandomDataToRange(b, store, aDesc.RangeID, []byte("aaa"))
writeRandomDataToRange(b, store, bDesc.RangeID, []byte("ccc"))
// Merge the b range back into the a range.
mArgs := adminMergeArgs(roachpb.KeyMin)
if _, err := client.SendWrapped(rg1(store), nil, &mArgs); err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
// Split the range.
b.StartTimer()
if _, err := client.SendWrapped(rg1(store), nil, &sArgs); err != nil {
b.Fatal(err)
}
// Merge the ranges.
b.StopTimer()
if _, err := client.SendWrapped(rg1(store), nil, &mArgs); err != nil {
b.Fatal(err)
}
}
}
// TestStoreRangeMergeLastRange verifies that merging the last range
// fails.
func TestStoreRangeMergeLastRange(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
// Merge last range.
args := adminMergeArgs(roachpb.KeyMin)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); !testutils.IsPError(pErr, "cannot merge final range") {
t.Fatalf("expected 'cannot merge final range' error; got %s", pErr)
}
}
// TestStoreRangeSplitInsideRow verifies an attempt to split a range inside of
// a table row will cause a split at a boundary between rows.
func TestStoreRangeSplitInsideRow(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
// Manually create some the column keys corresponding to the table:
//
// CREATE TABLE t (id STRING PRIMARY KEY, col1 INT, col2 INT)
tableKey := keys.MakeTablePrefix(keys.MaxReservedDescID + 1)
rowKey := roachpb.Key(encoding.EncodeVarintAscending(append([]byte(nil), tableKey...), 1))
rowKey = encoding.EncodeStringAscending(encoding.EncodeVarintAscending(rowKey, 1), "a")
col1Key := keys.MakeFamilyKey(append([]byte(nil), rowKey...), 1)
col2Key := keys.MakeFamilyKey(append([]byte(nil), rowKey...), 2)
// We don't care about the value, so just store any old thing.
if err := store.DB().Put(col1Key, "column 1"); err != nil {
t.Fatal(err)
}
if err := store.DB().Put(col2Key, "column 2"); err != nil {
t.Fatal(err)
}
// Split between col1Key and col2Key by splitting before col2Key.
args := adminSplitArgs(col2Key, col2Key)
_, err := client.SendWrapped(rg1(store), nil, &args)
if err != nil {
t.Fatalf("%s: split unexpected error: %s", col1Key, err)
}
rng1 := store.LookupReplica(col1Key, nil)
rng2 := store.LookupReplica(col2Key, nil)
// Verify the two columns are still on the same range.
if !reflect.DeepEqual(rng1, rng2) {
t.Fatalf("%s: ranges differ: %+v vs %+v", roachpb.Key(col1Key), rng1, rng2)
}
// Verify we split on a row key.
if startKey := rng1.Desc().StartKey; !startKey.Equal(rowKey) {
t.Fatalf("%s: expected split on %s, but found %s",
roachpb.Key(col1Key), roachpb.Key(rowKey), startKey)
}
// Verify the previous range was split on a row key.
rng3 := store.LookupReplica(tableKey, nil)
if endKey := rng3.Desc().EndKey; !endKey.Equal(rowKey) {
t.Fatalf("%s: expected split on %s, but found %s",
roachpb.Key(col1Key), roachpb.Key(rowKey), endKey)
}
}
// TestStoreRangeSplitAtTablePrefix verifies a range can be split at
// UserTableDataMin and still gossip the SystemConfig properly.
func TestStoreRangeSplitAtTablePrefix(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
key := keys.MakeRowSentinelKey(append([]byte(nil), keys.UserTableDataMin...))
args := adminSplitArgs(key, key)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); pErr != nil {
t.Fatalf("%q: split unexpected error: %s", key, pErr)
}
var desc sqlbase.TableDescriptor
descBytes, err := protoutil.Marshal(&desc)
if err != nil {
t.Fatal(err)
}
// Update SystemConfig to trigger gossip.
if err := store.DB().Txn(func(txn *client.Txn) error {
txn.SetSystemConfigTrigger()
// We don't care about the values, just the keys.
k := sqlbase.MakeDescMetadataKey(sqlbase.ID(keys.MaxReservedDescID + 1))
return txn.Put(k, &desc)
}); err != nil {
t.Fatal(err)
}
successChan := make(chan struct{}, 1)
store.Gossip().RegisterCallback(gossip.KeySystemConfig, func(_ string, content roachpb.Value) {
contentBytes, err := content.GetBytes()
if err != nil {
t.Fatal(err)
}
if bytes.Contains(contentBytes, descBytes) {
select {
case successChan <- struct{}{}:
default:
}
}
})
select {
case <-time.After(time.Second):
t.Errorf("expected a schema gossip containing %q, but did not see one", descBytes)
case <-successChan:
}
}
func createSplitRanges(store *storage.Store) (*roachpb.RangeDescriptor, *roachpb.RangeDescriptor, *roachpb.Error) {
args := adminSplitArgs(roachpb.KeyMin, []byte("b"))
if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil {
return nil, nil, err
}
rangeADesc := store.LookupReplica([]byte("a"), nil).Desc()
rangeBDesc := store.LookupReplica([]byte("c"), nil).Desc()
if bytes.Equal(rangeADesc.StartKey, rangeBDesc.StartKey) {
log.Errorf(context.TODO(), "split ranges keys are equal %q!=%q", rangeADesc.StartKey, rangeBDesc.StartKey)
}
return rangeADesc, rangeBDesc, nil
}
// LookupRange returns the descriptor of the range containing key.
func (tc *TestCluster) LookupRange(key roachpb.Key) (roachpb.RangeDescriptor, error) {
rangeLookupReq := roachpb.RangeLookupRequest{
Span: roachpb.Span{
Key: keys.RangeMetaKey(keys.MustAddr(key)),
},
MaxRanges: 1,
ConsiderIntents: false,
}
resp, pErr := client.SendWrapped(tc.Servers[0].GetDistSender(), nil, &rangeLookupReq)
if pErr != nil {
return roachpb.RangeDescriptor{}, errors.Errorf(
"%q: lookup range unexpected error: %s", key, pErr)
}
return resp.(*roachpb.RangeLookupResponse).Ranges[0], nil
}
func (c *cluster) lookupRange(nodeIdx int, key roachpb.Key) (*roachpb.RangeDescriptor, error) {
req := &roachpb.RangeLookupRequest{
Span: roachpb.Span{
Key: keys.RangeMetaKey(keys.MustAddr(key)),
},
MaxRanges: 1,
ConsiderIntents: false,
}
sender := c.clients[nodeIdx].GetSender()
resp, pErr := client.SendWrapped(sender, nil, req)
if pErr != nil {
return nil, errors.Errorf("%s: lookup range: %s", key, pErr)
}
return &resp.(*roachpb.RangeLookupResponse).Ranges[0], nil
}
// TestProactiveRaftLogTruncate verifies that we proactively truncate the raft
// log even when replica scanning is disabled.
func TestProactiveRaftLogTruncate(t *testing.T) {
defer leaktest.AfterTest(t)()
store, _, stopper := createTestStore(t)
defer stopper.Stop()
store.scanner.SetDisabled(true)
r, err := store.GetReplica(1)
if err != nil {
t.Fatal(err)
}
r.mu.Lock()
oldFirstIndex, err := r.FirstIndex()
r.mu.Unlock()
if err != nil {
t.Fatal(err)
}
// Write a few keys to the range. While writing these keys, the raft log
// should be proactively truncated even though replica scanning is disabled.
for i := 0; i < 2*RaftLogQueueStaleThreshold; i++ {
key := roachpb.Key(fmt.Sprintf("key%02d", i))
args := putArgs(key, []byte(fmt.Sprintf("value%02d", i)))
if _, err := client.SendWrapped(store.testSender(), nil, &args); err != nil {
t.Fatal(err)
}
}
// Wait for any asynchronous tasks to finish.
stopper.Quiesce()
r.mu.Lock()
newFirstIndex, err := r.FirstIndex()
r.mu.Unlock()
if err != nil {
t.Fatal(err)
}
if newFirstIndex <= oldFirstIndex {
t.Errorf("log was not correctly truncated, old first index:%d, current first index:%d",
oldFirstIndex, newFirstIndex)
}
}
// SplitRange splits the range containing splitKey.
// The right range created by the split starts at the split key and extends to the
// original range's end key.
// Returns the new descriptors of the left and right ranges.
//
// splitKey must correspond to a SQL table key (it must end with a family ID /
// col ID).
func (tc *TestCluster) SplitRange(
splitKey roachpb.Key,
) (*roachpb.RangeDescriptor, *roachpb.RangeDescriptor, error) {
splitRKey, err := keys.Addr(splitKey)
if err != nil {
return nil, nil, err
}
origRangeDesc, err := tc.LookupRange(splitKey)
if err != nil {
return nil, nil, err
}
if origRangeDesc.StartKey.Equal(splitRKey) {
return nil, nil, errors.Errorf(
"cannot split range %+v at start key %q", origRangeDesc, splitKey)
}
splitReq := roachpb.AdminSplitRequest{
Span: roachpb.Span{
Key: splitKey,
},
SplitKey: splitKey,
}
_, pErr := client.SendWrapped(tc.Servers[0].GetDistSender(), nil, &splitReq)
if pErr != nil {
return nil, nil, errors.Errorf(
"%q: split unexpected error: %s", splitReq.SplitKey, pErr)
}
leftRangeDesc := new(roachpb.RangeDescriptor)
rightRangeDesc := new(roachpb.RangeDescriptor)
if err := tc.Servers[0].DB().GetProto(
keys.RangeDescriptorKey(origRangeDesc.StartKey), leftRangeDesc); err != nil {
return nil, nil, errors.Wrap(err, "could not look up left-hand side descriptor")
}
// The split point might not be exactly the one we requested (it can be
// adjusted slightly so we don't split in the middle of SQL rows). Update it
// to the real point.
splitRKey = leftRangeDesc.EndKey
if err := tc.Servers[0].DB().GetProto(
keys.RangeDescriptorKey(splitRKey), rightRangeDesc); err != nil {
return nil, nil, errors.Wrap(err, "could not look up right-hand side descriptor")
}
return leftRangeDesc, rightRangeDesc, nil
}
// TestStoreRangeSplitAtIllegalKeys verifies a range cannot be split
// at illegal keys.
func TestStoreRangeSplitAtIllegalKeys(t *testing.T) {
defer leaktest.AfterTest(t)()
store, stopper, _ := createTestStore(t)
defer stopper.Stop()
for _, key := range []roachpb.Key{
keys.Meta1Prefix,
testutils.MakeKey(keys.Meta1Prefix, []byte("a")),
testutils.MakeKey(keys.Meta1Prefix, roachpb.RKeyMax),
keys.Meta2KeyMax,
keys.MakeTablePrefix(10 /* system descriptor ID */),
} {
args := adminSplitArgs(roachpb.KeyMin, key)
_, pErr := client.SendWrapped(rg1(store), nil, &args)
if !testutils.IsPError(pErr, "cannot split") {
t.Errorf("%q: unexpected split error %s", key, pErr)
}
}
}
// TestStoreRangeSplitConcurrent verifies that concurrent range splits
// of the same range are executed serially, and all but the first fail
// because the split key is invalid after the first split succeeds.
func TestStoreRangeSplitConcurrent(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
splitKey := roachpb.Key("a")
concurrentCount := int32(10)
wg := sync.WaitGroup{}
wg.Add(int(concurrentCount))
failureCount := int32(0)
for i := int32(0); i < concurrentCount; i++ {
go func() {
args := adminSplitArgs(roachpb.KeyMin, splitKey)
_, pErr := client.SendWrapped(rg1(store), nil, &args)
if pErr != nil {
atomic.AddInt32(&failureCount, 1)
}
wg.Done()
}()
}
wg.Wait()
if failureCount != concurrentCount-1 {
t.Fatalf("concurrent splits succeeded unexpectedly; failureCount=%d", failureCount)
}
// Verify everything ended up as expected.
if a, e := store.ReplicaCount(), 2; a != e {
t.Fatalf("expected %d stores after concurrent splits; actual count=%d", e, a)
}
rng := store.LookupReplica(roachpb.RKeyMin, nil)
newRng := store.LookupReplica(roachpb.RKey(splitKey), nil)
if !bytes.Equal(newRng.Desc().StartKey, splitKey) || !bytes.Equal(splitKey, rng.Desc().EndKey) {
t.Errorf("ranges mismatched, wanted %q=%q=%q", newRng.Desc().StartKey, splitKey, rng.Desc().EndKey)
}
if !bytes.Equal(newRng.Desc().EndKey, roachpb.RKeyMax) || !bytes.Equal(rng.Desc().StartKey, roachpb.RKeyMin) {
t.Errorf("new ranges do not cover KeyMin-KeyMax, but only %q-%q", rng.Desc().StartKey, newRng.Desc().EndKey)
}
}
// Test that a lease extension (a RequestLeaseRequest that doesn't change the
// lease holder) is not blocked by ongoing reads.
// Note that lease transfers are blocked by reads through their
// PostCommitTrigger.noConcurrentReads.
func TestLeaseExtensionNotBlockedByRead(t *testing.T) {
defer leaktest.AfterTest(t)()
readBlocked := make(chan struct{})
cmdFilter := func(fArgs storagebase.FilterArgs) *roachpb.Error {
if fArgs.Hdr.UserPriority == 42 {
// Signal that the read is blocked.
readBlocked <- struct{}{}
// Wait for read to be unblocked.
<-readBlocked
}
return nil
}
srv, _, _ := serverutils.StartServer(t,
base.TestServerArgs{
Knobs: base.TestingKnobs{
Store: &storage.StoreTestingKnobs{
TestingCommandFilter: cmdFilter,
},
},
})
s := srv.(*server.TestServer)
defer s.Stopper().Stop()
// Start a read and wait for it to block.
key := roachpb.Key("a")
errChan := make(chan error)
go func() {
getReq := roachpb.GetRequest{
Span: roachpb.Span{
Key: key,
},
}
if _, pErr := client.SendWrappedWith(s.GetDistSender(), nil,
roachpb.Header{UserPriority: 42},
&getReq); pErr != nil {
errChan <- pErr.GoError()
}
}()
select {
case err := <-errChan:
t.Fatal(err)
case <-readBlocked:
// Send the lease request.
// We change the key slightly, otherwise the lease request will be blocked
// by the read through the command queue.
// TODO(andrei): don't change the key anymore once lease requests don't go
// through the command queue any more.
leaseHdrKey := roachpb.Key(append(key, 0x00))
rKey, err := keys.Addr(leaseHdrKey)
if err != nil {
t.Fatal(err)
}
_, repDesc, err := s.Stores().LookupReplica(rKey, nil)
if err != nil {
t.Fatal(err)
}
leaseReq := roachpb.RequestLeaseRequest{
Span: roachpb.Span{
Key: leaseHdrKey,
},
Lease: roachpb.Lease{
Start: s.Clock().Now(),
StartStasis: s.Clock().Now().Add(time.Second.Nanoseconds(), 0),
Expiration: s.Clock().Now().Add(2*time.Second.Nanoseconds(), 0),
Replica: repDesc,
},
}
if _, pErr := client.SendWrapped(s.GetDistSender(), nil, &leaseReq); pErr != nil {
t.Fatal(pErr)
}
// Unblock the read.
readBlocked <- struct{}{}
}
}
func TestRangeTransferLease(t *testing.T) {
defer leaktest.AfterTest(t)()
ctx := storage.TestStoreContext()
var filterMu syncutil.Mutex
var filter func(filterArgs storagebase.FilterArgs) *roachpb.Error
ctx.TestingKnobs.TestingCommandFilter =
func(filterArgs storagebase.FilterArgs) *roachpb.Error {
filterMu.Lock()
filterCopy := filter
filterMu.Unlock()
if filterCopy != nil {
return filterCopy(filterArgs)
}
return nil
}
var waitForTransferBlocked atomic.Value
waitForTransferBlocked.Store(false)
transferBlocked := make(chan struct{})
ctx.TestingKnobs.LeaseTransferBlockedOnExtensionEvent = func(
_ roachpb.ReplicaDescriptor) {
if waitForTransferBlocked.Load().(bool) {
transferBlocked <- struct{}{}
waitForTransferBlocked.Store(false)
}
}
mtc := &multiTestContext{}
mtc.storeContext = &ctx
mtc.Start(t, 2)
defer mtc.Stop()
// First, do a write; we'll use it to determine when the dust has settled.
leftKey := roachpb.Key("a")
incArgs := incrementArgs(leftKey, 1)
if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil {
t.Fatal(pErr)
}
// Get the left range's ID.
rangeID := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil).RangeID
// Replicate the left range onto node 1.
mtc.replicateRange(rangeID, 1)
replica0 := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil)
replica1 := mtc.stores[1].LookupReplica(roachpb.RKey("a"), nil)
gArgs := getArgs(leftKey)
replica0Desc, err := replica0.GetReplicaDescriptor()
if err != nil {
t.Fatal(err)
}
// Check that replica0 can serve reads OK.
if _, pErr := client.SendWrappedWith(
mtc.senders[0], nil, roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil {
t.Fatal(pErr)
}
{
// Transferring the lease to ourself should be a no-op.
origLeasePtr, _ := replica0.GetLease()
origLease := *origLeasePtr
if err := replica0.AdminTransferLease(replica0Desc.StoreID); err != nil {
t.Fatal(err)
}
newLeasePtr, _ := replica0.GetLease()
if origLeasePtr != newLeasePtr || origLease != *newLeasePtr {
t.Fatalf("expected %+v, but found %+v", origLeasePtr, newLeasePtr)
}
}
{
// An invalid target should result in an error.
const expected = "unable to find store .* in range"
if err := replica0.AdminTransferLease(1000); !testutils.IsError(err, expected) {
t.Fatalf("expected %s, but found %v", expected, err)
}
}
// Move the lease to store 1.
var newHolderDesc roachpb.ReplicaDescriptor
util.SucceedsSoon(t, func() error {
var err error
newHolderDesc, err = replica1.GetReplicaDescriptor()
return err
})
if err := replica0.AdminTransferLease(newHolderDesc.StoreID); err != nil {
t.Fatal(err)
}
// Check that replica0 doesn't serve reads any more.
replica0Desc, err = replica0.GetReplicaDescriptor()
if err != nil {
t.Fatal(err)
}
_, pErr := client.SendWrappedWith(
mtc.senders[0], nil, roachpb.Header{Replica: replica0Desc}, &gArgs)
nlhe, ok := pErr.GetDetail().(*roachpb.NotLeaseHolderError)
if !ok {
t.Fatalf("expected %T, got %s", &roachpb.NotLeaseHolderError{}, pErr)
}
if *(nlhe.LeaseHolder) != newHolderDesc {
t.Fatalf("expected lease holder %+v, got %+v",
newHolderDesc, nlhe.LeaseHolder)
}
// Check that replica1 now has the lease (or gets it soon).
util.SucceedsSoon(t, func() error {
if _, pErr := client.SendWrappedWith(
mtc.senders[1], nil, roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil {
return pErr.GoError()
}
return nil
})
replica1Lease, _ := replica1.GetLease()
// Verify the timestamp cache low water. Because we executed a transfer lease
// request, the low water should be set to the new lease start time which is
// less than the previous lease's expiration time.
if lowWater := replica1.GetTimestampCacheLowWater(); lowWater != replica1Lease.Start {
t.Fatalf("expected timestamp cache low water %s, but found %s",
replica1Lease.Start, lowWater)
}
// Make replica1 extend its lease and transfer the lease immediately after
// that. Test that the transfer still happens (it'll wait until the extension
// is done).
extensionSem := make(chan struct{})
filterMu.Lock()
filter = func(filterArgs storagebase.FilterArgs) *roachpb.Error {
if filterArgs.Sid != mtc.stores[1].Ident.StoreID {
return nil
}
llReq, ok := filterArgs.Req.(*roachpb.RequestLeaseRequest)
if !ok {
return nil
}
if llReq.Lease.Replica == newHolderDesc {
// Notify the main thread that the extension is in progress and wait for
// the signal to proceed.
filterMu.Lock()
filter = nil
filterMu.Unlock()
extensionSem <- struct{}{}
<-extensionSem
}
return nil
}
filterMu.Unlock()
// Initiate an extension.
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
shouldRenewTS := replica1Lease.StartStasis.Add(-1, 0)
mtc.manualClock.Set(shouldRenewTS.WallTime + 1)
if _, pErr := client.SendWrappedWith(
mtc.senders[1], nil,
roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil {
panic(pErr)
}
}()
<-extensionSem
waitForTransferBlocked.Store(true)
// Initiate a transfer.
wg.Add(1)
go func() {
defer wg.Done()
// Transfer back from replica1 to replica0.
if err := replica1.AdminTransferLease(replica0Desc.StoreID); err != nil {
panic(err)
}
}()
// Wait for the transfer to be blocked by the extension.
<-transferBlocked
// Now unblock the extension.
extensionSem <- struct{}{}
// Check that the transfer to replica1 eventually happens.
util.SucceedsSoon(t, func() error {
if _, pErr := client.SendWrappedWith(
mtc.senders[0], nil,
roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil {
return pErr.GoError()
}
return nil
})
filterMu.Lock()
filter = nil
filterMu.Unlock()
wg.Wait()
}
// TestRangeLookupUseReverse tests whether the results and the results count
// are correct when scanning in reverse order.
func TestRangeLookupUseReverse(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
// Init test ranges:
// ["","a"), ["a","c"), ["c","e"), ["e","g") and ["g","\xff\xff").
splits := []roachpb.AdminSplitRequest{
adminSplitArgs(roachpb.Key("g"), roachpb.Key("g")),
adminSplitArgs(roachpb.Key("e"), roachpb.Key("e")),
adminSplitArgs(roachpb.Key("c"), roachpb.Key("c")),
adminSplitArgs(roachpb.Key("a"), roachpb.Key("a")),
}
for _, split := range splits {
_, pErr := client.SendWrapped(rg1(store), nil, &split)
if pErr != nil {
t.Fatalf("%q: split unexpected error: %s", split.SplitKey, pErr)
}
}
// Resolve the intents.
scanArgs := roachpb.ScanRequest{
Span: roachpb.Span{
Key: keys.RangeMetaKey(roachpb.RKeyMin.Next()),
EndKey: keys.RangeMetaKey(roachpb.RKeyMax),
},
}
util.SucceedsSoon(t, func() error {
_, pErr := client.SendWrapped(rg1(store), nil, &scanArgs)
return pErr.GoError()
})
revScanArgs := func(key []byte, maxResults int32) *roachpb.RangeLookupRequest {
return &roachpb.RangeLookupRequest{
Span: roachpb.Span{
Key: key,
},
MaxRanges: maxResults,
Reverse: true,
}
}
// Test cases.
testCases := []struct {
request *roachpb.RangeLookupRequest
expected []roachpb.RangeDescriptor
expectedPre []roachpb.RangeDescriptor
}{
// Test key in the middle of the range.
{
request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("f")), 2),
// ["e","g") and ["c","e").
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")},
},
},
// Test key in the end key of the range.
{
request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("g")), 3),
// ["e","g"), ["c","e") and ["a","c").
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")},
{StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("c")},
},
},
{
request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("e")), 2),
// ["c","e") and ["a","c").
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("c")},
},
},
// Test Meta2KeyMax.
{
request: revScanArgs(keys.Meta2KeyMax, 2),
// ["e","g") and ["g","\xff\xff")
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("g"), EndKey: roachpb.RKey("\xff\xff")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")},
},
},
// Test Meta1KeyMax.
{
request: revScanArgs(keys.Meta1KeyMax, 1),
// ["","a")
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKeyMin, EndKey: roachpb.RKey("a")},
},
},
}
for testIdx, test := range testCases {
resp, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
ReadConsistency: roachpb.INCONSISTENT,
}, test.request)
if pErr != nil {
t.Fatalf("%d: RangeLookup error: %s", testIdx, pErr)
}
rlReply := resp.(*roachpb.RangeLookupResponse)
// Checks the results count.
if int32(len(rlReply.Ranges))+int32(len(rlReply.PrefetchedRanges)) != test.request.MaxRanges {
t.Fatalf("%d: returned results count, expected %d,but got %d", testIdx, test.request.MaxRanges, len(rlReply.Ranges))
}
// Checks the range descriptors.
for _, rngSlice := range []struct {
expect, reply []roachpb.RangeDescriptor
}{
{test.expected, rlReply.Ranges},
{test.expectedPre, rlReply.PrefetchedRanges},
} {
for i, rng := range rngSlice.expect {
if !(rng.StartKey.Equal(rngSlice.reply[i].StartKey) && rng.EndKey.Equal(rngSlice.reply[i].EndKey)) {
t.Fatalf("%d: returned range is not correct, expected %v ,but got %v", testIdx, rng, rngSlice.reply[i])
}
}
}
}
}
// TestStoreRangeMergeMetadataCleanup tests that all metadata of a
// subsumed range is cleaned up on merge.
func TestStoreRangeMergeMetadataCleanup(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
scan := func(f func(roachpb.KeyValue) (bool, error)) {
if _, err := engine.MVCCIterate(context.Background(), store.Engine(), roachpb.KeyMin, roachpb.KeyMax, hlc.ZeroTimestamp, true, nil, false, f); err != nil {
t.Fatal(err)
}
}
content := roachpb.Key("testing!")
// Write some values left of the proposed split key.
pArgs := putArgs([]byte("aaa"), content)
if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil {
t.Fatal(err)
}
// Collect all the keys.
preKeys := make(map[string]struct{})
scan(func(kv roachpb.KeyValue) (bool, error) {
preKeys[string(kv.Key)] = struct{}{}
return false, nil
})
// Split the range.
_, bDesc, err := createSplitRanges(store)
if err != nil {
t.Fatal(err)
}
// Write some values right of the split key.
pArgs = putArgs([]byte("ccc"), content)
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: bDesc.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil {
t.Fatal(err)
}
// Collect all the keys again.
postKeys := make(map[string]struct{})
scan(func(kv roachpb.KeyValue) (bool, error) {
postKeys[string(kv.Key)] = struct{}{}
return false, nil
})
// Compute the new keys.
for k := range preKeys {
delete(postKeys, k)
}
// Keep only the subsumed range's local keys.
localRangeKeyPrefix := string(keys.MakeRangeIDPrefix(bDesc.RangeID))
for k := range postKeys {
if !strings.HasPrefix(k, localRangeKeyPrefix) {
delete(postKeys, k)
}
}
if numKeys := len(postKeys); numKeys > 0 {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%d keys were not cleaned up:\n", numKeys)
for k := range postKeys {
fmt.Fprintf(&buf, "%q\n", k)
}
t.Fatal(buf.String())
}
}
// TestRaftLogQueue verifies that the raft log queue correctly truncates the
// raft log.
func TestRaftLogQueue(t *testing.T) {
defer leaktest.AfterTest(t)()
var mtc multiTestContext
// Set maxBytes to something small so we can trigger the raft log truncation
// without adding 64MB of logs.
const maxBytes = 1 << 16
defer config.TestingSetDefaultZoneConfig(config.ZoneConfig{
RangeMaxBytes: maxBytes,
})()
// Turn off raft elections so the raft leader won't change out from under
// us in this test.
sc := storage.TestStoreContext()
sc.RaftTickInterval = time.Hour * 24
sc.RaftElectionTimeoutTicks = 1000000
mtc.storeContext = &sc
mtc.Start(t, 3)
defer mtc.Stop()
// Write a single value to ensure we have a leader.
pArgs := putArgs([]byte("key"), []byte("value"))
if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &pArgs); err != nil {
t.Fatal(err)
}
// Get the raft leader (and ensure one exists).
rangeID := mtc.stores[0].LookupReplica([]byte("a"), nil).RangeID
raftLeaderRepl := mtc.getRaftLeader(rangeID)
if raftLeaderRepl == nil {
t.Fatalf("could not find raft leader replica for range %d", rangeID)
}
originalIndex, err := raftLeaderRepl.GetFirstIndex()
if err != nil {
t.Fatal(err)
}
// Disable splits since we're increasing the raft log with puts.
for _, store := range mtc.stores {
store.DisableSplitQueue(true)
}
// Write a collection of values to increase the raft log.
value := bytes.Repeat([]byte("a"), 1000) // 1KB
for size := int64(0); size < 2*maxBytes; size += int64(len(value)) {
pArgs = putArgs([]byte(fmt.Sprintf("key-%d", size)), value)
if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &pArgs); err != nil {
t.Fatal(err)
}
}
// Sadly, occasionally the queue has a race with the force processing so
// this succeeds within will captures those rare cases.
var afterTruncationIndex uint64
util.SucceedsSoon(t, func() error {
// Force a truncation check.
for _, store := range mtc.stores {
store.ForceRaftLogScanAndProcess()
}
// Ensure that firstIndex has increased indicating that the log
// truncation has occurred.
var err error
afterTruncationIndex, err = raftLeaderRepl.GetFirstIndex()
if err != nil {
t.Fatal(err)
}
if afterTruncationIndex <= originalIndex {
return errors.Errorf("raft log has not been truncated yet, afterTruncationIndex:%d originalIndex:%d",
afterTruncationIndex, originalIndex)
}
return nil
})
// Force a truncation check again to ensure that attempting to truncate an
// already truncated log has no effect.
for _, store := range mtc.stores {
store.ForceRaftLogScanAndProcess()
}
after2ndTruncationIndex, err := raftLeaderRepl.GetFirstIndex()
if err != nil {
t.Fatal(err)
}
if afterTruncationIndex > after2ndTruncationIndex {
t.Fatalf("second truncation destroyed state: afterTruncationIndex:%d after2ndTruncationIndex:%d",
afterTruncationIndex, after2ndTruncationIndex)
}
}
// TestGetTruncatableIndexes verifies that old raft log entries are correctly
// removed.
func TestGetTruncatableIndexes(t *testing.T) {
defer leaktest.AfterTest(t)()
store, _, stopper := createTestStore(t)
defer stopper.Stop()
if _, err := store.GetReplica(0); err == nil {
t.Fatal("expected GetRange to fail on missing range")
}
store.SetRaftLogQueueActive(false)
// Test on a new range which should not have a raft group yet.
rngNew := createRange(store, 100, roachpb.RKey("a"), roachpb.RKey("c"))
truncatableIndexes, oldestIndex, err := getTruncatableIndexes(rngNew)
if err != nil {
t.Errorf("expected no error, got %s", err)
}
if truncatableIndexes != 0 {
t.Errorf("expected 0 for truncatable index, got %d", truncatableIndexes)
}
if oldestIndex != 0 {
t.Errorf("expected 0 for oldest index, got %d", oldestIndex)
}
r, err := store.GetReplica(1)
if err != nil {
t.Fatal(err)
}
r.mu.Lock()
firstIndex, err := r.FirstIndex()
r.mu.Unlock()
if err != nil {
t.Fatal(err)
}
// Write a few keys to the range.
for i := 0; i < RaftLogQueueStaleThreshold+1; i++ {
key := roachpb.Key(fmt.Sprintf("key%02d", i))
args := putArgs(key, []byte(fmt.Sprintf("value%02d", i)))
if _, err := client.SendWrapped(store.testSender(), nil, &args); err != nil {
t.Fatal(err)
}
}
truncatableIndexes, oldestIndex, err = getTruncatableIndexes(r)
if err != nil {
t.Errorf("expected no error, got %s", err)
}
if truncatableIndexes == 0 {
t.Errorf("expected a value for truncatable index, got 0")
}
if oldestIndex < firstIndex {
t.Errorf("expected oldest index (%d) to be greater than or equal to first index (%d)", oldestIndex,
firstIndex)
}
// Enable the raft log scanner and and force a truncation.
store.SetRaftLogQueueActive(true)
store.ForceRaftLogScanAndProcess()
// Wait for tasks to finish, in case the processLoop grabbed the event
// before ForceRaftLogScanAndProcess but is still working on it.
stopper.Quiesce()
r.mu.Lock()
newFirstIndex, err := r.FirstIndex()
r.mu.Unlock()
if err != nil {
t.Fatal(err)
}
if newFirstIndex <= firstIndex {
t.Errorf("log was not correctly truncated, older first index:%d, current first index:%d", firstIndex,
newFirstIndex)
}
// Once truncated, we should have no truncatable indexes. If this turns out
// to be flaky, we can remove it as the same functionality is tested in
// client_raft_log_queue_test.
util.SucceedsSoon(t, func() error {
store.ForceRaftLogScanAndProcess()
truncatableIndexes, oldestIndex, err := getTruncatableIndexes(rngNew)
if err != nil {
return errors.Errorf("expected no error, got %s", err)
}
if truncatableIndexes != 0 {
return errors.Errorf("expected 0 for truncatable index, got %d", truncatableIndexes)
}
if oldestIndex != 0 {
return errors.Errorf("expected 0 for oldest index, got %d", oldestIndex)
}
return nil
})
}
// TestMultiRangeScanDeleteRange tests that commands which access multiple
// ranges are carried out properly.
func TestMultiRangeScanDeleteRange(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
ts := s.(*TestServer)
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = ts.stopper.ShouldQuiesce()
ds := kv.NewDistSender(&kv.DistSenderConfig{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RPCRetryOptions: &retryOpts,
}, ts.Gossip())
ctx := tracing.WithTracer(context.Background(), tracing.NewTracer())
tds := kv.NewTxnCoordSender(ctx, ds, s.Clock(), ts.Ctx.Linearizable,
ts.stopper, kv.MakeTxnMetrics())
if err := ts.node.ctx.DB.AdminSplit("m"); err != nil {
t.Fatal(err)
}
writes := []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}
get := &roachpb.GetRequest{
Span: roachpb.Span{Key: writes[0]},
}
get.EndKey = writes[len(writes)-1]
if _, err := client.SendWrapped(tds, nil, get); err == nil {
t.Errorf("able to call Get with a key range: %v", get)
}
var delTS hlc.Timestamp
for i, k := range writes {
put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k))
reply, err := client.SendWrapped(tds, nil, put)
if err != nil {
t.Fatal(err)
}
scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next())
reply, err = client.SendWrapped(tds, nil, scan)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if sr.Txn != nil {
// This was the other way around at some point in the past.
// Same below for Delete, etc.
t.Errorf("expected no transaction in response header")
}
if rows := sr.Rows; len(rows) != i+1 {
t.Fatalf("expected %d rows, but got %d", i+1, len(rows))
}
}
del := &roachpb.DeleteRangeRequest{
Span: roachpb.Span{
Key: writes[0],
EndKey: roachpb.Key(writes[len(writes)-1]).Next(),
},
ReturnKeys: true,
}
reply, err := client.SendWrappedWith(tds, nil, roachpb.Header{Timestamp: delTS}, del)
if err != nil {
t.Fatal(err)
}
dr := reply.(*roachpb.DeleteRangeResponse)
if dr.Txn != nil {
t.Errorf("expected no transaction in response header")
}
if !reflect.DeepEqual(dr.Keys, writes) {
t.Errorf("expected %d keys to be deleted, but got %d instead", writes, dr.Keys)
}
scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next())
txn := &roachpb.Transaction{Name: "MyTxn"}
reply, err = client.SendWrappedWith(tds, nil, roachpb.Header{Txn: txn}, scan)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if txn := sr.Txn; txn == nil || txn.Name != "MyTxn" {
t.Errorf("wanted Txn to persist, but it changed to %v", txn)
}
if rows := sr.Rows; len(rows) > 0 {
t.Fatalf("scan after delete returned rows: %v", rows)
}
}
// TestStoreRangeMergeWithData attempts to merge two collocate ranges
// each containing data.
func TestStoreRangeMergeWithData(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
content := roachpb.Key("testing!")
aDesc, bDesc, err := createSplitRanges(store)
if err != nil {
t.Fatal(err)
}
// Write some values left and right of the proposed split key.
pArgs := putArgs([]byte("aaa"), content)
if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil {
t.Fatal(err)
}
pArgs = putArgs([]byte("ccc"), content)
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: bDesc.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Confirm the values are there.
gArgs := getArgs([]byte("aaa"))
if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("ccc"))
if reply, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: bDesc.RangeID,
}, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil {
t.Fatal(err)
}
// Verify no intents remains on range descriptor keys.
for _, key := range []roachpb.Key{keys.RangeDescriptorKey(aDesc.StartKey), keys.RangeDescriptorKey(bDesc.StartKey)} {
if _, _, err := engine.MVCCGet(context.Background(), store.Engine(), key, store.Clock().Now(), true, nil); err != nil {
t.Fatal(err)
}
}
// Verify the merge by looking up keys from both ranges.
rangeA := store.LookupReplica([]byte("a"), nil)
rangeB := store.LookupReplica([]byte("c"), nil)
rangeADesc := rangeA.Desc()
rangeBDesc := rangeB.Desc()
if !reflect.DeepEqual(rangeA, rangeB) {
t.Fatalf("ranges were not merged %+v=%+v", rangeADesc, rangeBDesc)
}
if !bytes.Equal(rangeADesc.StartKey, roachpb.RKeyMin) {
t.Fatalf("The start key is not equal to KeyMin %q=%q", rangeADesc.StartKey, roachpb.RKeyMin)
}
if !bytes.Equal(rangeADesc.EndKey, roachpb.RKeyMax) {
t.Fatalf("The end key is not equal to KeyMax %q=%q", rangeADesc.EndKey, roachpb.RKeyMax)
}
// Try to get values from after the merge.
gArgs = getArgs([]byte("aaa"))
if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("ccc"))
if reply, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeB.RangeID,
}, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Put new values after the merge on both sides.
pArgs = putArgs([]byte("aaaa"), content)
if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil {
t.Fatal(err)
}
pArgs = putArgs([]byte("cccc"), content)
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeB.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Try to get the newly placed values.
gArgs = getArgs([]byte("aaaa"))
if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("cccc"))
if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
}