// TestLeaderAfterSplit verifies that a raft group created by a split // elects a leader without waiting for an election timeout. func TestLeaderAfterSplit(t *testing.T) { defer leaktest.AfterTest(t)() storeContext := storage.TestStoreContext() storeContext.RaftElectionTimeoutTicks = 1000000 mtc := &multiTestContext{ storeContext: &storeContext, } mtc.Start(t, 3) defer mtc.Stop() mtc.replicateRange(1, 1, 2) leftKey := roachpb.Key("a") splitKey := roachpb.Key("m") rightKey := roachpb.Key("z") splitArgs := adminSplitArgs(roachpb.KeyMin, splitKey) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &splitArgs); pErr != nil { t.Fatal(pErr) } incArgs := incrementArgs(leftKey, 1) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil { t.Fatal(pErr) } incArgs = incrementArgs(rightKey, 2) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil { t.Fatal(pErr) } }
// TestSplitSnapshotRace_SplitWins exercises one outcome of the // split/snapshot race: The left side of the split propagates first, // so the split completes before it sees a competing snapshot. This is // the more common outcome in practice. func TestSplitSnapshotRace_SplitWins(t *testing.T) { defer leaktest.AfterTest(t)() mtc, leftKey, rightKey := setupSplitSnapshotRace(t) defer mtc.Stop() // Bring the left range up first so that the split happens before it sees a snapshot. for i := 1; i <= 3; i++ { mtc.restartStore(i) } // Perform a write on the left range and wait for it to propagate. incArgs := incrementArgs(leftKey, 10) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil { t.Fatal(pErr) } mtc.waitForValues(leftKey, []int64{0, 11, 11, 11, 0, 0}) // Now wake the other stores up. mtc.restartStore(4) mtc.restartStore(5) // Write to the right range. incArgs = incrementArgs(rightKey, 20) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil { t.Fatal(pErr) } mtc.waitForValues(rightKey, []int64{0, 0, 0, 25, 25, 25}) }
// TestMultiRangeScanWithMaxResults tests that commands which access multiple // ranges with MaxResults parameter are carried out properly. func TestMultiRangeScanWithMaxResults(t *testing.T) { defer leaktest.AfterTest(t)() testCases := []struct { splitKeys []roachpb.Key keys []roachpb.Key }{ {[]roachpb.Key{roachpb.Key("m")}, []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}}, {[]roachpb.Key{roachpb.Key("h"), roachpb.Key("q")}, []roachpb.Key{roachpb.Key("b"), roachpb.Key("f"), roachpb.Key("k"), roachpb.Key("r"), roachpb.Key("w"), roachpb.Key("y")}}, } for i, tc := range testCases { s, _, _ := serverutils.StartServer(t, base.TestServerArgs{}) defer s.Stopper().Stop() ts := s.(*TestServer) retryOpts := base.DefaultRetryOptions() retryOpts.Closer = ts.stopper.ShouldDrain() ds := kv.NewDistSender(&kv.DistSenderContext{ Clock: s.Clock(), RPCContext: s.RPCContext(), RPCRetryOptions: &retryOpts, }, ts.Gossip()) tds := kv.NewTxnCoordSender(ds, ts.Clock(), ts.Ctx.Linearizable, tracing.NewTracer(), ts.stopper, kv.NewTxnMetrics(metric.NewRegistry())) for _, sk := range tc.splitKeys { if err := ts.node.ctx.DB.AdminSplit(sk); err != nil { t.Fatal(err) } } for _, k := range tc.keys { put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k)) if _, err := client.SendWrapped(tds, nil, put); err != nil { t.Fatal(err) } } // Try every possible ScanRequest startKey. for start := 0; start < len(tc.keys); start++ { // Try every possible maxResults, from 1 to beyond the size of key array. for maxResults := 1; maxResults <= len(tc.keys)-start+1; maxResults++ { scan := roachpb.NewScan(tc.keys[start], tc.keys[len(tc.keys)-1].Next(), int64(maxResults)) reply, err := client.SendWrapped(tds, nil, scan) if err != nil { t.Fatal(err) } rows := reply.(*roachpb.ScanResponse).Rows if start+maxResults <= len(tc.keys) && len(rows) != maxResults { t.Errorf("%d: start=%s: expected %d rows, but got %d", i, tc.keys[start], maxResults, len(rows)) } else if start+maxResults == len(tc.keys)+1 && len(rows) != maxResults-1 { t.Errorf("%d: expected %d rows, but got %d", i, maxResults-1, len(rows)) } } } } }
// TestStoreRangeMergeTwoEmptyRanges tries to merge two empty ranges // together. func TestStoreRangeMergeTwoEmptyRanges(t *testing.T) { defer leaktest.AfterTest(t)() defer config.TestingDisableTableSplits()() store, stopper, _ := createTestStore(t) defer stopper.Stop() if _, _, err := createSplitRanges(store); err != nil { t.Fatal(err) } // Merge the b range back into the a range. args := adminMergeArgs(roachpb.KeyMin) _, err := client.SendWrapped(rg1(store), nil, &args) if err != nil { t.Fatal(err) } // Verify the merge by looking up keys from both ranges. rangeA := store.LookupReplica([]byte("a"), nil) rangeB := store.LookupReplica([]byte("c"), nil) if !reflect.DeepEqual(rangeA, rangeB) { t.Fatalf("ranges were not merged %+v=%+v", rangeA.Desc(), rangeB.Desc()) } }
// TestStoreRangeMergeTwoEmptyRanges tries to merge two empty ranges together. func TestStoreRangeMergeTwoEmptyRanges(t *testing.T) { defer leaktest.AfterTest(t)() sCtx := storage.TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, stopper, _ := createTestStoreWithContext(t, sCtx) defer stopper.Stop() if _, _, err := createSplitRanges(store); err != nil { t.Fatal(err) } // Merge the b range back into the a range. args := adminMergeArgs(roachpb.KeyMin) _, err := client.SendWrapped(rg1(store), nil, &args) if err != nil { t.Fatal(err) } // Verify the merge by looking up keys from both ranges. replicaA := store.LookupReplica([]byte("a"), nil) replicaB := store.LookupReplica([]byte("c"), nil) if !reflect.DeepEqual(replicaA, replicaB) { t.Fatalf("ranges were not merged %s!=%s", replicaA, replicaB) } }
// TestRejectFutureCommand verifies that lease holders reject commands that // would cause a large time jump. func TestRejectFutureCommand(t *testing.T) { defer leaktest.AfterTest(t)() const maxOffset = 100 * time.Millisecond manual := hlc.NewManualClock(0) clock := hlc.NewClock(manual.UnixNano) clock.SetMaxOffset(maxOffset) mtc := multiTestContext{ clock: clock, } mtc.Start(t, 1) defer mtc.Stop() // First do a write. The first write will advance the clock by MaxOffset // because of the read cache's low water mark. getArgs := putArgs([]byte("b"), []byte("b")) if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &getArgs); err != nil { t.Fatal(err) } if now := clock.Now(); now.WallTime != int64(maxOffset) { t.Fatalf("expected clock to advance to 100ms; got %s", now) } // The logical clock has advanced past the physical clock; increment // the "physical" clock to catch up. manual.Increment(int64(maxOffset)) startTime := manual.UnixNano() // Commands with a future timestamp that is within the MaxOffset // bound will be accepted and will cause the clock to advance. for i := int64(0); i < 3; i++ { incArgs := incrementArgs([]byte("a"), 5) ts := hlc.ZeroTimestamp.Add(startTime+((i+1)*30)*int64(time.Millisecond), 0) if _, err := client.SendWrappedWith(rg1(mtc.stores[0]), nil, roachpb.Header{Timestamp: ts}, &incArgs); err != nil { t.Fatal(err) } } if now := clock.Now(); now.WallTime != int64(190*time.Millisecond) { t.Fatalf("expected clock to advance to 190ms; got %s", now) } // Once the accumulated offset reaches MaxOffset, commands will be rejected. incArgs := incrementArgs([]byte("a"), 11) ts := hlc.ZeroTimestamp.Add(int64((time.Duration(startTime)+maxOffset+1)*time.Millisecond), 0) if _, err := client.SendWrappedWith(rg1(mtc.stores[0]), nil, roachpb.Header{Timestamp: ts}, &incArgs); err == nil { t.Fatalf("expected clock offset error but got nil") } // The clock remained at 190ms and the final command was not executed. if now := clock.Now(); now.WallTime != int64(190*time.Millisecond) { t.Errorf("expected clock to advance to 190ms; got %s", now) } val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], roachpb.Key("a"), clock.Now(), true, nil) if err != nil { t.Fatal(err) } if v := mustGetInt(val); v != 15 { t.Errorf("expected 15, got %v", v) } }
// TestSplitSnapshotRace_SnapshotWins exercises one outcome of the // split/snapshot race: The right side of the split replicates first, // so target node sees a raft snapshot before it has processed the split, // so it still has a conflicting range. func TestSplitSnapshotRace_SnapshotWins(t *testing.T) { defer leaktest.AfterTest(t)() mtc, leftKey, rightKey := setupSplitSnapshotRace(t) defer mtc.Stop() // Bring the right range up first. for i := 3; i <= 5; i++ { mtc.restartStore(i) } // Perform a write on the right range. incArgs := incrementArgs(rightKey, 20) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil { t.Fatal(pErr) } // It immediately propagates between nodes 4 and 5, but node 3 // remains at its old value. It can't accept the right-hand range // because it conflicts with its not-yet-split copy of the left-hand // range. This test is not completely deterministic: we want to make // sure that node 3 doesn't panic when it receives the snapshot, but // since it silently drops the message there is nothing we can wait // for. There is a high probability that the message will have been // received by the time that nodes 4 and 5 have processed their // update. mtc.waitForValues(rightKey, []int64{0, 0, 0, 2, 25, 25}) // Wake up the left-hand range. This will allow the left-hand // range's split to complete and unblock the right-hand range. mtc.restartStore(1) mtc.restartStore(2) // Perform writes on both sides. This is not strictly necessary but // it helps wake up dormant ranges that would otherwise have to wait // for retry timeouts. incArgs = incrementArgs(leftKey, 10) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil { t.Fatal(pErr) } mtc.waitForValues(leftKey, []int64{0, 11, 11, 11, 0, 0}) incArgs = incrementArgs(rightKey, 200) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil { t.Fatal(pErr) } mtc.waitForValues(rightKey, []int64{0, 0, 0, 225, 225, 225}) }
// TestStoreRangeMergeNonCollocated attempts to merge two ranges // that are not on the same stores. func TestStoreRangeMergeNonCollocated(t *testing.T) { defer leaktest.AfterTest(t)() mtc := startMultiTestContext(t, 4) defer mtc.Stop() store := mtc.stores[0] // Split into 3 ranges argsSplit := adminSplitArgs(roachpb.KeyMin, []byte("d")) if _, pErr := client.SendWrapped(rg1(store), nil, &argsSplit); pErr != nil { t.Fatalf("Can't split range %s", pErr) } argsSplit = adminSplitArgs(roachpb.KeyMin, []byte("b")) if _, pErr := client.SendWrapped(rg1(store), nil, &argsSplit); pErr != nil { t.Fatalf("Can't split range %s", pErr) } rangeA := store.LookupReplica([]byte("a"), nil) rangeADesc := rangeA.Desc() rangeB := store.LookupReplica([]byte("c"), nil) rangeBDesc := rangeB.Desc() rangeC := store.LookupReplica([]byte("e"), nil) rangeCDesc := rangeC.Desc() if bytes.Equal(rangeADesc.StartKey, rangeBDesc.StartKey) { log.Errorf(context.TODO(), "split ranges keys are equal %q!=%q", rangeADesc.StartKey, rangeBDesc.StartKey) } if bytes.Equal(rangeBDesc.StartKey, rangeCDesc.StartKey) { log.Errorf(context.TODO(), "split ranges keys are equal %q!=%q", rangeBDesc.StartKey, rangeCDesc.StartKey) } if bytes.Equal(rangeADesc.StartKey, rangeCDesc.StartKey) { log.Errorf(context.TODO(), "split ranges keys are equal %q!=%q", rangeADesc.StartKey, rangeCDesc.StartKey) } // Replicate the ranges to different sets of stores. Ranges A and C // are collocated, but B is different. mtc.replicateRange(rangeA.RangeID, 1, 2) mtc.replicateRange(rangeB.RangeID, 1, 3) mtc.replicateRange(rangeC.RangeID, 1, 2) // Attempt to merge. rangeADesc = rangeA.Desc() argsMerge := adminMergeArgs(roachpb.Key(rangeADesc.StartKey)) if _, pErr := rangeA.AdminMerge(context.Background(), argsMerge, rangeADesc); !testutils.IsPError(pErr, "ranges not collocated") { t.Fatalf("did not got expected error; got %s", pErr) } }
// TestStoreRangeMergeStats starts by splitting a range, then writing random data // to both sides of the split. It then merges the ranges and verifies the merged // range has stats consistent with recomputations. func TestStoreRangeMergeStats(t *testing.T) { defer leaktest.AfterTest(t)() sCtx := storage.TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, stopper, manual := createTestStoreWithContext(t, sCtx) defer stopper.Stop() // Split the range. aDesc, bDesc, err := createSplitRanges(store) if err != nil { t.Fatal(err) } // Write some values left and right of the proposed split key. writeRandomDataToRange(t, store, aDesc.RangeID, []byte("aaa")) writeRandomDataToRange(t, store, bDesc.RangeID, []byte("ccc")) // Get the range stats for both ranges now that we have data. var msA, msB enginepb.MVCCStats snap := store.Engine().NewSnapshot() defer snap.Close() if err := engine.MVCCGetRangeStats(context.Background(), snap, aDesc.RangeID, &msA); err != nil { t.Fatal(err) } if err := engine.MVCCGetRangeStats(context.Background(), snap, bDesc.RangeID, &msB); err != nil { t.Fatal(err) } // Stats should agree with recomputation. if err := verifyRecomputedStats(snap, aDesc, msA, manual.UnixNano()); err != nil { t.Fatalf("failed to verify range A's stats before split: %v", err) } if err := verifyRecomputedStats(snap, bDesc, msB, manual.UnixNano()); err != nil { t.Fatalf("failed to verify range B's stats before split: %v", err) } manual.Increment(100) // Merge the b range back into the a range. args := adminMergeArgs(roachpb.KeyMin) if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil { t.Fatal(err) } rngMerged := store.LookupReplica(aDesc.StartKey, nil) // Get the range stats for the merged range and verify. snap = store.Engine().NewSnapshot() defer snap.Close() var msMerged enginepb.MVCCStats if err := engine.MVCCGetRangeStats(context.Background(), snap, rngMerged.RangeID, &msMerged); err != nil { t.Fatal(err) } // Merged stats should agree with recomputation. if err := verifyRecomputedStats(snap, rngMerged.Desc(), msMerged, manual.UnixNano()); err != nil { t.Errorf("failed to verify range's stats after merge: %v", err) } }
// TestStoreRangeSplitAtRangeBounds verifies a range cannot be split // at its start or end keys (would create zero-length range!). This // sort of thing might happen in the wild if two split requests // arrived for same key. The first one succeeds and second would try // to split at the start of the newly split range. func TestStoreRangeSplitAtRangeBounds(t *testing.T) { defer leaktest.AfterTest(t)() defer config.TestingDisableTableSplits()() store, stopper, _ := createTestStore(t) defer stopper.Stop() args := adminSplitArgs(roachpb.KeyMin, []byte("a")) if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil { t.Fatal(err) } // This second split will try to split at end of first split range. if _, err := client.SendWrapped(rg1(store), nil, &args); err == nil { t.Fatalf("split succeeded unexpectedly") } // Now try to split at start of new range. args = adminSplitArgs(roachpb.KeyMin, []byte("a")) if _, err := client.SendWrapped(rg1(store), nil, &args); err == nil { t.Fatalf("split succeeded unexpectedly") } }
// TestStoreRangeMergeLastRange verifies that merging the last range // fails. func TestStoreRangeMergeLastRange(t *testing.T) { defer leaktest.AfterTest(t)() defer config.TestingDisableTableSplits()() store, stopper, _ := createTestStore(t) defer stopper.Stop() // Merge last range. args := adminMergeArgs(roachpb.KeyMin) if _, pErr := client.SendWrapped(rg1(store), nil, &args); !testutils.IsPError(pErr, "cannot merge final range") { t.Fatalf("expected 'cannot merge final range' error; got %s", pErr) } }
func BenchmarkStoreRangeSplit(b *testing.B) { defer tracing.Disable()() sCtx := storage.TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, stopper, _ := createTestStoreWithContext(b, sCtx) defer stopper.Stop() // Perform initial split of ranges. sArgs := adminSplitArgs(roachpb.KeyMin, []byte("b")) if _, err := client.SendWrapped(rg1(store), nil, &sArgs); err != nil { b.Fatal(err) } // Write some values left and right of the split key. aDesc := store.LookupReplica([]byte("a"), nil).Desc() bDesc := store.LookupReplica([]byte("c"), nil).Desc() writeRandomDataToRange(b, store, aDesc.RangeID, []byte("aaa")) writeRandomDataToRange(b, store, bDesc.RangeID, []byte("ccc")) // Merge the b range back into the a range. mArgs := adminMergeArgs(roachpb.KeyMin) if _, err := client.SendWrapped(rg1(store), nil, &mArgs); err != nil { b.Fatal(err) } b.ResetTimer() for i := 0; i < b.N; i++ { // Split the range. b.StartTimer() if _, err := client.SendWrapped(rg1(store), nil, &sArgs); err != nil { b.Fatal(err) } // Merge the ranges. b.StopTimer() if _, err := client.SendWrapped(rg1(store), nil, &mArgs); err != nil { b.Fatal(err) } } }
// TestStoreRangeMergeLastRange verifies that merging the last range // fails. func TestStoreRangeMergeLastRange(t *testing.T) { defer leaktest.AfterTest(t)() sCtx := storage.TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, stopper, _ := createTestStoreWithContext(t, sCtx) defer stopper.Stop() // Merge last range. args := adminMergeArgs(roachpb.KeyMin) if _, pErr := client.SendWrapped(rg1(store), nil, &args); !testutils.IsPError(pErr, "cannot merge final range") { t.Fatalf("expected 'cannot merge final range' error; got %s", pErr) } }
// TestStoreRangeSplitInsideRow verifies an attempt to split a range inside of // a table row will cause a split at a boundary between rows. func TestStoreRangeSplitInsideRow(t *testing.T) { defer leaktest.AfterTest(t)() sCtx := storage.TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, stopper, _ := createTestStoreWithContext(t, sCtx) defer stopper.Stop() // Manually create some the column keys corresponding to the table: // // CREATE TABLE t (id STRING PRIMARY KEY, col1 INT, col2 INT) tableKey := keys.MakeTablePrefix(keys.MaxReservedDescID + 1) rowKey := roachpb.Key(encoding.EncodeVarintAscending(append([]byte(nil), tableKey...), 1)) rowKey = encoding.EncodeStringAscending(encoding.EncodeVarintAscending(rowKey, 1), "a") col1Key := keys.MakeFamilyKey(append([]byte(nil), rowKey...), 1) col2Key := keys.MakeFamilyKey(append([]byte(nil), rowKey...), 2) // We don't care about the value, so just store any old thing. if err := store.DB().Put(col1Key, "column 1"); err != nil { t.Fatal(err) } if err := store.DB().Put(col2Key, "column 2"); err != nil { t.Fatal(err) } // Split between col1Key and col2Key by splitting before col2Key. args := adminSplitArgs(col2Key, col2Key) _, err := client.SendWrapped(rg1(store), nil, &args) if err != nil { t.Fatalf("%s: split unexpected error: %s", col1Key, err) } rng1 := store.LookupReplica(col1Key, nil) rng2 := store.LookupReplica(col2Key, nil) // Verify the two columns are still on the same range. if !reflect.DeepEqual(rng1, rng2) { t.Fatalf("%s: ranges differ: %+v vs %+v", roachpb.Key(col1Key), rng1, rng2) } // Verify we split on a row key. if startKey := rng1.Desc().StartKey; !startKey.Equal(rowKey) { t.Fatalf("%s: expected split on %s, but found %s", roachpb.Key(col1Key), roachpb.Key(rowKey), startKey) } // Verify the previous range was split on a row key. rng3 := store.LookupReplica(tableKey, nil) if endKey := rng3.Desc().EndKey; !endKey.Equal(rowKey) { t.Fatalf("%s: expected split on %s, but found %s", roachpb.Key(col1Key), roachpb.Key(rowKey), endKey) } }
// TestStoreRangeSplitAtTablePrefix verifies a range can be split at // UserTableDataMin and still gossip the SystemConfig properly. func TestStoreRangeSplitAtTablePrefix(t *testing.T) { defer leaktest.AfterTest(t)() sCtx := storage.TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, stopper, _ := createTestStoreWithContext(t, sCtx) defer stopper.Stop() key := keys.MakeRowSentinelKey(append([]byte(nil), keys.UserTableDataMin...)) args := adminSplitArgs(key, key) if _, pErr := client.SendWrapped(rg1(store), nil, &args); pErr != nil { t.Fatalf("%q: split unexpected error: %s", key, pErr) } var desc sqlbase.TableDescriptor descBytes, err := protoutil.Marshal(&desc) if err != nil { t.Fatal(err) } // Update SystemConfig to trigger gossip. if err := store.DB().Txn(func(txn *client.Txn) error { txn.SetSystemConfigTrigger() // We don't care about the values, just the keys. k := sqlbase.MakeDescMetadataKey(sqlbase.ID(keys.MaxReservedDescID + 1)) return txn.Put(k, &desc) }); err != nil { t.Fatal(err) } successChan := make(chan struct{}, 1) store.Gossip().RegisterCallback(gossip.KeySystemConfig, func(_ string, content roachpb.Value) { contentBytes, err := content.GetBytes() if err != nil { t.Fatal(err) } if bytes.Contains(contentBytes, descBytes) { select { case successChan <- struct{}{}: default: } } }) select { case <-time.After(time.Second): t.Errorf("expected a schema gossip containing %q, but did not see one", descBytes) case <-successChan: } }
func createSplitRanges(store *storage.Store) (*roachpb.RangeDescriptor, *roachpb.RangeDescriptor, *roachpb.Error) { args := adminSplitArgs(roachpb.KeyMin, []byte("b")) if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil { return nil, nil, err } rangeADesc := store.LookupReplica([]byte("a"), nil).Desc() rangeBDesc := store.LookupReplica([]byte("c"), nil).Desc() if bytes.Equal(rangeADesc.StartKey, rangeBDesc.StartKey) { log.Errorf(context.TODO(), "split ranges keys are equal %q!=%q", rangeADesc.StartKey, rangeBDesc.StartKey) } return rangeADesc, rangeBDesc, nil }
// LookupRange returns the descriptor of the range containing key. func (tc *TestCluster) LookupRange(key roachpb.Key) (roachpb.RangeDescriptor, error) { rangeLookupReq := roachpb.RangeLookupRequest{ Span: roachpb.Span{ Key: keys.RangeMetaKey(keys.MustAddr(key)), }, MaxRanges: 1, ConsiderIntents: false, } resp, pErr := client.SendWrapped(tc.Servers[0].GetDistSender(), nil, &rangeLookupReq) if pErr != nil { return roachpb.RangeDescriptor{}, errors.Errorf( "%q: lookup range unexpected error: %s", key, pErr) } return resp.(*roachpb.RangeLookupResponse).Ranges[0], nil }
func (c *cluster) lookupRange(nodeIdx int, key roachpb.Key) (*roachpb.RangeDescriptor, error) { req := &roachpb.RangeLookupRequest{ Span: roachpb.Span{ Key: keys.RangeMetaKey(keys.MustAddr(key)), }, MaxRanges: 1, ConsiderIntents: false, } sender := c.clients[nodeIdx].GetSender() resp, pErr := client.SendWrapped(sender, nil, req) if pErr != nil { return nil, errors.Errorf("%s: lookup range: %s", key, pErr) } return &resp.(*roachpb.RangeLookupResponse).Ranges[0], nil }
// TestProactiveRaftLogTruncate verifies that we proactively truncate the raft // log even when replica scanning is disabled. func TestProactiveRaftLogTruncate(t *testing.T) { defer leaktest.AfterTest(t)() store, _, stopper := createTestStore(t) defer stopper.Stop() store.scanner.SetDisabled(true) r, err := store.GetReplica(1) if err != nil { t.Fatal(err) } r.mu.Lock() oldFirstIndex, err := r.FirstIndex() r.mu.Unlock() if err != nil { t.Fatal(err) } // Write a few keys to the range. While writing these keys, the raft log // should be proactively truncated even though replica scanning is disabled. for i := 0; i < 2*RaftLogQueueStaleThreshold; i++ { key := roachpb.Key(fmt.Sprintf("key%02d", i)) args := putArgs(key, []byte(fmt.Sprintf("value%02d", i))) if _, err := client.SendWrapped(store.testSender(), nil, &args); err != nil { t.Fatal(err) } } // Wait for any asynchronous tasks to finish. stopper.Quiesce() r.mu.Lock() newFirstIndex, err := r.FirstIndex() r.mu.Unlock() if err != nil { t.Fatal(err) } if newFirstIndex <= oldFirstIndex { t.Errorf("log was not correctly truncated, old first index:%d, current first index:%d", oldFirstIndex, newFirstIndex) } }
// SplitRange splits the range containing splitKey. // The right range created by the split starts at the split key and extends to the // original range's end key. // Returns the new descriptors of the left and right ranges. // // splitKey must correspond to a SQL table key (it must end with a family ID / // col ID). func (tc *TestCluster) SplitRange( splitKey roachpb.Key, ) (*roachpb.RangeDescriptor, *roachpb.RangeDescriptor, error) { splitRKey, err := keys.Addr(splitKey) if err != nil { return nil, nil, err } origRangeDesc, err := tc.LookupRange(splitKey) if err != nil { return nil, nil, err } if origRangeDesc.StartKey.Equal(splitRKey) { return nil, nil, errors.Errorf( "cannot split range %+v at start key %q", origRangeDesc, splitKey) } splitReq := roachpb.AdminSplitRequest{ Span: roachpb.Span{ Key: splitKey, }, SplitKey: splitKey, } _, pErr := client.SendWrapped(tc.Servers[0].GetDistSender(), nil, &splitReq) if pErr != nil { return nil, nil, errors.Errorf( "%q: split unexpected error: %s", splitReq.SplitKey, pErr) } leftRangeDesc := new(roachpb.RangeDescriptor) rightRangeDesc := new(roachpb.RangeDescriptor) if err := tc.Servers[0].DB().GetProto( keys.RangeDescriptorKey(origRangeDesc.StartKey), leftRangeDesc); err != nil { return nil, nil, errors.Wrap(err, "could not look up left-hand side descriptor") } // The split point might not be exactly the one we requested (it can be // adjusted slightly so we don't split in the middle of SQL rows). Update it // to the real point. splitRKey = leftRangeDesc.EndKey if err := tc.Servers[0].DB().GetProto( keys.RangeDescriptorKey(splitRKey), rightRangeDesc); err != nil { return nil, nil, errors.Wrap(err, "could not look up right-hand side descriptor") } return leftRangeDesc, rightRangeDesc, nil }
// TestStoreRangeSplitAtIllegalKeys verifies a range cannot be split // at illegal keys. func TestStoreRangeSplitAtIllegalKeys(t *testing.T) { defer leaktest.AfterTest(t)() store, stopper, _ := createTestStore(t) defer stopper.Stop() for _, key := range []roachpb.Key{ keys.Meta1Prefix, testutils.MakeKey(keys.Meta1Prefix, []byte("a")), testutils.MakeKey(keys.Meta1Prefix, roachpb.RKeyMax), keys.Meta2KeyMax, keys.MakeTablePrefix(10 /* system descriptor ID */), } { args := adminSplitArgs(roachpb.KeyMin, key) _, pErr := client.SendWrapped(rg1(store), nil, &args) if !testutils.IsPError(pErr, "cannot split") { t.Errorf("%q: unexpected split error %s", key, pErr) } } }
// TestStoreRangeSplitConcurrent verifies that concurrent range splits // of the same range are executed serially, and all but the first fail // because the split key is invalid after the first split succeeds. func TestStoreRangeSplitConcurrent(t *testing.T) { defer leaktest.AfterTest(t)() sCtx := storage.TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, stopper, _ := createTestStoreWithContext(t, sCtx) defer stopper.Stop() splitKey := roachpb.Key("a") concurrentCount := int32(10) wg := sync.WaitGroup{} wg.Add(int(concurrentCount)) failureCount := int32(0) for i := int32(0); i < concurrentCount; i++ { go func() { args := adminSplitArgs(roachpb.KeyMin, splitKey) _, pErr := client.SendWrapped(rg1(store), nil, &args) if pErr != nil { atomic.AddInt32(&failureCount, 1) } wg.Done() }() } wg.Wait() if failureCount != concurrentCount-1 { t.Fatalf("concurrent splits succeeded unexpectedly; failureCount=%d", failureCount) } // Verify everything ended up as expected. if a, e := store.ReplicaCount(), 2; a != e { t.Fatalf("expected %d stores after concurrent splits; actual count=%d", e, a) } rng := store.LookupReplica(roachpb.RKeyMin, nil) newRng := store.LookupReplica(roachpb.RKey(splitKey), nil) if !bytes.Equal(newRng.Desc().StartKey, splitKey) || !bytes.Equal(splitKey, rng.Desc().EndKey) { t.Errorf("ranges mismatched, wanted %q=%q=%q", newRng.Desc().StartKey, splitKey, rng.Desc().EndKey) } if !bytes.Equal(newRng.Desc().EndKey, roachpb.RKeyMax) || !bytes.Equal(rng.Desc().StartKey, roachpb.RKeyMin) { t.Errorf("new ranges do not cover KeyMin-KeyMax, but only %q-%q", rng.Desc().StartKey, newRng.Desc().EndKey) } }
// Test that a lease extension (a RequestLeaseRequest that doesn't change the // lease holder) is not blocked by ongoing reads. // Note that lease transfers are blocked by reads through their // PostCommitTrigger.noConcurrentReads. func TestLeaseExtensionNotBlockedByRead(t *testing.T) { defer leaktest.AfterTest(t)() readBlocked := make(chan struct{}) cmdFilter := func(fArgs storagebase.FilterArgs) *roachpb.Error { if fArgs.Hdr.UserPriority == 42 { // Signal that the read is blocked. readBlocked <- struct{}{} // Wait for read to be unblocked. <-readBlocked } return nil } srv, _, _ := serverutils.StartServer(t, base.TestServerArgs{ Knobs: base.TestingKnobs{ Store: &storage.StoreTestingKnobs{ TestingCommandFilter: cmdFilter, }, }, }) s := srv.(*server.TestServer) defer s.Stopper().Stop() // Start a read and wait for it to block. key := roachpb.Key("a") errChan := make(chan error) go func() { getReq := roachpb.GetRequest{ Span: roachpb.Span{ Key: key, }, } if _, pErr := client.SendWrappedWith(s.GetDistSender(), nil, roachpb.Header{UserPriority: 42}, &getReq); pErr != nil { errChan <- pErr.GoError() } }() select { case err := <-errChan: t.Fatal(err) case <-readBlocked: // Send the lease request. // We change the key slightly, otherwise the lease request will be blocked // by the read through the command queue. // TODO(andrei): don't change the key anymore once lease requests don't go // through the command queue any more. leaseHdrKey := roachpb.Key(append(key, 0x00)) rKey, err := keys.Addr(leaseHdrKey) if err != nil { t.Fatal(err) } _, repDesc, err := s.Stores().LookupReplica(rKey, nil) if err != nil { t.Fatal(err) } leaseReq := roachpb.RequestLeaseRequest{ Span: roachpb.Span{ Key: leaseHdrKey, }, Lease: roachpb.Lease{ Start: s.Clock().Now(), StartStasis: s.Clock().Now().Add(time.Second.Nanoseconds(), 0), Expiration: s.Clock().Now().Add(2*time.Second.Nanoseconds(), 0), Replica: repDesc, }, } if _, pErr := client.SendWrapped(s.GetDistSender(), nil, &leaseReq); pErr != nil { t.Fatal(pErr) } // Unblock the read. readBlocked <- struct{}{} } }
func TestRangeTransferLease(t *testing.T) { defer leaktest.AfterTest(t)() ctx := storage.TestStoreContext() var filterMu syncutil.Mutex var filter func(filterArgs storagebase.FilterArgs) *roachpb.Error ctx.TestingKnobs.TestingCommandFilter = func(filterArgs storagebase.FilterArgs) *roachpb.Error { filterMu.Lock() filterCopy := filter filterMu.Unlock() if filterCopy != nil { return filterCopy(filterArgs) } return nil } var waitForTransferBlocked atomic.Value waitForTransferBlocked.Store(false) transferBlocked := make(chan struct{}) ctx.TestingKnobs.LeaseTransferBlockedOnExtensionEvent = func( _ roachpb.ReplicaDescriptor) { if waitForTransferBlocked.Load().(bool) { transferBlocked <- struct{}{} waitForTransferBlocked.Store(false) } } mtc := &multiTestContext{} mtc.storeContext = &ctx mtc.Start(t, 2) defer mtc.Stop() // First, do a write; we'll use it to determine when the dust has settled. leftKey := roachpb.Key("a") incArgs := incrementArgs(leftKey, 1) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil { t.Fatal(pErr) } // Get the left range's ID. rangeID := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil).RangeID // Replicate the left range onto node 1. mtc.replicateRange(rangeID, 1) replica0 := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil) replica1 := mtc.stores[1].LookupReplica(roachpb.RKey("a"), nil) gArgs := getArgs(leftKey) replica0Desc, err := replica0.GetReplicaDescriptor() if err != nil { t.Fatal(err) } // Check that replica0 can serve reads OK. if _, pErr := client.SendWrappedWith( mtc.senders[0], nil, roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil { t.Fatal(pErr) } { // Transferring the lease to ourself should be a no-op. origLeasePtr, _ := replica0.GetLease() origLease := *origLeasePtr if err := replica0.AdminTransferLease(replica0Desc.StoreID); err != nil { t.Fatal(err) } newLeasePtr, _ := replica0.GetLease() if origLeasePtr != newLeasePtr || origLease != *newLeasePtr { t.Fatalf("expected %+v, but found %+v", origLeasePtr, newLeasePtr) } } { // An invalid target should result in an error. const expected = "unable to find store .* in range" if err := replica0.AdminTransferLease(1000); !testutils.IsError(err, expected) { t.Fatalf("expected %s, but found %v", expected, err) } } // Move the lease to store 1. var newHolderDesc roachpb.ReplicaDescriptor util.SucceedsSoon(t, func() error { var err error newHolderDesc, err = replica1.GetReplicaDescriptor() return err }) if err := replica0.AdminTransferLease(newHolderDesc.StoreID); err != nil { t.Fatal(err) } // Check that replica0 doesn't serve reads any more. replica0Desc, err = replica0.GetReplicaDescriptor() if err != nil { t.Fatal(err) } _, pErr := client.SendWrappedWith( mtc.senders[0], nil, roachpb.Header{Replica: replica0Desc}, &gArgs) nlhe, ok := pErr.GetDetail().(*roachpb.NotLeaseHolderError) if !ok { t.Fatalf("expected %T, got %s", &roachpb.NotLeaseHolderError{}, pErr) } if *(nlhe.LeaseHolder) != newHolderDesc { t.Fatalf("expected lease holder %+v, got %+v", newHolderDesc, nlhe.LeaseHolder) } // Check that replica1 now has the lease (or gets it soon). util.SucceedsSoon(t, func() error { if _, pErr := client.SendWrappedWith( mtc.senders[1], nil, roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil { return pErr.GoError() } return nil }) replica1Lease, _ := replica1.GetLease() // Verify the timestamp cache low water. Because we executed a transfer lease // request, the low water should be set to the new lease start time which is // less than the previous lease's expiration time. if lowWater := replica1.GetTimestampCacheLowWater(); lowWater != replica1Lease.Start { t.Fatalf("expected timestamp cache low water %s, but found %s", replica1Lease.Start, lowWater) } // Make replica1 extend its lease and transfer the lease immediately after // that. Test that the transfer still happens (it'll wait until the extension // is done). extensionSem := make(chan struct{}) filterMu.Lock() filter = func(filterArgs storagebase.FilterArgs) *roachpb.Error { if filterArgs.Sid != mtc.stores[1].Ident.StoreID { return nil } llReq, ok := filterArgs.Req.(*roachpb.RequestLeaseRequest) if !ok { return nil } if llReq.Lease.Replica == newHolderDesc { // Notify the main thread that the extension is in progress and wait for // the signal to proceed. filterMu.Lock() filter = nil filterMu.Unlock() extensionSem <- struct{}{} <-extensionSem } return nil } filterMu.Unlock() // Initiate an extension. var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() shouldRenewTS := replica1Lease.StartStasis.Add(-1, 0) mtc.manualClock.Set(shouldRenewTS.WallTime + 1) if _, pErr := client.SendWrappedWith( mtc.senders[1], nil, roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil { panic(pErr) } }() <-extensionSem waitForTransferBlocked.Store(true) // Initiate a transfer. wg.Add(1) go func() { defer wg.Done() // Transfer back from replica1 to replica0. if err := replica1.AdminTransferLease(replica0Desc.StoreID); err != nil { panic(err) } }() // Wait for the transfer to be blocked by the extension. <-transferBlocked // Now unblock the extension. extensionSem <- struct{}{} // Check that the transfer to replica1 eventually happens. util.SucceedsSoon(t, func() error { if _, pErr := client.SendWrappedWith( mtc.senders[0], nil, roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil { return pErr.GoError() } return nil }) filterMu.Lock() filter = nil filterMu.Unlock() wg.Wait() }
// TestRangeLookupUseReverse tests whether the results and the results count // are correct when scanning in reverse order. func TestRangeLookupUseReverse(t *testing.T) { defer leaktest.AfterTest(t)() sCtx := storage.TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, stopper, _ := createTestStoreWithContext(t, sCtx) defer stopper.Stop() // Init test ranges: // ["","a"), ["a","c"), ["c","e"), ["e","g") and ["g","\xff\xff"). splits := []roachpb.AdminSplitRequest{ adminSplitArgs(roachpb.Key("g"), roachpb.Key("g")), adminSplitArgs(roachpb.Key("e"), roachpb.Key("e")), adminSplitArgs(roachpb.Key("c"), roachpb.Key("c")), adminSplitArgs(roachpb.Key("a"), roachpb.Key("a")), } for _, split := range splits { _, pErr := client.SendWrapped(rg1(store), nil, &split) if pErr != nil { t.Fatalf("%q: split unexpected error: %s", split.SplitKey, pErr) } } // Resolve the intents. scanArgs := roachpb.ScanRequest{ Span: roachpb.Span{ Key: keys.RangeMetaKey(roachpb.RKeyMin.Next()), EndKey: keys.RangeMetaKey(roachpb.RKeyMax), }, } util.SucceedsSoon(t, func() error { _, pErr := client.SendWrapped(rg1(store), nil, &scanArgs) return pErr.GoError() }) revScanArgs := func(key []byte, maxResults int32) *roachpb.RangeLookupRequest { return &roachpb.RangeLookupRequest{ Span: roachpb.Span{ Key: key, }, MaxRanges: maxResults, Reverse: true, } } // Test cases. testCases := []struct { request *roachpb.RangeLookupRequest expected []roachpb.RangeDescriptor expectedPre []roachpb.RangeDescriptor }{ // Test key in the middle of the range. { request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("f")), 2), // ["e","g") and ["c","e"). expected: []roachpb.RangeDescriptor{ {StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")}, }, expectedPre: []roachpb.RangeDescriptor{ {StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")}, }, }, // Test key in the end key of the range. { request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("g")), 3), // ["e","g"), ["c","e") and ["a","c"). expected: []roachpb.RangeDescriptor{ {StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")}, }, expectedPre: []roachpb.RangeDescriptor{ {StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")}, {StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("c")}, }, }, { request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("e")), 2), // ["c","e") and ["a","c"). expected: []roachpb.RangeDescriptor{ {StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")}, }, expectedPre: []roachpb.RangeDescriptor{ {StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("c")}, }, }, // Test Meta2KeyMax. { request: revScanArgs(keys.Meta2KeyMax, 2), // ["e","g") and ["g","\xff\xff") expected: []roachpb.RangeDescriptor{ {StartKey: roachpb.RKey("g"), EndKey: roachpb.RKey("\xff\xff")}, }, expectedPre: []roachpb.RangeDescriptor{ {StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")}, }, }, // Test Meta1KeyMax. { request: revScanArgs(keys.Meta1KeyMax, 1), // ["","a") expected: []roachpb.RangeDescriptor{ {StartKey: roachpb.RKeyMin, EndKey: roachpb.RKey("a")}, }, }, } for testIdx, test := range testCases { resp, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{ ReadConsistency: roachpb.INCONSISTENT, }, test.request) if pErr != nil { t.Fatalf("%d: RangeLookup error: %s", testIdx, pErr) } rlReply := resp.(*roachpb.RangeLookupResponse) // Checks the results count. if int32(len(rlReply.Ranges))+int32(len(rlReply.PrefetchedRanges)) != test.request.MaxRanges { t.Fatalf("%d: returned results count, expected %d,but got %d", testIdx, test.request.MaxRanges, len(rlReply.Ranges)) } // Checks the range descriptors. for _, rngSlice := range []struct { expect, reply []roachpb.RangeDescriptor }{ {test.expected, rlReply.Ranges}, {test.expectedPre, rlReply.PrefetchedRanges}, } { for i, rng := range rngSlice.expect { if !(rng.StartKey.Equal(rngSlice.reply[i].StartKey) && rng.EndKey.Equal(rngSlice.reply[i].EndKey)) { t.Fatalf("%d: returned range is not correct, expected %v ,but got %v", testIdx, rng, rngSlice.reply[i]) } } } } }
// TestStoreRangeMergeMetadataCleanup tests that all metadata of a // subsumed range is cleaned up on merge. func TestStoreRangeMergeMetadataCleanup(t *testing.T) { defer leaktest.AfterTest(t)() sCtx := storage.TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, stopper, _ := createTestStoreWithContext(t, sCtx) defer stopper.Stop() scan := func(f func(roachpb.KeyValue) (bool, error)) { if _, err := engine.MVCCIterate(context.Background(), store.Engine(), roachpb.KeyMin, roachpb.KeyMax, hlc.ZeroTimestamp, true, nil, false, f); err != nil { t.Fatal(err) } } content := roachpb.Key("testing!") // Write some values left of the proposed split key. pArgs := putArgs([]byte("aaa"), content) if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil { t.Fatal(err) } // Collect all the keys. preKeys := make(map[string]struct{}) scan(func(kv roachpb.KeyValue) (bool, error) { preKeys[string(kv.Key)] = struct{}{} return false, nil }) // Split the range. _, bDesc, err := createSplitRanges(store) if err != nil { t.Fatal(err) } // Write some values right of the split key. pArgs = putArgs([]byte("ccc"), content) if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{ RangeID: bDesc.RangeID, }, &pArgs); err != nil { t.Fatal(err) } // Merge the b range back into the a range. args := adminMergeArgs(roachpb.KeyMin) if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil { t.Fatal(err) } // Collect all the keys again. postKeys := make(map[string]struct{}) scan(func(kv roachpb.KeyValue) (bool, error) { postKeys[string(kv.Key)] = struct{}{} return false, nil }) // Compute the new keys. for k := range preKeys { delete(postKeys, k) } // Keep only the subsumed range's local keys. localRangeKeyPrefix := string(keys.MakeRangeIDPrefix(bDesc.RangeID)) for k := range postKeys { if !strings.HasPrefix(k, localRangeKeyPrefix) { delete(postKeys, k) } } if numKeys := len(postKeys); numKeys > 0 { var buf bytes.Buffer fmt.Fprintf(&buf, "%d keys were not cleaned up:\n", numKeys) for k := range postKeys { fmt.Fprintf(&buf, "%q\n", k) } t.Fatal(buf.String()) } }
// TestRaftLogQueue verifies that the raft log queue correctly truncates the // raft log. func TestRaftLogQueue(t *testing.T) { defer leaktest.AfterTest(t)() var mtc multiTestContext // Set maxBytes to something small so we can trigger the raft log truncation // without adding 64MB of logs. const maxBytes = 1 << 16 defer config.TestingSetDefaultZoneConfig(config.ZoneConfig{ RangeMaxBytes: maxBytes, })() // Turn off raft elections so the raft leader won't change out from under // us in this test. sc := storage.TestStoreContext() sc.RaftTickInterval = time.Hour * 24 sc.RaftElectionTimeoutTicks = 1000000 mtc.storeContext = &sc mtc.Start(t, 3) defer mtc.Stop() // Write a single value to ensure we have a leader. pArgs := putArgs([]byte("key"), []byte("value")) if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &pArgs); err != nil { t.Fatal(err) } // Get the raft leader (and ensure one exists). rangeID := mtc.stores[0].LookupReplica([]byte("a"), nil).RangeID raftLeaderRepl := mtc.getRaftLeader(rangeID) if raftLeaderRepl == nil { t.Fatalf("could not find raft leader replica for range %d", rangeID) } originalIndex, err := raftLeaderRepl.GetFirstIndex() if err != nil { t.Fatal(err) } // Disable splits since we're increasing the raft log with puts. for _, store := range mtc.stores { store.DisableSplitQueue(true) } // Write a collection of values to increase the raft log. value := bytes.Repeat([]byte("a"), 1000) // 1KB for size := int64(0); size < 2*maxBytes; size += int64(len(value)) { pArgs = putArgs([]byte(fmt.Sprintf("key-%d", size)), value) if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &pArgs); err != nil { t.Fatal(err) } } // Sadly, occasionally the queue has a race with the force processing so // this succeeds within will captures those rare cases. var afterTruncationIndex uint64 util.SucceedsSoon(t, func() error { // Force a truncation check. for _, store := range mtc.stores { store.ForceRaftLogScanAndProcess() } // Ensure that firstIndex has increased indicating that the log // truncation has occurred. var err error afterTruncationIndex, err = raftLeaderRepl.GetFirstIndex() if err != nil { t.Fatal(err) } if afterTruncationIndex <= originalIndex { return errors.Errorf("raft log has not been truncated yet, afterTruncationIndex:%d originalIndex:%d", afterTruncationIndex, originalIndex) } return nil }) // Force a truncation check again to ensure that attempting to truncate an // already truncated log has no effect. for _, store := range mtc.stores { store.ForceRaftLogScanAndProcess() } after2ndTruncationIndex, err := raftLeaderRepl.GetFirstIndex() if err != nil { t.Fatal(err) } if afterTruncationIndex > after2ndTruncationIndex { t.Fatalf("second truncation destroyed state: afterTruncationIndex:%d after2ndTruncationIndex:%d", afterTruncationIndex, after2ndTruncationIndex) } }
// TestGetTruncatableIndexes verifies that old raft log entries are correctly // removed. func TestGetTruncatableIndexes(t *testing.T) { defer leaktest.AfterTest(t)() store, _, stopper := createTestStore(t) defer stopper.Stop() if _, err := store.GetReplica(0); err == nil { t.Fatal("expected GetRange to fail on missing range") } store.SetRaftLogQueueActive(false) // Test on a new range which should not have a raft group yet. rngNew := createRange(store, 100, roachpb.RKey("a"), roachpb.RKey("c")) truncatableIndexes, oldestIndex, err := getTruncatableIndexes(rngNew) if err != nil { t.Errorf("expected no error, got %s", err) } if truncatableIndexes != 0 { t.Errorf("expected 0 for truncatable index, got %d", truncatableIndexes) } if oldestIndex != 0 { t.Errorf("expected 0 for oldest index, got %d", oldestIndex) } r, err := store.GetReplica(1) if err != nil { t.Fatal(err) } r.mu.Lock() firstIndex, err := r.FirstIndex() r.mu.Unlock() if err != nil { t.Fatal(err) } // Write a few keys to the range. for i := 0; i < RaftLogQueueStaleThreshold+1; i++ { key := roachpb.Key(fmt.Sprintf("key%02d", i)) args := putArgs(key, []byte(fmt.Sprintf("value%02d", i))) if _, err := client.SendWrapped(store.testSender(), nil, &args); err != nil { t.Fatal(err) } } truncatableIndexes, oldestIndex, err = getTruncatableIndexes(r) if err != nil { t.Errorf("expected no error, got %s", err) } if truncatableIndexes == 0 { t.Errorf("expected a value for truncatable index, got 0") } if oldestIndex < firstIndex { t.Errorf("expected oldest index (%d) to be greater than or equal to first index (%d)", oldestIndex, firstIndex) } // Enable the raft log scanner and and force a truncation. store.SetRaftLogQueueActive(true) store.ForceRaftLogScanAndProcess() // Wait for tasks to finish, in case the processLoop grabbed the event // before ForceRaftLogScanAndProcess but is still working on it. stopper.Quiesce() r.mu.Lock() newFirstIndex, err := r.FirstIndex() r.mu.Unlock() if err != nil { t.Fatal(err) } if newFirstIndex <= firstIndex { t.Errorf("log was not correctly truncated, older first index:%d, current first index:%d", firstIndex, newFirstIndex) } // Once truncated, we should have no truncatable indexes. If this turns out // to be flaky, we can remove it as the same functionality is tested in // client_raft_log_queue_test. util.SucceedsSoon(t, func() error { store.ForceRaftLogScanAndProcess() truncatableIndexes, oldestIndex, err := getTruncatableIndexes(rngNew) if err != nil { return errors.Errorf("expected no error, got %s", err) } if truncatableIndexes != 0 { return errors.Errorf("expected 0 for truncatable index, got %d", truncatableIndexes) } if oldestIndex != 0 { return errors.Errorf("expected 0 for oldest index, got %d", oldestIndex) } return nil }) }
// TestMultiRangeScanDeleteRange tests that commands which access multiple // ranges are carried out properly. func TestMultiRangeScanDeleteRange(t *testing.T) { defer leaktest.AfterTest(t)() s, _, _ := serverutils.StartServer(t, base.TestServerArgs{}) defer s.Stopper().Stop() ts := s.(*TestServer) retryOpts := base.DefaultRetryOptions() retryOpts.Closer = ts.stopper.ShouldQuiesce() ds := kv.NewDistSender(&kv.DistSenderConfig{ Clock: s.Clock(), RPCContext: s.RPCContext(), RPCRetryOptions: &retryOpts, }, ts.Gossip()) ctx := tracing.WithTracer(context.Background(), tracing.NewTracer()) tds := kv.NewTxnCoordSender(ctx, ds, s.Clock(), ts.Ctx.Linearizable, ts.stopper, kv.MakeTxnMetrics()) if err := ts.node.ctx.DB.AdminSplit("m"); err != nil { t.Fatal(err) } writes := []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")} get := &roachpb.GetRequest{ Span: roachpb.Span{Key: writes[0]}, } get.EndKey = writes[len(writes)-1] if _, err := client.SendWrapped(tds, nil, get); err == nil { t.Errorf("able to call Get with a key range: %v", get) } var delTS hlc.Timestamp for i, k := range writes { put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k)) reply, err := client.SendWrapped(tds, nil, put) if err != nil { t.Fatal(err) } scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next()) reply, err = client.SendWrapped(tds, nil, scan) if err != nil { t.Fatal(err) } sr := reply.(*roachpb.ScanResponse) if sr.Txn != nil { // This was the other way around at some point in the past. // Same below for Delete, etc. t.Errorf("expected no transaction in response header") } if rows := sr.Rows; len(rows) != i+1 { t.Fatalf("expected %d rows, but got %d", i+1, len(rows)) } } del := &roachpb.DeleteRangeRequest{ Span: roachpb.Span{ Key: writes[0], EndKey: roachpb.Key(writes[len(writes)-1]).Next(), }, ReturnKeys: true, } reply, err := client.SendWrappedWith(tds, nil, roachpb.Header{Timestamp: delTS}, del) if err != nil { t.Fatal(err) } dr := reply.(*roachpb.DeleteRangeResponse) if dr.Txn != nil { t.Errorf("expected no transaction in response header") } if !reflect.DeepEqual(dr.Keys, writes) { t.Errorf("expected %d keys to be deleted, but got %d instead", writes, dr.Keys) } scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next()) txn := &roachpb.Transaction{Name: "MyTxn"} reply, err = client.SendWrappedWith(tds, nil, roachpb.Header{Txn: txn}, scan) if err != nil { t.Fatal(err) } sr := reply.(*roachpb.ScanResponse) if txn := sr.Txn; txn == nil || txn.Name != "MyTxn" { t.Errorf("wanted Txn to persist, but it changed to %v", txn) } if rows := sr.Rows; len(rows) > 0 { t.Fatalf("scan after delete returned rows: %v", rows) } }
// TestStoreRangeMergeWithData attempts to merge two collocate ranges // each containing data. func TestStoreRangeMergeWithData(t *testing.T) { defer leaktest.AfterTest(t)() sCtx := storage.TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, stopper, _ := createTestStoreWithContext(t, sCtx) defer stopper.Stop() content := roachpb.Key("testing!") aDesc, bDesc, err := createSplitRanges(store) if err != nil { t.Fatal(err) } // Write some values left and right of the proposed split key. pArgs := putArgs([]byte("aaa"), content) if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil { t.Fatal(err) } pArgs = putArgs([]byte("ccc"), content) if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{ RangeID: bDesc.RangeID, }, &pArgs); err != nil { t.Fatal(err) } // Confirm the values are there. gArgs := getArgs([]byte("aaa")) if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil { t.Fatal(err) } else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil { t.Fatal(err) } else if !bytes.Equal(replyBytes, content) { t.Fatalf("actual value %q did not match expected value %q", replyBytes, content) } gArgs = getArgs([]byte("ccc")) if reply, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{ RangeID: bDesc.RangeID, }, &gArgs); err != nil { t.Fatal(err) } else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil { t.Fatal(err) } else if !bytes.Equal(replyBytes, content) { t.Fatalf("actual value %q did not match expected value %q", replyBytes, content) } // Merge the b range back into the a range. args := adminMergeArgs(roachpb.KeyMin) if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil { t.Fatal(err) } // Verify no intents remains on range descriptor keys. for _, key := range []roachpb.Key{keys.RangeDescriptorKey(aDesc.StartKey), keys.RangeDescriptorKey(bDesc.StartKey)} { if _, _, err := engine.MVCCGet(context.Background(), store.Engine(), key, store.Clock().Now(), true, nil); err != nil { t.Fatal(err) } } // Verify the merge by looking up keys from both ranges. rangeA := store.LookupReplica([]byte("a"), nil) rangeB := store.LookupReplica([]byte("c"), nil) rangeADesc := rangeA.Desc() rangeBDesc := rangeB.Desc() if !reflect.DeepEqual(rangeA, rangeB) { t.Fatalf("ranges were not merged %+v=%+v", rangeADesc, rangeBDesc) } if !bytes.Equal(rangeADesc.StartKey, roachpb.RKeyMin) { t.Fatalf("The start key is not equal to KeyMin %q=%q", rangeADesc.StartKey, roachpb.RKeyMin) } if !bytes.Equal(rangeADesc.EndKey, roachpb.RKeyMax) { t.Fatalf("The end key is not equal to KeyMax %q=%q", rangeADesc.EndKey, roachpb.RKeyMax) } // Try to get values from after the merge. gArgs = getArgs([]byte("aaa")) if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil { t.Fatal(err) } else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil { t.Fatal(err) } else if !bytes.Equal(replyBytes, content) { t.Fatalf("actual value %q did not match expected value %q", replyBytes, content) } gArgs = getArgs([]byte("ccc")) if reply, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{ RangeID: rangeB.RangeID, }, &gArgs); err != nil { t.Fatal(err) } else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil { t.Fatal(err) } else if !bytes.Equal(replyBytes, content) { t.Fatalf("actual value %q did not match expected value %q", replyBytes, content) } // Put new values after the merge on both sides. pArgs = putArgs([]byte("aaaa"), content) if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil { t.Fatal(err) } pArgs = putArgs([]byte("cccc"), content) if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{ RangeID: rangeB.RangeID, }, &pArgs); err != nil { t.Fatal(err) } // Try to get the newly placed values. gArgs = getArgs([]byte("aaaa")) if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil { t.Fatal(err) } else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil { t.Fatal(err) } else if !bytes.Equal(replyBytes, content) { t.Fatalf("actual value %q did not match expected value %q", replyBytes, content) } gArgs = getArgs([]byte("cccc")) if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil { t.Fatal(err) } else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil { t.Fatal(err) } else if !bytes.Equal(replyBytes, content) { t.Fatalf("actual value %q did not match expected value %q", replyBytes, content) } }