// TestRejectFutureCommand verifies that lease holders reject commands that
// would cause a large time jump.
func TestRejectFutureCommand(t *testing.T) {
defer leaktest.AfterTest(t)()
const maxOffset = 100 * time.Millisecond
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(maxOffset)
mtc := multiTestContext{
clock: clock,
}
mtc.Start(t, 1)
defer mtc.Stop()
// First do a write. The first write will advance the clock by MaxOffset
// because of the read cache's low water mark.
getArgs := putArgs([]byte("b"), []byte("b"))
if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &getArgs); err != nil {
t.Fatal(err)
}
if now := clock.Now(); now.WallTime != int64(maxOffset) {
t.Fatalf("expected clock to advance to 100ms; got %s", now)
}
// The logical clock has advanced past the physical clock; increment
// the "physical" clock to catch up.
manual.Increment(int64(maxOffset))
startTime := manual.UnixNano()
// Commands with a future timestamp that is within the MaxOffset
// bound will be accepted and will cause the clock to advance.
for i := int64(0); i < 3; i++ {
incArgs := incrementArgs([]byte("a"), 5)
ts := hlc.ZeroTimestamp.Add(startTime+((i+1)*30)*int64(time.Millisecond), 0)
if _, err := client.SendWrappedWith(rg1(mtc.stores[0]), nil, roachpb.Header{Timestamp: ts}, &incArgs); err != nil {
t.Fatal(err)
}
}
if now := clock.Now(); now.WallTime != int64(190*time.Millisecond) {
t.Fatalf("expected clock to advance to 190ms; got %s", now)
}
// Once the accumulated offset reaches MaxOffset, commands will be rejected.
incArgs := incrementArgs([]byte("a"), 11)
ts := hlc.ZeroTimestamp.Add(int64((time.Duration(startTime)+maxOffset+1)*time.Millisecond), 0)
if _, err := client.SendWrappedWith(rg1(mtc.stores[0]), nil, roachpb.Header{Timestamp: ts}, &incArgs); err == nil {
t.Fatalf("expected clock offset error but got nil")
}
// The clock remained at 190ms and the final command was not executed.
if now := clock.Now(); now.WallTime != int64(190*time.Millisecond) {
t.Errorf("expected clock to advance to 190ms; got %s", now)
}
val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], roachpb.Key("a"), clock.Now(), true, nil)
if err != nil {
t.Fatal(err)
}
if v := mustGetInt(val); v != 15 {
t.Errorf("expected 15, got %v", v)
}
}
// TestTxnMultipleCoord checks that a coordinator uses the Writing flag to
// enforce that only one coordinator can be used for transactional writes.
func TestTxnMultipleCoord(t *testing.T) {
defer leaktest.AfterTest(t)()
s, sender := createTestDB(t)
defer s.Stop()
testCases := []struct {
args roachpb.Request
writing bool
ok bool
}{
{roachpb.NewGet(roachpb.Key("a")), true, false},
{roachpb.NewGet(roachpb.Key("a")), false, true},
{roachpb.NewPut(roachpb.Key("a"), roachpb.Value{}), false, false}, // transactional write before begin
{roachpb.NewPut(roachpb.Key("a"), roachpb.Value{}), true, false}, // must have switched coordinators
}
for i, tc := range testCases {
txn := roachpb.NewTransaction("test", roachpb.Key("a"), 1, enginepb.SERIALIZABLE,
s.Clock.Now(), s.Clock.MaxOffset().Nanoseconds())
txn.Writing = tc.writing
reply, pErr := client.SendWrappedWith(sender, nil, roachpb.Header{
Txn: txn,
}, tc.args)
if pErr == nil != tc.ok {
t.Errorf("%d: %T (writing=%t): success_expected=%t, but got: %v",
i, tc.args, tc.writing, tc.ok, pErr)
}
if pErr != nil {
continue
}
txn = reply.Header().Txn
// The transaction should come back rw if it started rw or if we just
// wrote.
isWrite := roachpb.IsTransactionWrite(tc.args)
if (tc.writing || isWrite) != txn.Writing {
t.Errorf("%d: unexpected writing state: %s", i, txn)
}
if !isWrite {
continue
}
// Abort for clean shutdown.
if _, pErr := client.SendWrappedWith(sender, nil, roachpb.Header{
Txn: txn,
}, &roachpb.EndTransactionRequest{
Commit: false,
}); pErr != nil {
t.Fatal(pErr)
}
}
}
// fillRange writes keys with the given prefix and associated values
// until bytes bytes have been written or the given range has split.
func fillRange(store *storage.Store, rangeID roachpb.RangeID, prefix roachpb.Key, bytes int64, t *testing.T) {
src := rand.New(rand.NewSource(0))
for {
var ms enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), store.Engine(), rangeID, &ms); err != nil {
t.Fatal(err)
}
keyBytes, valBytes := ms.KeyBytes, ms.ValBytes
if keyBytes+valBytes >= bytes {
return
}
key := append(append([]byte(nil), prefix...), randutil.RandBytes(src, 100)...)
key = keys.MakeRowSentinelKey(key)
val := randutil.RandBytes(src, int(src.Int31n(1<<8)))
pArgs := putArgs(key, val)
_, pErr := client.SendWrappedWith(store, nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs)
// When the split occurs in the background, our writes may start failing.
// We know we can stop writing when this happens.
if _, ok := pErr.GetDetail().(*roachpb.RangeKeyMismatchError); ok {
return
} else if pErr != nil {
t.Fatal(pErr)
}
}
}
func TestComputeStatsForKeySpan(t *testing.T) {
defer leaktest.AfterTest(t)()
mtc := startMultiTestContext(t, 3)
defer mtc.Stop()
// Create a number of ranges using splits.
splitKeys := []string{"a", "c", "e", "g", "i"}
for _, k := range splitKeys {
key := []byte(k)
repl := mtc.stores[0].LookupReplica(key, roachpb.RKeyMin)
args := adminSplitArgs(key, key)
header := roachpb.Header{
RangeID: repl.RangeID,
}
if _, err := client.SendWrappedWith(mtc.stores[0], nil, header, &args); err != nil {
t.Fatal(err)
}
}
// Wait for splits to finish.
util.SucceedsSoon(t, func() error {
repl := mtc.stores[0].LookupReplica(roachpb.RKey("z"), nil)
if actualRSpan := repl.Desc().RSpan(); !actualRSpan.Key.Equal(roachpb.RKey("i")) {
return errors.Errorf("expected range %s to begin at key 'i'", repl)
}
return nil
})
// Create some keys across the ranges.
incKeys := []string{"b", "bb", "bbb", "d", "dd", "h"}
for _, k := range incKeys {
if _, err := mtc.dbs[0].Inc([]byte(k), 5); err != nil {
t.Fatal(err)
}
}
// Verify stats across different spans.
for _, tcase := range []struct {
startKey string
endKey string
expectedRanges int
expectedKeys int64
}{
{"a", "i", 4, 6},
{"a", "c", 1, 3},
{"b", "e", 2, 5},
{"e", "i", 2, 1},
} {
start, end := tcase.startKey, tcase.endKey
stats, count := mtc.stores[0].ComputeStatsForKeySpan(
roachpb.RKey(start), roachpb.RKey(end))
if a, e := count, tcase.expectedRanges; a != e {
t.Errorf("Expected %d ranges in span [%s - %s], found %d", e, start, end, a)
}
if a, e := stats.LiveCount, tcase.expectedKeys; a != e {
t.Errorf("Expected %d keys in span [%s - %s], found %d", e, start, end, a)
}
}
}
// TestRangeCommandClockUpdate verifies that followers update their
// clocks when executing a command, even if the lease holder's clock is far
// in the future.
func TestRangeCommandClockUpdate(t *testing.T) {
defer leaktest.AfterTest(t)()
const numNodes = 3
var manuals []*hlc.ManualClock
var clocks []*hlc.Clock
for i := 0; i < numNodes; i++ {
manuals = append(manuals, hlc.NewManualClock(1))
clocks = append(clocks, hlc.NewClock(manuals[i].UnixNano))
clocks[i].SetMaxOffset(100 * time.Millisecond)
}
mtc := multiTestContext{
clocks: clocks,
}
mtc.Start(t, numNodes)
defer mtc.Stop()
mtc.replicateRange(1, 1, 2)
// Advance the lease holder's clock ahead of the followers (by more than
// MaxOffset but less than the range lease) and execute a command.
manuals[0].Increment(int64(500 * time.Millisecond))
incArgs := incrementArgs([]byte("a"), 5)
ts := clocks[0].Now()
if _, err := client.SendWrappedWith(rg1(mtc.stores[0]), nil, roachpb.Header{Timestamp: ts}, &incArgs); err != nil {
t.Fatal(err)
}
// Wait for that command to execute on all the followers.
util.SucceedsSoon(t, func() error {
values := []int64{}
for _, eng := range mtc.engines {
val, _, err := engine.MVCCGet(context.Background(), eng, roachpb.Key("a"), clocks[0].Now(), true, nil)
if err != nil {
return err
}
values = append(values, mustGetInt(val))
}
if !reflect.DeepEqual(values, []int64{5, 5, 5}) {
return errors.Errorf("expected (5, 5, 5), got %v", values)
}
return nil
})
// Verify that all the followers have accepted the clock update from
// node 0 even though it comes from outside the usual max offset.
now := clocks[0].Now()
for i, clock := range clocks {
// Only compare the WallTimes: it's normal for clock 0 to be a few logical ticks ahead.
if clock.Now().WallTime < now.WallTime {
t.Errorf("clock %d is behind clock 0: %s vs %s", i, clock.Now(), now)
}
}
}
func writeRandomTimeSeriesDataToRange(
t testing.TB,
store *storage.Store,
rangeID roachpb.RangeID,
keyPrefix []byte,
) (midpoint []byte) {
src := rand.New(rand.NewSource(0))
r := ts.Resolution10s
for i := 0; i < 20; i++ {
var data []tspb.TimeSeriesData
for j := int64(0); j <= src.Int63n(5); j++ {
d := tspb.TimeSeriesData{
Name: "test.random.metric",
Source: "cpu01",
}
for k := int64(0); k <= src.Int63n(10); k++ {
d.Datapoints = append(d.Datapoints, tspb.TimeSeriesDatapoint{
TimestampNanos: src.Int63n(200) * r.KeyDuration(),
Value: src.Float64(),
})
}
data = append(data, d)
}
for _, d := range data {
idatas, err := d.ToInternal(r.KeyDuration(), r.SampleDuration())
if err != nil {
t.Fatal(err)
}
for _, idata := range idatas {
var value roachpb.Value
if err := value.SetProto(&idata); err != nil {
t.Fatal(err)
}
mArgs := roachpb.MergeRequest{
Span: roachpb.Span{
Key: encoding.EncodeVarintAscending(keyPrefix, idata.StartTimestampNanos),
},
Value: value,
}
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeID,
}, &mArgs); pErr != nil {
t.Fatal(pErr)
}
}
}
}
// Return approximate midway point (100 is midway between random timestamps in range [0,200)).
midKey := append([]byte(nil), keyPrefix...)
midKey = encoding.EncodeVarintAscending(midKey, 100*r.KeyDuration())
return keys.MakeRowSentinelKey(midKey)
}
func fillTestRange(t testing.TB, rep *Replica, size int) {
src := rand.New(rand.NewSource(0))
for i := 0; i < snapSize/(keySize+valSize); i++ {
key := keys.MakeRowSentinelKey(randutil.RandBytes(src, keySize))
val := randutil.RandBytes(src, valSize)
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(rep, nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
t.Fatal(pErr)
}
}
}
// getTxn fetches the requested key and returns the transaction info.
func getTxn(coord *TxnCoordSender, txn *roachpb.Transaction) (*roachpb.Transaction, *roachpb.Error) {
hb := &roachpb.HeartbeatTxnRequest{
Span: roachpb.Span{
Key: txn.Key,
},
}
reply, pErr := client.SendWrappedWith(coord, nil, roachpb.Header{
Txn: txn,
}, hb)
if pErr != nil {
return nil, pErr
}
return reply.(*roachpb.HeartbeatTxnResponse).Txn, nil
}
func writeRandomDataToRange(t testing.TB, store *storage.Store, rangeID roachpb.RangeID, keyPrefix []byte) {
src := rand.New(rand.NewSource(0))
for i := 0; i < 100; i++ {
key := append([]byte(nil), keyPrefix...)
key = append(key, randutil.RandBytes(src, int(src.Int31n(1<<7)))...)
key = keys.MakeRowSentinelKey(key)
val := randutil.RandBytes(src, int(src.Int31n(1<<8)))
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
t.Fatal(pErr)
}
}
}
// FindRangeLeaseHolder returns the current lease holder for the given range. If
// there is no lease at the time of the call, a replica is gets one as a
// side-effect of calling this; if hint is not nil, that replica will be the
// one.
//
// One of the Stores in the cluster is used as a Sender to send a dummy read
// command, which will either result in success (if a replica on that Node has
// the lease), in a NotLeaseHolderError pointing to the current lease holder (if
// there is an active lease), or in the replica on that store acquiring the
// lease (if there isn't an active lease).
// If an active lease existed for the range, it's extended as a side-effect.
func (tc *TestCluster) FindRangeLeaseHolder(
rangeDesc *roachpb.RangeDescriptor,
hint *ReplicationTarget,
) (ReplicationTarget, error) {
var hintReplicaDesc roachpb.ReplicaDescriptor
if hint != nil {
var ok bool
if hintReplicaDesc, ok = rangeDesc.GetReplicaDescriptor(hint.StoreID); !ok {
return ReplicationTarget{}, errors.Errorf(
"bad hint; store doesn't have a replica of the range")
}
} else {
hint = &ReplicationTarget{
NodeID: rangeDesc.Replicas[0].NodeID,
StoreID: rangeDesc.Replicas[0].StoreID}
hintReplicaDesc = rangeDesc.Replicas[0]
}
// TODO(andrei): Using a dummy GetRequest for the purpose of figuring out the
// lease holder is a hack. Instead, we should have a dedicate admin command.
getReq := roachpb.GetRequest{
Span: roachpb.Span{
Key: rangeDesc.StartKey.AsRawKey(),
},
}
store, err := tc.findMemberStore(hint.StoreID)
if err != nil {
return ReplicationTarget{}, err
}
_, pErr := client.SendWrappedWith(
store, nil,
roachpb.Header{RangeID: rangeDesc.RangeID, Replica: hintReplicaDesc},
&getReq)
if pErr != nil {
if nle, ok := pErr.GetDetail().(*roachpb.NotLeaseHolderError); ok {
if nle.LeaseHolder == nil {
return ReplicationTarget{}, errors.Errorf(
"unexpected NotLeaseHolderError with lease holder unknown")
}
return ReplicationTarget{
NodeID: nle.LeaseHolder.NodeID, StoreID: nle.LeaseHolder.StoreID}, nil
}
return ReplicationTarget{}, pErr.GoError()
}
// The replica we sent the request to either was already or just became
// the lease holder.
return *hint, nil
}
func fillTestRange(t testing.TB, rep *Replica, size int64) {
src := rand.New(rand.NewSource(0))
for i := int64(0); i < size/int64(keySize+valSize); i++ {
key := keys.MakeRowSentinelKey(randutil.RandBytes(src, keySize))
val := randutil.RandBytes(src, valSize)
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(rep, nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
t.Fatal(pErr)
}
}
rep.mu.Lock()
after := rep.mu.state.Stats.Total()
rep.mu.Unlock()
if after < size {
t.Fatalf("range not full after filling: wrote %d, but range at %d", size, after)
}
}
// process synchronously invokes admin split for each proposed split key.
func (sq *splitQueue) process(
ctx context.Context,
now hlc.Timestamp,
rng *Replica,
sysCfg config.SystemConfig,
) error {
// First handle case of splitting due to zone config maps.
desc := rng.Desc()
splitKeys := sysCfg.ComputeSplitKeys(desc.StartKey, desc.EndKey)
if len(splitKeys) > 0 {
log.Infof("splitting %s at keys %v", rng, splitKeys)
log.Trace(ctx, fmt.Sprintf("splitting at keys %v", splitKeys))
for _, splitKey := range splitKeys {
if err := sq.db.AdminSplit(splitKey.AsRawKey()); err != nil {
return errors.Errorf("unable to split %s at key %q: %s", rng, splitKey, err)
}
}
return nil
}
// Next handle case of splitting due to size.
zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
if err != nil {
return err
}
size := rng.GetMVCCStats().Total()
// FIXME: why is this implementation not the same as the one above?
if float64(size)/float64(zone.RangeMaxBytes) > 1 {
log.Infof("splitting %s size=%d max=%d", rng, size, zone.RangeMaxBytes)
log.Trace(ctx, fmt.Sprintf("splitting size=%d max=%d", size, zone.RangeMaxBytes))
if _, pErr := client.SendWrappedWith(rng, ctx, roachpb.Header{
Timestamp: now,
}, &roachpb.AdminSplitRequest{
Span: roachpb.Span{Key: desc.StartKey.AsRawKey()},
}); pErr != nil {
return pErr.GoError()
}
}
return nil
}
func BenchmarkReplicaSnapshot(b *testing.B) {
defer tracing.Disable()()
defer config.TestingDisableTableSplits()()
store, stopper, _ := createTestStore(b)
// We want to manually control the size of the raft log.
store.DisableRaftLogQueue(true)
defer stopper.Stop()
const rangeID = 1
const keySize = 1 << 7 // 128 B
const valSize = 1 << 10 // 1 KiB
const snapSize = 1 << 25 // 32 MiB
rep, err := store.GetReplica(rangeID)
if err != nil {
b.Fatal(err)
}
src := rand.New(rand.NewSource(0))
for i := 0; i < snapSize/(keySize+valSize); i++ {
key := keys.MakeRowSentinelKey(randutil.RandBytes(src, keySize))
val := randutil.RandBytes(src, valSize)
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(rep, nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
b.Fatal(pErr)
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := rep.GetSnapshot(); err != nil {
b.Fatal(err)
}
}
}
func writeRandomDataToRange(
t testing.TB,
store *storage.Store,
rangeID roachpb.RangeID,
keyPrefix []byte,
) (midpoint []byte) {
src := rand.New(rand.NewSource(0))
for i := 0; i < 100; i++ {
key := append([]byte(nil), keyPrefix...)
key = append(key, randutil.RandBytes(src, int(src.Int31n(1<<7)))...)
key = keys.MakeRowSentinelKey(key)
val := randutil.RandBytes(src, int(src.Int31n(1<<8)))
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
t.Fatal(pErr)
}
}
// Return approximate midway point ("Z" in string "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz").
midKey := append([]byte(nil), keyPrefix...)
midKey = append(midKey, []byte("Z")...)
return keys.MakeRowSentinelKey(midKey)
}
// TestTxnPutOutOfOrder tests a case where a put operation of an older
// timestamp comes after a put operation of a newer timestamp in a
// txn. The test ensures such an out-of-order put succeeds and
// overrides an old value. The test uses a "Writer" and a "Reader"
// to reproduce an out-of-order put.
//
// 1) The Writer executes a put operation and writes a write intent with
// time T in a txn.
// 2) Before the Writer's txn is committed, the Reader sends a high priority
// get operation with time T+100. This pushes the Writer txn timestamp to
// T+100 and triggers the restart of the Writer's txn. The original
// write intent timestamp is also updated to T+100.
// 3) The Writer starts a new epoch of the txn, but before it writes, the
// Reader sends another high priority get operation with time T+200. This
// pushes the Writer txn timestamp to T+200 to trigger a restart of the
// Writer txn. The Writer will not actually restart until it tries to commit
// the current epoch of the transaction. The Reader updates the timestamp of
// the write intent to T+200. The test deliberately fails the Reader get
// operation, and cockroach doesn't update its read timestamp cache.
// 4) The Writer executes the put operation again. This put operation comes
// out-of-order since its timestamp is T+100, while the intent timestamp
// updated at Step 3 is T+200.
// 5) The put operation overrides the old value using timestamp T+100.
// 6) When the Writer attempts to commit its txn, the txn will be restarted
// again at a new epoch timestamp T+200, which will finally succeed.
func TestTxnPutOutOfOrder(t *testing.T) {
defer leaktest.AfterTest(t)()
key := "key"
// Set up a filter to so that the get operation at Step 3 will return an error.
var numGets int32
manualClock := hlc.NewManualClock(0)
clock := hlc.NewClock(manualClock.UnixNano)
stopper := stop.NewStopper()
defer stopper.Stop()
ctx := storage.TestStoreContext()
ctx.TestingKnobs.TestingCommandFilter =
func(filterArgs storagebase.FilterArgs) *roachpb.Error {
if _, ok := filterArgs.Req.(*roachpb.GetRequest); ok &&
filterArgs.Req.Header().Key.Equal(roachpb.Key(key)) &&
filterArgs.Hdr.Txn == nil {
// The Reader executes two get operations, each of which triggers two get requests
// (the first request fails and triggers txn push, and then the second request
// succeeds). Returns an error for the fourth get request to avoid timestamp cache
// update after the third get operation pushes the txn timestamp.
if atomic.AddInt32(&numGets, 1) == 4 {
return roachpb.NewErrorWithTxn(errors.Errorf("Test"), filterArgs.Hdr.Txn)
}
}
return nil
}
store := createTestStoreWithEngine(t,
engine.NewInMem(roachpb.Attributes{}, 10<<20, stopper),
clock,
true,
ctx,
stopper)
// Put an initial value.
initVal := []byte("initVal")
err := store.DB().Put(key, initVal)
if err != nil {
t.Fatalf("failed to put: %s", err)
}
waitPut := make(chan struct{})
waitFirstGet := make(chan struct{})
waitTxnRestart := make(chan struct{})
waitSecondGet := make(chan struct{})
waitTxnComplete := make(chan struct{})
// Start the Writer.
go func() {
epoch := -1
// Start a txn that does read-after-write.
// The txn will be restarted twice, and the out-of-order put
// will happen in the second epoch.
if err := store.DB().Txn(func(txn *client.Txn) error {
epoch++
if epoch == 1 {
// Wait until the second get operation is issued.
close(waitTxnRestart)
<-waitSecondGet
}
updatedVal := []byte("updatedVal")
if err := txn.Put(key, updatedVal); err != nil {
return err
}
// Make sure a get will return the value that was just written.
actual, err := txn.Get(key)
if err != nil {
return err
}
if !bytes.Equal(actual.ValueBytes(), updatedVal) {
t.Fatalf("unexpected get result: %s", actual)
}
if epoch == 0 {
// Wait until the first get operation will push the txn timestamp.
close(waitPut)
<-waitFirstGet
}
b := txn.NewBatch()
return txn.CommitInBatch(b)
}); err != nil {
t.Fatal(err)
}
if epoch != 2 {
t.Fatalf("unexpected number of txn retries: %d", epoch)
}
close(waitTxnComplete)
}()
<-waitPut
// Start the Reader.
// Advance the clock and send a get operation with higher
// priority to trigger the txn restart.
manualClock.Increment(100)
priority := roachpb.UserPriority(-math.MaxInt32)
requestHeader := roachpb.Span{
Key: roachpb.Key(key),
}
ts := clock.Now()
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Timestamp: ts,
UserPriority: priority,
}, &roachpb.GetRequest{Span: requestHeader}); err != nil {
t.Fatalf("failed to get: %s", err)
}
// Wait until the writer restarts the txn.
close(waitFirstGet)
<-waitTxnRestart
// Advance the clock and send a get operation again. This time
// we use TestingCommandFilter so that a get operation is not
// processed after the write intent is resolved (to prevent the
// timestamp cache from being updated).
manualClock.Increment(100)
ts = clock.Now()
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Timestamp: ts,
UserPriority: priority,
}, &roachpb.GetRequest{Span: requestHeader}); err == nil {
t.Fatal("unexpected success of get")
}
close(waitSecondGet)
<-waitTxnComplete
}
// TestStoreRangeMergeMetadataCleanup tests that all metadata of a
// subsumed range is cleaned up on merge.
func TestStoreRangeMergeMetadataCleanup(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
scan := func(f func(roachpb.KeyValue) (bool, error)) {
if _, err := engine.MVCCIterate(context.Background(), store.Engine(), roachpb.KeyMin, roachpb.KeyMax, hlc.ZeroTimestamp, true, nil, false, f); err != nil {
t.Fatal(err)
}
}
content := roachpb.Key("testing!")
// Write some values left of the proposed split key.
pArgs := putArgs([]byte("aaa"), content)
if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil {
t.Fatal(err)
}
// Collect all the keys.
preKeys := make(map[string]struct{})
scan(func(kv roachpb.KeyValue) (bool, error) {
preKeys[string(kv.Key)] = struct{}{}
return false, nil
})
// Split the range.
_, bDesc, err := createSplitRanges(store)
if err != nil {
t.Fatal(err)
}
// Write some values right of the split key.
pArgs = putArgs([]byte("ccc"), content)
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: bDesc.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil {
t.Fatal(err)
}
// Collect all the keys again.
postKeys := make(map[string]struct{})
scan(func(kv roachpb.KeyValue) (bool, error) {
postKeys[string(kv.Key)] = struct{}{}
return false, nil
})
// Compute the new keys.
for k := range preKeys {
delete(postKeys, k)
}
// Keep only the subsumed range's local keys.
localRangeKeyPrefix := string(keys.MakeRangeIDPrefix(bDesc.RangeID))
for k := range postKeys {
if !strings.HasPrefix(k, localRangeKeyPrefix) {
delete(postKeys, k)
}
}
if numKeys := len(postKeys); numKeys > 0 {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%d keys were not cleaned up:\n", numKeys)
for k := range postKeys {
fmt.Fprintf(&buf, "%q\n", k)
}
t.Fatal(buf.String())
}
}
// TestStoreRangeMergeWithData attempts to merge two collocate ranges
// each containing data.
func TestStoreRangeMergeWithData(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
content := roachpb.Key("testing!")
aDesc, bDesc, err := createSplitRanges(store)
if err != nil {
t.Fatal(err)
}
// Write some values left and right of the proposed split key.
pArgs := putArgs([]byte("aaa"), content)
if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil {
t.Fatal(err)
}
pArgs = putArgs([]byte("ccc"), content)
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: bDesc.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Confirm the values are there.
gArgs := getArgs([]byte("aaa"))
if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("ccc"))
if reply, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: bDesc.RangeID,
}, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil {
t.Fatal(err)
}
// Verify no intents remains on range descriptor keys.
for _, key := range []roachpb.Key{keys.RangeDescriptorKey(aDesc.StartKey), keys.RangeDescriptorKey(bDesc.StartKey)} {
if _, _, err := engine.MVCCGet(context.Background(), store.Engine(), key, store.Clock().Now(), true, nil); err != nil {
t.Fatal(err)
}
}
// Verify the merge by looking up keys from both ranges.
rangeA := store.LookupReplica([]byte("a"), nil)
rangeB := store.LookupReplica([]byte("c"), nil)
rangeADesc := rangeA.Desc()
rangeBDesc := rangeB.Desc()
if !reflect.DeepEqual(rangeA, rangeB) {
t.Fatalf("ranges were not merged %+v=%+v", rangeADesc, rangeBDesc)
}
if !bytes.Equal(rangeADesc.StartKey, roachpb.RKeyMin) {
t.Fatalf("The start key is not equal to KeyMin %q=%q", rangeADesc.StartKey, roachpb.RKeyMin)
}
if !bytes.Equal(rangeADesc.EndKey, roachpb.RKeyMax) {
t.Fatalf("The end key is not equal to KeyMax %q=%q", rangeADesc.EndKey, roachpb.RKeyMax)
}
// Try to get values from after the merge.
gArgs = getArgs([]byte("aaa"))
if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("ccc"))
if reply, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeB.RangeID,
}, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Put new values after the merge on both sides.
pArgs = putArgs([]byte("aaaa"), content)
if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil {
t.Fatal(err)
}
pArgs = putArgs([]byte("cccc"), content)
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rangeB.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Try to get the newly placed values.
gArgs = getArgs([]byte("aaaa"))
if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("cccc"))
if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
}
// TestMultiRangeScanWithMaxResults tests that commands which access multiple
// ranges with MaxResults parameter are carried out properly.
func TestMultiRangeScanWithMaxResults(t *testing.T) {
defer leaktest.AfterTest(t)()
testCases := []struct {
splitKeys []roachpb.Key
keys []roachpb.Key
}{
{[]roachpb.Key{roachpb.Key("m")},
[]roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}},
{[]roachpb.Key{roachpb.Key("h"), roachpb.Key("q")},
[]roachpb.Key{roachpb.Key("b"), roachpb.Key("f"), roachpb.Key("k"),
roachpb.Key("r"), roachpb.Key("w"), roachpb.Key("y")}},
}
for i, tc := range testCases {
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
ts := s.(*TestServer)
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = ts.stopper.ShouldQuiesce()
ds := kv.NewDistSender(&kv.DistSenderConfig{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RPCRetryOptions: &retryOpts,
}, ts.Gossip())
ctx := tracing.WithTracer(context.Background(), tracing.NewTracer())
tds := kv.NewTxnCoordSender(ctx, ds, ts.Clock(), ts.Ctx.Linearizable,
ts.stopper, kv.MakeTxnMetrics())
for _, sk := range tc.splitKeys {
if err := ts.node.ctx.DB.AdminSplit(sk); err != nil {
t.Fatal(err)
}
}
for _, k := range tc.keys {
put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k))
if _, err := client.SendWrapped(tds, nil, put); err != nil {
t.Fatal(err)
}
}
// Try every possible ScanRequest startKey.
for start := 0; start < len(tc.keys); start++ {
// Try every possible maxResults, from 1 to beyond the size of key array.
for maxResults := 1; maxResults <= len(tc.keys)-start+1; maxResults++ {
scan := roachpb.NewScan(tc.keys[start], tc.keys[len(tc.keys)-1].Next())
reply, err := client.SendWrappedWith(
tds, nil, roachpb.Header{MaxSpanRequestKeys: int64(maxResults)}, scan,
)
if err != nil {
t.Fatal(err)
}
rows := reply.(*roachpb.ScanResponse).Rows
if start+maxResults <= len(tc.keys) && len(rows) != maxResults {
t.Errorf("%d: start=%s: expected %d rows, but got %d", i, tc.keys[start], maxResults, len(rows))
} else if start+maxResults == len(tc.keys)+1 && len(rows) != maxResults-1 {
t.Errorf("%d: expected %d rows, but got %d", i, maxResults-1, len(rows))
}
}
}
}
}
// TestMultiRangeScanDeleteRange tests that commands which access multiple
// ranges are carried out properly.
func TestMultiRangeScanDeleteRange(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
ts := s.(*TestServer)
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = ts.stopper.ShouldQuiesce()
ds := kv.NewDistSender(&kv.DistSenderConfig{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RPCRetryOptions: &retryOpts,
}, ts.Gossip())
ctx := tracing.WithTracer(context.Background(), tracing.NewTracer())
tds := kv.NewTxnCoordSender(ctx, ds, s.Clock(), ts.Ctx.Linearizable,
ts.stopper, kv.MakeTxnMetrics())
if err := ts.node.ctx.DB.AdminSplit("m"); err != nil {
t.Fatal(err)
}
writes := []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}
get := &roachpb.GetRequest{
Span: roachpb.Span{Key: writes[0]},
}
get.EndKey = writes[len(writes)-1]
if _, err := client.SendWrapped(tds, nil, get); err == nil {
t.Errorf("able to call Get with a key range: %v", get)
}
var delTS hlc.Timestamp
for i, k := range writes {
put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k))
reply, err := client.SendWrapped(tds, nil, put)
if err != nil {
t.Fatal(err)
}
scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next())
reply, err = client.SendWrapped(tds, nil, scan)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if sr.Txn != nil {
// This was the other way around at some point in the past.
// Same below for Delete, etc.
t.Errorf("expected no transaction in response header")
}
if rows := sr.Rows; len(rows) != i+1 {
t.Fatalf("expected %d rows, but got %d", i+1, len(rows))
}
}
del := &roachpb.DeleteRangeRequest{
Span: roachpb.Span{
Key: writes[0],
EndKey: roachpb.Key(writes[len(writes)-1]).Next(),
},
ReturnKeys: true,
}
reply, err := client.SendWrappedWith(tds, nil, roachpb.Header{Timestamp: delTS}, del)
if err != nil {
t.Fatal(err)
}
dr := reply.(*roachpb.DeleteRangeResponse)
if dr.Txn != nil {
t.Errorf("expected no transaction in response header")
}
if !reflect.DeepEqual(dr.Keys, writes) {
t.Errorf("expected %d keys to be deleted, but got %d instead", writes, dr.Keys)
}
scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next())
txn := &roachpb.Transaction{Name: "MyTxn"}
reply, err = client.SendWrappedWith(tds, nil, roachpb.Header{Txn: txn}, scan)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if txn := sr.Txn; txn == nil || txn.Name != "MyTxn" {
t.Errorf("wanted Txn to persist, but it changed to %v", txn)
}
if rows := sr.Rows; len(rows) > 0 {
t.Fatalf("scan after delete returned rows: %v", rows)
}
}
// TestStoreRangeSplitStatsWithMerges starts by splitting the system keys from
// user-space keys and verifying that the user space side of the split (which is empty),
// has all zeros for stats. It then issues a number of Merge requests to the user
// space side, simulating TimeSeries data. Finally, the test splits the user space
// side halfway and verifies the stats on either side of the split are equal to a
// recomputation.
//
// Note that unlike TestStoreRangeSplitStats, we do not check if the two halves of the
// split's stats are equal to the pre-split stats when added, because this will not be
// true of ranges populated with Merge requests. The reason for this is that Merge
// requests' impact on MVCCStats are only estimated. See updateStatsOnMerge.
func TestStoreRangeSplitStatsWithMerges(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, manual := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
// Split the range after the last table data key.
keyPrefix := keys.MakeTablePrefix(keys.MaxReservedDescID + 1)
keyPrefix = keys.MakeRowSentinelKey(keyPrefix)
args := adminSplitArgs(roachpb.KeyMin, keyPrefix)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); pErr != nil {
t.Fatal(pErr)
}
// Verify empty range has empty stats.
rng := store.LookupReplica(keyPrefix, nil)
// NOTE that this value is expected to change over time, depending on what
// we store in the sys-local keyspace. Update it accordingly for this test.
empty := enginepb.MVCCStats{LastUpdateNanos: manual.UnixNano()}
if err := verifyRangeStats(store.Engine(), rng.RangeID, empty); err != nil {
t.Fatal(err)
}
// Write random TimeSeries data.
midKey := writeRandomTimeSeriesDataToRange(t, store, rng.RangeID, keyPrefix)
manual.Increment(100)
// Split the range at approximate halfway point.
args = adminSplitArgs(keyPrefix, midKey)
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rng.RangeID,
}, &args); pErr != nil {
t.Fatal(pErr)
}
snap := store.Engine().NewSnapshot()
defer snap.Close()
var msLeft, msRight enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), snap, rng.RangeID, &msLeft); err != nil {
t.Fatal(err)
}
rngRight := store.LookupReplica(midKey, nil)
if err := engine.MVCCGetRangeStats(context.Background(), snap, rngRight.RangeID, &msRight); err != nil {
t.Fatal(err)
}
// Stats should both have the new timestamp.
now := manual.UnixNano()
if lTs := msLeft.LastUpdateNanos; lTs != now {
t.Errorf("expected left range stats to have new timestamp, want %d, got %d", now, lTs)
}
if rTs := msRight.LastUpdateNanos; rTs != now {
t.Errorf("expected right range stats to have new timestamp, want %d, got %d", now, rTs)
}
// Stats should agree with recomputation.
if err := verifyRecomputedStats(snap, rng.Desc(), msLeft, now); err != nil {
t.Fatalf("failed to verify left range's stats after split: %v", err)
}
if err := verifyRecomputedStats(snap, rngRight.Desc(), msRight, now); err != nil {
t.Fatalf("failed to verify right range's stats after split: %v", err)
}
}
// TestStoreRangeSplit executes a split of a range and verifies that the
// resulting ranges respond to the right key ranges and that their stats
// have been properly accounted for and requests can't be replayed.
func TestStoreRangeSplitIdempotency(t *testing.T) {
defer leaktest.AfterTest(t)()
defer config.TestingDisableTableSplits()()
store, stopper, _ := createTestStore(t)
defer stopper.Stop()
rangeID := roachpb.RangeID(1)
splitKey := roachpb.Key("m")
content := roachpb.Key("asdvb")
// First, write some values left and right of the proposed split key.
pArgs := putArgs([]byte("c"), content)
if _, pErr := client.SendWrapped(rg1(store), nil, &pArgs); pErr != nil {
t.Fatal(pErr)
}
pArgs = putArgs([]byte("x"), content)
if _, pErr := client.SendWrapped(rg1(store), nil, &pArgs); pErr != nil {
t.Fatal(pErr)
}
// Increments are a good way of testing idempotency. Up here, we
// address them to the original range, then later to the one that
// contains the key.
txn := roachpb.NewTransaction("test", []byte("c"), 10, enginepb.SERIALIZABLE,
store.Clock().Now(), 0)
lIncArgs := incrementArgs([]byte("apoptosis"), 100)
lTxn := *txn
lTxn.Sequence++
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Txn: &lTxn,
}, &lIncArgs); pErr != nil {
t.Fatal(pErr)
}
rIncArgs := incrementArgs([]byte("wobble"), 10)
rTxn := *txn
rTxn.Sequence++
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Txn: &rTxn,
}, &rIncArgs); pErr != nil {
t.Fatal(pErr)
}
// Get the original stats for key and value bytes.
var ms enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), store.Engine(), rangeID, &ms); err != nil {
t.Fatal(err)
}
keyBytes, valBytes := ms.KeyBytes, ms.ValBytes
// Split the range.
args := adminSplitArgs(roachpb.KeyMin, splitKey)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); pErr != nil {
t.Fatal(pErr)
}
// Verify no intents remains on range descriptor keys.
splitKeyAddr, err := keys.Addr(splitKey)
if err != nil {
t.Fatal(err)
}
for _, key := range []roachpb.Key{keys.RangeDescriptorKey(roachpb.RKeyMin), keys.RangeDescriptorKey(splitKeyAddr)} {
if _, _, err := engine.MVCCGet(context.Background(), store.Engine(), key, store.Clock().Now(), true, nil); err != nil {
t.Fatal(err)
}
}
rng := store.LookupReplica(roachpb.RKeyMin, nil)
rngDesc := rng.Desc()
newRng := store.LookupReplica([]byte("m"), nil)
newRngDesc := newRng.Desc()
if !bytes.Equal(newRngDesc.StartKey, splitKey) || !bytes.Equal(splitKey, rngDesc.EndKey) {
t.Errorf("ranges mismatched, wanted %q=%q=%q", newRngDesc.StartKey, splitKey, rngDesc.EndKey)
}
if !bytes.Equal(newRngDesc.EndKey, roachpb.RKeyMax) || !bytes.Equal(rngDesc.StartKey, roachpb.RKeyMin) {
t.Errorf("new ranges do not cover KeyMin-KeyMax, but only %q-%q", rngDesc.StartKey, newRngDesc.EndKey)
}
// Try to get values from both left and right of where the split happened.
gArgs := getArgs([]byte("c"))
if reply, pErr := client.SendWrapped(rg1(store), nil, &gArgs); pErr != nil {
t.Fatal(pErr)
} else if replyBytes, pErr := reply.(*roachpb.GetResponse).Value.GetBytes(); pErr != nil {
t.Fatal(pErr)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("x"))
if reply, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: newRng.RangeID,
}, &gArgs); pErr != nil {
t.Fatal(pErr)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Send out an increment request copied from above (same txn/sequence)
// which remains in the old range.
_, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Txn: &lTxn,
}, &lIncArgs)
if _, ok := pErr.GetDetail().(*roachpb.TransactionRetryError); !ok {
t.Fatalf("unexpected idempotency failure: %v", pErr)
}
// Send out the same increment copied from above (same txn/sequence), but
// now to the newly created range (which should hold that key).
_, pErr = client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: newRng.RangeID,
Txn: &rTxn,
}, &rIncArgs)
if _, ok := pErr.GetDetail().(*roachpb.TransactionRetryError); !ok {
t.Fatalf("unexpected idempotency failure: %v", pErr)
}
// Compare stats of split ranges to ensure they are non zero and
// exceed the original range when summed.
var left, right enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), store.Engine(), rangeID, &left); err != nil {
t.Fatal(err)
}
lKeyBytes, lValBytes := left.KeyBytes, left.ValBytes
if err := engine.MVCCGetRangeStats(context.Background(), store.Engine(), newRng.RangeID, &right); err != nil {
t.Fatal(err)
}
rKeyBytes, rValBytes := right.KeyBytes, right.ValBytes
if lKeyBytes == 0 || rKeyBytes == 0 {
t.Errorf("expected non-zero key bytes; got %d, %d", lKeyBytes, rKeyBytes)
}
if lValBytes == 0 || rValBytes == 0 {
t.Errorf("expected non-zero val bytes; got %d, %d", lValBytes, rValBytes)
}
if lKeyBytes+rKeyBytes <= keyBytes {
t.Errorf("left + right key bytes don't match; %d + %d <= %d", lKeyBytes, rKeyBytes, keyBytes)
}
if lValBytes+rValBytes <= valBytes {
t.Errorf("left + right val bytes don't match; %d + %d <= %d", lValBytes, rValBytes, valBytes)
}
}
// TestStoreRangeSplitStats starts by splitting the system keys from user-space
// keys and verifying that the user space side of the split (which is empty),
// has all zeros for stats. It then writes random data to the user space side,
// splits it halfway and verifies the two splits have stats exactly equaling
// the pre-split.
func TestStoreRangeSplitStats(t *testing.T) {
defer leaktest.AfterTest(t)()
defer config.TestingDisableTableSplits()()
store, stopper, manual := createTestStore(t)
defer stopper.Stop()
// Split the range after the last table data key.
keyPrefix := keys.MakeTablePrefix(keys.MaxReservedDescID + 1)
keyPrefix = keys.MakeRowSentinelKey(keyPrefix)
args := adminSplitArgs(roachpb.KeyMin, keyPrefix)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); pErr != nil {
t.Fatal(pErr)
}
// Verify empty range has empty stats.
rng := store.LookupReplica(keyPrefix, nil)
// NOTE that this value is expected to change over time, depending on what
// we store in the sys-local keyspace. Update it accordingly for this test.
if err := verifyRangeStats(store.Engine(), rng.RangeID, enginepb.MVCCStats{LastUpdateNanos: manual.UnixNano()}); err != nil {
t.Fatal(err)
}
// Write random data.
writeRandomDataToRange(t, store, rng.RangeID, keyPrefix)
// Get the range stats now that we have data.
snap := store.Engine().NewSnapshot()
defer snap.Close()
var ms enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), snap, rng.RangeID, &ms); err != nil {
t.Fatal(err)
}
if err := verifyRecomputedStats(snap, rng.Desc(), ms, manual.UnixNano()); err != nil {
t.Fatalf("failed to verify range's stats before split: %v", err)
}
if inMemMS := rng.GetMVCCStats(); inMemMS != ms {
t.Fatalf("in-memory and on-disk diverged:\n%+v\n!=\n%+v", inMemMS, ms)
}
manual.Increment(100)
// Split the range at approximate halfway point ("Z" in string "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz").
midKey := append([]byte(nil), keyPrefix...)
midKey = append(midKey, []byte("Z")...)
midKey = keys.MakeRowSentinelKey(midKey)
args = adminSplitArgs(keyPrefix, midKey)
if _, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: rng.RangeID,
}, &args); pErr != nil {
t.Fatal(pErr)
}
snap = store.Engine().NewSnapshot()
defer snap.Close()
var msLeft, msRight enginepb.MVCCStats
if err := engine.MVCCGetRangeStats(context.Background(), snap, rng.RangeID, &msLeft); err != nil {
t.Fatal(err)
}
rngRight := store.LookupReplica(midKey, nil)
if err := engine.MVCCGetRangeStats(context.Background(), snap, rngRight.RangeID, &msRight); err != nil {
t.Fatal(err)
}
// The stats should be exactly equal when added.
expMS := enginepb.MVCCStats{
LiveBytes: msLeft.LiveBytes + msRight.LiveBytes,
KeyBytes: msLeft.KeyBytes + msRight.KeyBytes,
ValBytes: msLeft.ValBytes + msRight.ValBytes,
IntentBytes: msLeft.IntentBytes + msRight.IntentBytes,
LiveCount: msLeft.LiveCount + msRight.LiveCount,
KeyCount: msLeft.KeyCount + msRight.KeyCount,
ValCount: msLeft.ValCount + msRight.ValCount,
IntentCount: msLeft.IntentCount + msRight.IntentCount,
}
ms.SysBytes, ms.SysCount = 0, 0
ms.LastUpdateNanos = 0
if expMS != ms {
t.Errorf("expected left plus right ranges to equal original, but\n %+v\n+\n %+v\n!=\n %+v", msLeft, msRight, ms)
}
// Stats should both have the new timestamp.
now := manual.UnixNano()
if lTs := msLeft.LastUpdateNanos; lTs != now {
t.Errorf("expected left range stats to have new timestamp, want %d, got %d", now, lTs)
}
if rTs := msRight.LastUpdateNanos; rTs != now {
t.Errorf("expected right range stats to have new timestamp, want %d, got %d", now, rTs)
}
// Stats should agree with recomputation.
if err := verifyRecomputedStats(snap, rng.Desc(), msLeft, now); err != nil {
t.Fatalf("failed to verify left range's stats after split: %v", err)
}
if err := verifyRecomputedStats(snap, rngRight.Desc(), msRight, now); err != nil {
t.Fatalf("failed to verify right range's stats after split: %v", err)
}
}
// TestRangeLookupUseReverse tests whether the results and the results count
// are correct when scanning in reverse order.
func TestRangeLookupUseReverse(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, stopper, _ := createTestStoreWithContext(t, sCtx)
defer stopper.Stop()
// Init test ranges:
// ["","a"), ["a","c"), ["c","e"), ["e","g") and ["g","\xff\xff").
splits := []roachpb.AdminSplitRequest{
adminSplitArgs(roachpb.Key("g"), roachpb.Key("g")),
adminSplitArgs(roachpb.Key("e"), roachpb.Key("e")),
adminSplitArgs(roachpb.Key("c"), roachpb.Key("c")),
adminSplitArgs(roachpb.Key("a"), roachpb.Key("a")),
}
for _, split := range splits {
_, pErr := client.SendWrapped(rg1(store), nil, &split)
if pErr != nil {
t.Fatalf("%q: split unexpected error: %s", split.SplitKey, pErr)
}
}
// Resolve the intents.
scanArgs := roachpb.ScanRequest{
Span: roachpb.Span{
Key: keys.RangeMetaKey(roachpb.RKeyMin.Next()),
EndKey: keys.RangeMetaKey(roachpb.RKeyMax),
},
}
util.SucceedsSoon(t, func() error {
_, pErr := client.SendWrapped(rg1(store), nil, &scanArgs)
return pErr.GoError()
})
revScanArgs := func(key []byte, maxResults int32) *roachpb.RangeLookupRequest {
return &roachpb.RangeLookupRequest{
Span: roachpb.Span{
Key: key,
},
MaxRanges: maxResults,
Reverse: true,
}
}
// Test cases.
testCases := []struct {
request *roachpb.RangeLookupRequest
expected []roachpb.RangeDescriptor
expectedPre []roachpb.RangeDescriptor
}{
// Test key in the middle of the range.
{
request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("f")), 2),
// ["e","g") and ["c","e").
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")},
},
},
// Test key in the end key of the range.
{
request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("g")), 3),
// ["e","g"), ["c","e") and ["a","c").
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")},
{StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("c")},
},
},
{
request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("e")), 2),
// ["c","e") and ["a","c").
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("c")},
},
},
// Test Meta2KeyMax.
{
request: revScanArgs(keys.Meta2KeyMax, 2),
// ["e","g") and ["g","\xff\xff")
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("g"), EndKey: roachpb.RKey("\xff\xff")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")},
},
},
// Test Meta1KeyMax.
{
request: revScanArgs(keys.Meta1KeyMax, 1),
// ["","a")
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKeyMin, EndKey: roachpb.RKey("a")},
},
},
}
for testIdx, test := range testCases {
resp, pErr := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
ReadConsistency: roachpb.INCONSISTENT,
}, test.request)
if pErr != nil {
t.Fatalf("%d: RangeLookup error: %s", testIdx, pErr)
}
rlReply := resp.(*roachpb.RangeLookupResponse)
// Checks the results count.
if int32(len(rlReply.Ranges))+int32(len(rlReply.PrefetchedRanges)) != test.request.MaxRanges {
t.Fatalf("%d: returned results count, expected %d,but got %d", testIdx, test.request.MaxRanges, len(rlReply.Ranges))
}
// Checks the range descriptors.
for _, rngSlice := range []struct {
expect, reply []roachpb.RangeDescriptor
}{
{test.expected, rlReply.Ranges},
{test.expectedPre, rlReply.PrefetchedRanges},
} {
for i, rng := range rngSlice.expect {
if !(rng.StartKey.Equal(rngSlice.reply[i].StartKey) && rng.EndKey.Equal(rngSlice.reply[i].EndKey)) {
t.Fatalf("%d: returned range is not correct, expected %v ,but got %v", testIdx, rng, rngSlice.reply[i])
}
}
}
}
}
// TestStoreSplitReadRace prevents regression of #3148. It begins a couple of
// read requests and lets them complete while a split is happening; the reads
// hit the second half of the split. If the split happens non-atomically with
// respect to the reads (and in particular their update of the timestamp
// cache), then some of them may not be reflected in the timestamp cache of the
// new range, in which case this test would fail.
func TestStoreSplitReadRace(t *testing.T) {
defer leaktest.AfterTest(t)()
defer config.TestingDisableTableSplits()()
splitKey := roachpb.Key("a")
key := func(i int) roachpb.Key {
splitCopy := append([]byte(nil), splitKey.Next()...)
return append(splitCopy, []byte(fmt.Sprintf("%03d", i))...)
}
getContinues := make(chan struct{})
var getStarted sync.WaitGroup
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.TestingCommandFilter =
func(filterArgs storagebase.FilterArgs) *roachpb.Error {
if et, ok := filterArgs.Req.(*roachpb.EndTransactionRequest); ok {
st := et.InternalCommitTrigger.GetSplitTrigger()
if st == nil || !st.UpdatedDesc.EndKey.Equal(splitKey) {
return nil
}
close(getContinues)
} else if filterArgs.Req.Method() == roachpb.Get &&
bytes.HasPrefix(filterArgs.Req.Header().Key, splitKey.Next()) {
getStarted.Done()
<-getContinues
}
return nil
}
store, stopper, _ := createTestStoreWithContext(t, &sCtx)
defer stopper.Stop()
now := store.Clock().Now()
var wg sync.WaitGroup
ts := func(i int) hlc.Timestamp {
return now.Add(0, int32(1000+i))
}
const num = 10
for i := 0; i < num; i++ {
wg.Add(1)
getStarted.Add(1)
go func(i int) {
defer wg.Done()
args := getArgs(key(i))
var h roachpb.Header
h.Timestamp = ts(i)
if _, pErr := client.SendWrappedWith(rg1(store), nil, h, &args); pErr != nil {
t.Fatal(pErr)
}
}(i)
}
getStarted.Wait()
wg.Add(1)
func() {
defer wg.Done()
args := adminSplitArgs(roachpb.KeyMin, splitKey)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); pErr != nil {
t.Fatal(pErr)
}
}()
wg.Wait()
for i := 0; i < num; i++ {
var h roachpb.Header
h.Timestamp = now
args := putArgs(key(i), []byte("foo"))
keyAddr, err := keys.Addr(args.Key)
if err != nil {
t.Fatal(err)
}
h.RangeID = store.LookupReplica(keyAddr, nil).RangeID
_, respH, pErr := storage.SendWrapped(store, context.Background(), h, &args)
if pErr != nil {
t.Fatal(pErr)
}
if respH.Timestamp.Less(ts(i)) {
t.Fatalf("%d: expected Put to be forced higher than %s by timestamp caches, but wrote at %s", i, ts(i), respH.Timestamp)
}
}
}
func TestRangeTransferLease(t *testing.T) {
defer leaktest.AfterTest(t)()
ctx := storage.TestStoreContext()
var filterMu syncutil.Mutex
var filter func(filterArgs storagebase.FilterArgs) *roachpb.Error
ctx.TestingKnobs.TestingCommandFilter =
func(filterArgs storagebase.FilterArgs) *roachpb.Error {
filterMu.Lock()
filterCopy := filter
filterMu.Unlock()
if filterCopy != nil {
return filterCopy(filterArgs)
}
return nil
}
var waitForTransferBlocked atomic.Value
waitForTransferBlocked.Store(false)
transferBlocked := make(chan struct{})
ctx.TestingKnobs.LeaseTransferBlockedOnExtensionEvent = func(
_ roachpb.ReplicaDescriptor) {
if waitForTransferBlocked.Load().(bool) {
transferBlocked <- struct{}{}
waitForTransferBlocked.Store(false)
}
}
mtc := &multiTestContext{}
mtc.storeContext = &ctx
mtc.Start(t, 2)
defer mtc.Stop()
// First, do a write; we'll use it to determine when the dust has settled.
leftKey := roachpb.Key("a")
incArgs := incrementArgs(leftKey, 1)
if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &incArgs); pErr != nil {
t.Fatal(pErr)
}
// Get the left range's ID.
rangeID := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil).RangeID
// Replicate the left range onto node 1.
mtc.replicateRange(rangeID, 1)
replica0 := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil)
replica1 := mtc.stores[1].LookupReplica(roachpb.RKey("a"), nil)
gArgs := getArgs(leftKey)
replica0Desc, err := replica0.GetReplicaDescriptor()
if err != nil {
t.Fatal(err)
}
// Check that replica0 can serve reads OK.
if _, pErr := client.SendWrappedWith(
mtc.senders[0], nil, roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil {
t.Fatal(pErr)
}
{
// Transferring the lease to ourself should be a no-op.
origLeasePtr, _ := replica0.GetLease()
origLease := *origLeasePtr
if err := replica0.AdminTransferLease(replica0Desc.StoreID); err != nil {
t.Fatal(err)
}
newLeasePtr, _ := replica0.GetLease()
if origLeasePtr != newLeasePtr || origLease != *newLeasePtr {
t.Fatalf("expected %+v, but found %+v", origLeasePtr, newLeasePtr)
}
}
{
// An invalid target should result in an error.
const expected = "unable to find store .* in range"
if err := replica0.AdminTransferLease(1000); !testutils.IsError(err, expected) {
t.Fatalf("expected %s, but found %v", expected, err)
}
}
// Move the lease to store 1.
var newHolderDesc roachpb.ReplicaDescriptor
util.SucceedsSoon(t, func() error {
var err error
newHolderDesc, err = replica1.GetReplicaDescriptor()
return err
})
if err := replica0.AdminTransferLease(newHolderDesc.StoreID); err != nil {
t.Fatal(err)
}
// Check that replica0 doesn't serve reads any more.
replica0Desc, err = replica0.GetReplicaDescriptor()
if err != nil {
t.Fatal(err)
}
_, pErr := client.SendWrappedWith(
mtc.senders[0], nil, roachpb.Header{Replica: replica0Desc}, &gArgs)
nlhe, ok := pErr.GetDetail().(*roachpb.NotLeaseHolderError)
if !ok {
t.Fatalf("expected %T, got %s", &roachpb.NotLeaseHolderError{}, pErr)
}
if *(nlhe.LeaseHolder) != newHolderDesc {
t.Fatalf("expected lease holder %+v, got %+v",
newHolderDesc, nlhe.LeaseHolder)
}
// Check that replica1 now has the lease (or gets it soon).
util.SucceedsSoon(t, func() error {
if _, pErr := client.SendWrappedWith(
mtc.senders[1], nil, roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil {
return pErr.GoError()
}
return nil
})
replica1Lease, _ := replica1.GetLease()
// Verify the timestamp cache low water. Because we executed a transfer lease
// request, the low water should be set to the new lease start time which is
// less than the previous lease's expiration time.
if lowWater := replica1.GetTimestampCacheLowWater(); lowWater != replica1Lease.Start {
t.Fatalf("expected timestamp cache low water %s, but found %s",
replica1Lease.Start, lowWater)
}
// Make replica1 extend its lease and transfer the lease immediately after
// that. Test that the transfer still happens (it'll wait until the extension
// is done).
extensionSem := make(chan struct{})
filterMu.Lock()
filter = func(filterArgs storagebase.FilterArgs) *roachpb.Error {
if filterArgs.Sid != mtc.stores[1].Ident.StoreID {
return nil
}
llReq, ok := filterArgs.Req.(*roachpb.RequestLeaseRequest)
if !ok {
return nil
}
if llReq.Lease.Replica == newHolderDesc {
// Notify the main thread that the extension is in progress and wait for
// the signal to proceed.
filterMu.Lock()
filter = nil
filterMu.Unlock()
extensionSem <- struct{}{}
<-extensionSem
}
return nil
}
filterMu.Unlock()
// Initiate an extension.
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
shouldRenewTS := replica1Lease.StartStasis.Add(-1, 0)
mtc.manualClock.Set(shouldRenewTS.WallTime + 1)
if _, pErr := client.SendWrappedWith(
mtc.senders[1], nil,
roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil {
panic(pErr)
}
}()
<-extensionSem
waitForTransferBlocked.Store(true)
// Initiate a transfer.
wg.Add(1)
go func() {
defer wg.Done()
// Transfer back from replica1 to replica0.
if err := replica1.AdminTransferLease(replica0Desc.StoreID); err != nil {
panic(err)
}
}()
// Wait for the transfer to be blocked by the extension.
<-transferBlocked
// Now unblock the extension.
extensionSem <- struct{}{}
// Check that the transfer to replica1 eventually happens.
util.SucceedsSoon(t, func() error {
if _, pErr := client.SendWrappedWith(
mtc.senders[0], nil,
roachpb.Header{Replica: replica0Desc}, &gArgs); pErr != nil {
return pErr.GoError()
}
return nil
})
filterMu.Lock()
filter = nil
filterMu.Unlock()
wg.Wait()
}
// TestMultiRangeScanReverseScanInconsistent verifies that a Scan/ReverseScan
// across ranges that doesn't require read consistency will set a timestamp
// using the clock local to the distributed sender.
func TestMultiRangeScanReverseScanInconsistent(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
db := setupMultipleRanges(t, s, "b")
// Write keys "a" and "b", the latter of which is the first key in the
// second range.
keys := [2]string{"a", "b"}
ts := [2]hlc.Timestamp{}
for i, key := range keys {
b := &client.Batch{}
b.Put(key, "value")
if err := db.Run(b); err != nil {
t.Fatal(err)
}
ts[i] = s.Clock().Now()
log.Infof("%d: %s %d", i, key, ts[i])
if i == 0 {
util.SucceedsSoon(t, func() error {
// Enforce that when we write the second key, it's written
// with a strictly higher timestamp. We're dropping logical
// ticks and the clock may just have been pushed into the
// future, so that's necessary. See #3122.
if ts[0].WallTime >= s.Clock().Now().WallTime {
return errors.New("time stands still")
}
return nil
})
}
}
// Do an inconsistent Scan/ReverseScan from a new DistSender and verify
// it does the read at its local clock and doesn't receive an
// OpRequiresTxnError. We set the local clock to the timestamp of
// just above the first key to verify it's used to read only key "a".
for i, request := range []roachpb.Request{
roachpb.NewScan(roachpb.Key("a"), roachpb.Key("c"), 0),
roachpb.NewReverseScan(roachpb.Key("a"), roachpb.Key("c"), 0),
} {
manual := hlc.NewManualClock(ts[0].WallTime + 1)
clock := hlc.NewClock(manual.UnixNano)
ds := kv.NewDistSender(&kv.DistSenderContext{Clock: clock, RPCContext: s.RPCContext()}, s.(*server.TestServer).Gossip())
reply, err := client.SendWrappedWith(ds, nil, roachpb.Header{
ReadConsistency: roachpb.INCONSISTENT,
}, request)
if err != nil {
t.Fatal(err)
}
var rows []roachpb.KeyValue
switch r := reply.(type) {
case *roachpb.ScanResponse:
rows = r.Rows
case *roachpb.ReverseScanResponse:
rows = r.Rows
default:
t.Fatalf("unexpected response %T: %v", reply, reply)
}
if l := len(rows); l != 1 {
t.Fatalf("%d: expected 1 row; got %d\n%s", i, l, rows)
}
if key := string(rows[0].Key); keys[0] != key {
t.Errorf("expected key %q; got %q", keys[0], key)
}
}
}
// Test that a lease extension (a RequestLeaseRequest that doesn't change the
// lease holder) is not blocked by ongoing reads.
// Note that lease transfers are blocked by reads through their
// PostCommitTrigger.noConcurrentReads.
func TestLeaseExtensionNotBlockedByRead(t *testing.T) {
defer leaktest.AfterTest(t)()
readBlocked := make(chan struct{})
cmdFilter := func(fArgs storagebase.FilterArgs) *roachpb.Error {
if fArgs.Hdr.UserPriority == 42 {
// Signal that the read is blocked.
readBlocked <- struct{}{}
// Wait for read to be unblocked.
<-readBlocked
}
return nil
}
srv, _, _ := serverutils.StartServer(t,
base.TestServerArgs{
Knobs: base.TestingKnobs{
Store: &storage.StoreTestingKnobs{
TestingCommandFilter: cmdFilter,
},
},
})
s := srv.(*server.TestServer)
defer s.Stopper().Stop()
// Start a read and wait for it to block.
key := roachpb.Key("a")
errChan := make(chan error)
go func() {
getReq := roachpb.GetRequest{
Span: roachpb.Span{
Key: key,
},
}
if _, pErr := client.SendWrappedWith(s.GetDistSender(), nil,
roachpb.Header{UserPriority: 42},
&getReq); pErr != nil {
errChan <- pErr.GoError()
}
}()
select {
case err := <-errChan:
t.Fatal(err)
case <-readBlocked:
// Send the lease request.
// We change the key slightly, otherwise the lease request will be blocked
// by the read through the command queue.
// TODO(andrei): don't change the key anymore once lease requests don't go
// through the command queue any more.
leaseHdrKey := roachpb.Key(append(key, 0x00))
rKey, err := keys.Addr(leaseHdrKey)
if err != nil {
t.Fatal(err)
}
_, repDesc, err := s.Stores().LookupReplica(rKey, nil)
if err != nil {
t.Fatal(err)
}
leaseReq := roachpb.RequestLeaseRequest{
Span: roachpb.Span{
Key: leaseHdrKey,
},
Lease: roachpb.Lease{
Start: s.Clock().Now(),
StartStasis: s.Clock().Now().Add(time.Second.Nanoseconds(), 0),
Expiration: s.Clock().Now().Add(2*time.Second.Nanoseconds(), 0),
Replica: repDesc,
},
}
if _, pErr := client.SendWrapped(s.GetDistSender(), nil, &leaseReq); pErr != nil {
t.Fatal(pErr)
}
// Unblock the read.
readBlocked <- struct{}{}
}
}
// TestRequestToUninitializedRange tests the behavior when a request
// is sent to a node which should be a replica of the correct range
// but has not yet received its initial snapshot. This would
// previously panic due to a malformed error response from the server,
// as seen in https://github.com/cockroachdb/cockroach/issues/6027.
//
// Prior to the other changes in the commit that introduced it, this
// test would reliable trigger the panic from #6027. However, it
// relies on some hacky tricks to both trigger the panic and shut down
// cleanly. If this test needs a lot of maintenance in the future we
// should be willing to get rid of it.
func TestRequestToUninitializedRange(t *testing.T) {
defer leaktest.AfterTest(t)()
srv, _, _ := serverutils.StartServer(t, base.TestServerArgs{StoresPerNode: 2})
defer srv.Stopper().Stop()
s := srv.(*server.TestServer)
// Choose a range ID that is much larger than any that would be
// created by initial splits.
const rangeID = roachpb.RangeID(1000)
// Set up a range with replicas on two stores of the same node. This
// ensures that the DistSender will consider both replicas healthy
// and will try to talk to both (so we can get a non-retryable error
// from the second store).
replica1 := roachpb.ReplicaDescriptor{
NodeID: 1,
StoreID: 1,
ReplicaID: 1,
}
replica2 := roachpb.ReplicaDescriptor{
NodeID: 1,
StoreID: 2,
ReplicaID: 2,
}
// HACK: remove the second store from the node to generate a
// non-retryable error when we try to talk to it.
store2, err := s.Stores().GetStore(2)
if err != nil {
t.Fatal(err)
}
s.Stores().RemoveStore(store2)
// Create the uninitialized range by sending an isolated raft
// message to the first store.
conn, err := s.RPCContext().GRPCDial(s.ServingAddr())
if err != nil {
t.Fatal(err)
}
raftClient := storage.NewMultiRaftClient(conn)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stream, err := raftClient.RaftMessage(ctx)
if err != nil {
t.Fatal(err)
}
msg := storage.RaftMessageRequest{
RangeID: rangeID,
ToReplica: replica1,
FromReplica: replica2,
Message: raftpb.Message{
Type: raftpb.MsgApp,
To: 1,
},
}
if err := stream.Send(&msg); err != nil {
t.Fatal(err)
}
// Make sure the replica was created.
store1, err := s.Stores().GetStore(1)
if err != nil {
t.Fatal(err)
}
util.SucceedsSoon(t, func() error {
if replica, err := store1.GetReplica(rangeID); err != nil {
return errors.Errorf("failed to look up replica: %s", err)
} else if replica.IsInitialized() {
return errors.Errorf("expected replica to be uninitialized")
}
return nil
})
// Create our own DistSender so we can force some requests to the
// bogus range. The DistSender needs to be in scope for its own
// MockRangeDescriptorDB closure.
var sender *kv.DistSender
sender = kv.NewDistSender(&kv.DistSenderContext{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RangeDescriptorDB: kv.MockRangeDescriptorDB(
func(key roachpb.RKey, considerIntents, useReverseScan bool,
) ([]roachpb.RangeDescriptor, []roachpb.RangeDescriptor, *roachpb.Error) {
if key.Equal(roachpb.RKeyMin) {
// Pass through requests for the first range to the real sender.
desc, err := sender.FirstRange()
if err != nil {
return nil, nil, roachpb.NewError(err)
}
return []roachpb.RangeDescriptor{*desc}, nil, nil
}
return []roachpb.RangeDescriptor{{
RangeID: rangeID,
StartKey: roachpb.RKey(keys.Meta2Prefix),
EndKey: roachpb.RKeyMax,
Replicas: []roachpb.ReplicaDescriptor{replica1, replica2},
}}, nil, nil
}),
}, s.Gossip())
// Only inconsistent reads triggered the panic in #6027.
hdr := roachpb.Header{
ReadConsistency: roachpb.INCONSISTENT,
}
req := roachpb.NewGet(roachpb.Key("asdf"))
// Repeat the test a few times: due to the randomization between the
// two replicas, each attempt only had a 50% chance of triggering
// the panic.
for i := 0; i < 5; i++ {
_, pErr := client.SendWrappedWith(sender, context.Background(), hdr, req)
// Each attempt fails with "store 2 not found" because that is the
// non-retryable error.
if !testutils.IsPError(pErr, "store 2 not found") {
t.Fatal(pErr)
}
}
}