// TestTxnMultipleCoord checks that a coordinator uses the Writing flag to
// enforce that only one coordinator can be used for transactional writes.
func TestTxnMultipleCoord(t *testing.T) {
defer leaktest.AfterTest(t)()
s, sender := createTestDB(t)
defer s.Stop()
testCases := []struct {
args roachpb.Request
writing bool
ok bool
}{
{roachpb.NewGet(roachpb.Key("a")), true, false},
{roachpb.NewGet(roachpb.Key("a")), false, true},
{roachpb.NewPut(roachpb.Key("a"), roachpb.Value{}), false, false}, // transactional write before begin
{roachpb.NewPut(roachpb.Key("a"), roachpb.Value{}), true, false}, // must have switched coordinators
}
for i, tc := range testCases {
txn := roachpb.NewTransaction("test", roachpb.Key("a"), 1, enginepb.SERIALIZABLE,
s.Clock.Now(), s.Clock.MaxOffset().Nanoseconds())
txn.Writing = tc.writing
reply, pErr := client.SendWrappedWith(context.Background(), sender, roachpb.Header{
Txn: txn,
}, tc.args)
if pErr == nil != tc.ok {
t.Errorf("%d: %T (writing=%t): success_expected=%t, but got: %v",
i, tc.args, tc.writing, tc.ok, pErr)
}
if pErr != nil {
continue
}
txn = reply.Header().Txn
// The transaction should come back rw if it started rw or if we just
// wrote.
isWrite := roachpb.IsTransactionWrite(tc.args)
if (tc.writing || isWrite) != txn.Writing {
t.Errorf("%d: unexpected writing state: %s", i, txn)
}
if !isWrite {
continue
}
// Abort for clean shutdown.
if _, pErr := client.SendWrappedWith(context.Background(), sender, roachpb.Header{
Txn: txn,
}, &roachpb.EndTransactionRequest{
Commit: false,
}); pErr != nil {
t.Fatal(pErr)
}
}
}
// TestRejectFutureCommand verifies that lease holders reject commands that
// would cause a large time jump.
func TestRejectFutureCommand(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, 100*time.Millisecond)
mtc := &multiTestContext{clock: clock}
mtc.Start(t, 1)
defer mtc.Stop()
ts1 := clock.Now()
key := roachpb.Key("a")
incArgs := incrementArgs(key, 5)
// Commands with a future timestamp that is within the MaxOffset
// bound will be accepted and will cause the clock to advance.
const numCmds = 3
clockOffset := clock.MaxOffset() / numCmds
for i := int64(1); i <= numCmds; i++ {
ts := ts1.Add(i*clockOffset.Nanoseconds(), 0)
if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err != nil {
t.Fatal(err)
}
}
ts2 := clock.Now()
if expAdvance, advance := ts2.GoTime().Sub(ts1.GoTime()), numCmds*clockOffset; advance != expAdvance {
t.Fatalf("expected clock to advance %s; got %s", expAdvance, advance)
}
// Once the accumulated offset reaches MaxOffset, commands will be rejected.
_, pErr := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts1.Add(clock.MaxOffset().Nanoseconds()+1, 0)}, &incArgs)
if !testutils.IsPError(pErr, "rejecting command with timestamp in the future") {
t.Fatalf("unexpected error %v", pErr)
}
// The clock did not advance and the final command was not executed.
ts3 := clock.Now()
if advance := ts3.GoTime().Sub(ts2.GoTime()); advance != 0 {
t.Fatalf("expected clock not to advance, but it advanced by %s", advance)
}
val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], key, ts3, true, nil)
if err != nil {
t.Fatal(err)
}
if a, e := mustGetInt(val), incArgs.Increment*numCmds; a != e {
t.Errorf("expected %d, got %d", e, a)
}
}
// process synchronously invokes admin split for each proposed split key.
func (sq *splitQueue) process(
ctx context.Context, now hlc.Timestamp, r *Replica, sysCfg config.SystemConfig,
) error {
// First handle case of splitting due to zone config maps.
desc := r.Desc()
splitKeys := sysCfg.ComputeSplitKeys(desc.StartKey, desc.EndKey)
if len(splitKeys) > 0 {
log.Infof(ctx, "splitting at keys %v", splitKeys)
for _, splitKey := range splitKeys {
if err := sq.db.AdminSplit(ctx, splitKey.AsRawKey()); err != nil {
return errors.Errorf("unable to split %s at key %q: %s", r, splitKey, err)
}
}
return nil
}
// Next handle case of splitting due to size.
zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
if err != nil {
return err
}
size := r.GetMVCCStats().Total()
// FIXME: why is this implementation not the same as the one above?
if float64(size)/float64(zone.RangeMaxBytes) > 1 {
log.Infof(ctx, "splitting size=%d max=%d", size, zone.RangeMaxBytes)
if _, pErr := client.SendWrappedWith(ctx, r, roachpb.Header{
Timestamp: now,
}, &roachpb.AdminSplitRequest{
Span: roachpb.Span{Key: desc.StartKey.AsRawKey()},
}); pErr != nil {
return pErr.GoError()
}
}
return nil
}
func TestComputeStatsForKeySpan(t *testing.T) {
defer leaktest.AfterTest(t)()
mtc := &multiTestContext{}
defer mtc.Stop()
mtc.Start(t, 3)
// Create a number of ranges using splits.
splitKeys := []string{"a", "c", "e", "g", "i"}
for _, k := range splitKeys {
key := []byte(k)
repl := mtc.stores[0].LookupReplica(key, roachpb.RKeyMin)
args := adminSplitArgs(key, key)
header := roachpb.Header{
RangeID: repl.RangeID,
}
if _, err := client.SendWrappedWith(context.Background(), mtc.stores[0], header, args); err != nil {
t.Fatal(err)
}
}
// Wait for splits to finish.
testutils.SucceedsSoon(t, func() error {
repl := mtc.stores[0].LookupReplica(roachpb.RKey("z"), nil)
if actualRSpan := repl.Desc().RSpan(); !actualRSpan.Key.Equal(roachpb.RKey("i")) {
return errors.Errorf("expected range %s to begin at key 'i'", repl)
}
return nil
})
// Create some keys across the ranges.
incKeys := []string{"b", "bb", "bbb", "d", "dd", "h"}
for _, k := range incKeys {
if _, err := mtc.dbs[0].Inc(context.TODO(), []byte(k), 5); err != nil {
t.Fatal(err)
}
}
// Verify stats across different spans.
for _, tcase := range []struct {
startKey string
endKey string
expectedRanges int
expectedKeys int64
}{
{"a", "i", 4, 6},
{"a", "c", 1, 3},
{"b", "e", 2, 5},
{"e", "i", 2, 1},
} {
start, end := tcase.startKey, tcase.endKey
stats, count := mtc.stores[0].ComputeStatsForKeySpan(
roachpb.RKey(start), roachpb.RKey(end))
if a, e := count, tcase.expectedRanges; a != e {
t.Errorf("Expected %d ranges in span [%s - %s], found %d", e, start, end, a)
}
if a, e := stats.LiveCount, tcase.expectedKeys; a != e {
t.Errorf("Expected %d keys in span [%s - %s], found %d", e, start, end, a)
}
}
}
// TestRejectFutureCommand verifies that lease holders reject commands that
// would cause a large time jump.
func TestRejectFutureCommand(t *testing.T) {
defer leaktest.AfterTest(t)()
const maxOffset = 100 * time.Millisecond
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(maxOffset)
mtc := &multiTestContext{clock: clock}
mtc.Start(t, 1)
defer mtc.Stop()
startTime := manual.UnixNano()
// Commands with a future timestamp that is within the MaxOffset
// bound will be accepted and will cause the clock to advance.
for i := int64(0); i < 3; i++ {
incArgs := incrementArgs([]byte("a"), 5)
ts := hlc.ZeroTimestamp.Add(startTime+((i+1)*30)*int64(time.Millisecond), 0)
if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err != nil {
t.Fatal(err)
}
}
if now := clock.Now(); now.WallTime != int64(90*time.Millisecond) {
t.Fatalf("expected clock to advance to 90ms; got %s", now)
}
// Once the accumulated offset reaches MaxOffset, commands will be rejected.
incArgs := incrementArgs([]byte("a"), 11)
ts := hlc.ZeroTimestamp.Add(int64((time.Duration(startTime)+maxOffset+1)*time.Millisecond), 0)
if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err == nil {
t.Fatalf("expected clock offset error but got nil")
}
// The clock remained at 90ms and the final command was not executed.
if now := clock.Now(); now.WallTime != int64(90*time.Millisecond) {
t.Errorf("expected clock to stay at 90ms; got %s", now)
}
val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], roachpb.Key("a"), clock.Now(), true, nil)
if err != nil {
t.Fatal(err)
}
if v := mustGetInt(val); v != 15 {
t.Errorf("expected 15, got %v", v)
}
}
// TestRangeCommandClockUpdate verifies that followers update their
// clocks when executing a command, even if the lease holder's clock is far
// in the future.
func TestRangeCommandClockUpdate(t *testing.T) {
defer leaktest.AfterTest(t)()
const numNodes = 3
var manuals []*hlc.ManualClock
var clocks []*hlc.Clock
for i := 0; i < numNodes; i++ {
manuals = append(manuals, hlc.NewManualClock(1))
clocks = append(clocks, hlc.NewClock(manuals[i].UnixNano))
clocks[i].SetMaxOffset(100 * time.Millisecond)
}
mtc := &multiTestContext{clocks: clocks}
mtc.Start(t, numNodes)
defer mtc.Stop()
mtc.replicateRange(1, 1, 2)
// Advance the lease holder's clock ahead of the followers (by more than
// MaxOffset but less than the range lease) and execute a command.
manuals[0].Increment(int64(500 * time.Millisecond))
incArgs := incrementArgs([]byte("a"), 5)
ts := clocks[0].Now()
if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err != nil {
t.Fatal(err)
}
// Wait for that command to execute on all the followers.
util.SucceedsSoon(t, func() error {
values := []int64{}
for _, eng := range mtc.engines {
val, _, err := engine.MVCCGet(context.Background(), eng, roachpb.Key("a"), clocks[0].Now(), true, nil)
if err != nil {
return err
}
values = append(values, mustGetInt(val))
}
if !reflect.DeepEqual(values, []int64{5, 5, 5}) {
return errors.Errorf("expected (5, 5, 5), got %v", values)
}
return nil
})
// Verify that all the followers have accepted the clock update from
// node 0 even though it comes from outside the usual max offset.
now := clocks[0].Now()
for i, clock := range clocks {
// Only compare the WallTimes: it's normal for clock 0 to be a few logical ticks ahead.
if clock.Now().WallTime < now.WallTime {
t.Errorf("clock %d is behind clock 0: %s vs %s", i, clock.Now(), now)
}
}
}
// getTxn fetches the requested key and returns the transaction info.
func getTxn(coord *TxnCoordSender, txn *roachpb.Transaction) (*roachpb.Transaction, *roachpb.Error) {
hb := &roachpb.HeartbeatTxnRequest{
Span: roachpb.Span{
Key: txn.Key,
},
}
reply, pErr := client.SendWrappedWith(context.Background(), coord, roachpb.Header{
Txn: txn,
}, hb)
if pErr != nil {
return nil, pErr
}
return reply.(*roachpb.HeartbeatTxnResponse).Txn, nil
}
// FindRangeLease is similar to FindRangeLeaseHolder but returns a Lease proto
// without verifying if the lease is still active. Instead, it returns a time-
// stamp taken off the queried node's clock.
func (tc *TestCluster) FindRangeLease(
rangeDesc *roachpb.RangeDescriptor, hint *ReplicationTarget,
) (_ *roachpb.Lease, now hlc.Timestamp, _ error) {
if hint != nil {
var ok bool
if _, ok = rangeDesc.GetReplicaDescriptor(hint.StoreID); !ok {
return nil, hlc.ZeroTimestamp, errors.Errorf(
"bad hint: %+v; store doesn't have a replica of the range", hint)
}
} else {
hint = &ReplicationTarget{
NodeID: rangeDesc.Replicas[0].NodeID,
StoreID: rangeDesc.Replicas[0].StoreID}
}
// Find the server indicated by the hint and send a LeaseInfoRequest through
// it.
var hintServer *server.TestServer
for _, s := range tc.Servers {
if s.GetNode().Descriptor.NodeID == hint.NodeID {
hintServer = s
break
}
}
if hintServer == nil {
return nil, hlc.ZeroTimestamp, errors.Errorf("bad hint: %+v; no such node", hint)
}
leaseReq := roachpb.LeaseInfoRequest{
Span: roachpb.Span{
Key: rangeDesc.StartKey.AsRawKey(),
},
}
leaseResp, pErr := client.SendWrappedWith(
context.TODO(),
hintServer.DB().GetSender(),
roachpb.Header{
// INCONSISTENT read, since we want to make sure that the node used to
// send this is the one that processes the command, for the hint to
// matter.
ReadConsistency: roachpb.INCONSISTENT,
},
&leaseReq)
if pErr != nil {
return nil, hlc.ZeroTimestamp, pErr.GoError()
}
return leaseResp.(*roachpb.LeaseInfoResponse).Lease, hintServer.Clock().Now(), nil
}
func fillTestRange(t testing.TB, rep *Replica, size int64) {
src := rand.New(rand.NewSource(0))
for i := int64(0); i < size/int64(keySize+valSize); i++ {
key := keys.MakeRowSentinelKey(randutil.RandBytes(src, keySize))
val := randutil.RandBytes(src, valSize)
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(context.Background(), rep, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
t.Fatal(pErr)
}
}
rep.mu.Lock()
after := rep.mu.state.Stats.Total()
rep.mu.Unlock()
if after < size {
t.Fatalf("range not full after filling: wrote %d, but range at %d", size, after)
}
}
// LeaseInfo runs a LeaseInfoRequest using the specified server.
func LeaseInfo(
t *testing.T,
db *client.DB,
rangeDesc roachpb.RangeDescriptor,
readConsistency roachpb.ReadConsistencyType,
) roachpb.LeaseInfoResponse {
leaseInfoReq := &roachpb.LeaseInfoRequest{
Span: roachpb.Span{
Key: rangeDesc.StartKey.AsRawKey(),
},
}
reply, pErr := client.SendWrappedWith(context.Background(), db.GetSender(), roachpb.Header{
ReadConsistency: readConsistency,
}, leaseInfoReq)
if pErr != nil {
t.Fatal(pErr)
}
return *(reply.(*roachpb.LeaseInfoResponse))
}
// Test that an error encountered by a read-only "NonKV" command is not
// swallowed, and doesn't otherwise cause a panic.
// We had a bug cause by the fact that errors for these commands aren't passed
// through the epilogue returned by replica.beginCommands() and were getting
// swallowed.
func TestErrorHandlingForNonKVCommand(t *testing.T) {
defer leaktest.AfterTest(t)()
cmdFilter := func(fArgs storagebase.FilterArgs) *roachpb.Error {
if fArgs.Hdr.UserPriority == 42 {
return roachpb.NewErrorf("injected error")
}
return nil
}
srv, _, _ := serverutils.StartServer(t,
base.TestServerArgs{
Knobs: base.TestingKnobs{
Store: &storage.StoreTestingKnobs{
TestingCommandFilter: cmdFilter,
},
},
})
s := srv.(*server.TestServer)
defer s.Stopper().Stop()
// Send the lease request.
key := roachpb.Key("a")
leaseReq := roachpb.LeaseInfoRequest{
Span: roachpb.Span{
Key: key,
},
}
_, pErr := client.SendWrappedWith(
context.Background(),
s.DistSender(),
roachpb.Header{UserPriority: 42},
&leaseReq,
)
if !testutils.IsPError(pErr, "injected error") {
t.Fatalf("expected error %q, got: %s", "injected error", pErr)
}
}
// Test that a lease extension (a RequestLeaseRequest that doesn't change the
// lease holder) is not blocked by ongoing reads.
// The test relies on two things:
// 1) Lease extensions, unlike lease transfers, are not blocked by reads through their
// PostCommitTrigger.noConcurrentReads.
// 2) Requests with the non-KV flag, such as RequestLeaseRequest, do not
// go through the command queue.
func TestLeaseExtensionNotBlockedByRead(t *testing.T) {
defer leaktest.AfterTest(t)()
readBlocked := make(chan struct{})
cmdFilter := func(fArgs storagebase.FilterArgs) *roachpb.Error {
if fArgs.Hdr.UserPriority == 42 {
// Signal that the read is blocked.
readBlocked <- struct{}{}
// Wait for read to be unblocked.
<-readBlocked
}
return nil
}
srv, _, _ := serverutils.StartServer(t,
base.TestServerArgs{
Knobs: base.TestingKnobs{
Store: &storage.StoreTestingKnobs{
TestingCommandFilter: cmdFilter,
},
},
})
s := srv.(*server.TestServer)
defer s.Stopper().Stop()
// Start a read and wait for it to block.
key := roachpb.Key("a")
errChan := make(chan error)
go func() {
getReq := roachpb.GetRequest{
Span: roachpb.Span{
Key: key,
},
}
if _, pErr := client.SendWrappedWith(context.Background(), s.DistSender(),
roachpb.Header{UserPriority: 42},
&getReq); pErr != nil {
errChan <- pErr.GoError()
}
}()
select {
case err := <-errChan:
t.Fatal(err)
case <-readBlocked:
// Send the lease request.
rKey, err := keys.Addr(key)
if err != nil {
t.Fatal(err)
}
_, repDesc, err := s.Stores().LookupReplica(rKey, nil)
if err != nil {
t.Fatal(err)
}
leaseReq := roachpb.RequestLeaseRequest{
Span: roachpb.Span{
Key: key,
},
Lease: roachpb.Lease{
Start: s.Clock().Now(),
StartStasis: s.Clock().Now().Add(time.Second.Nanoseconds(), 0),
Expiration: s.Clock().Now().Add(2*time.Second.Nanoseconds(), 0),
Replica: repDesc,
},
}
if _, pErr := client.SendWrapped(context.Background(), s.DistSender(), &leaseReq); pErr != nil {
t.Fatal(pErr)
}
// Unblock the read.
readBlocked <- struct{}{}
}
}
// TestTimeSeriesMaintenanceQueue verifies shouldQueue and process method
// pass the correct data to the store's TimeSeriesData
func TestTimeSeriesMaintenanceQueue(t *testing.T) {
defer leaktest.AfterTest(t)()
model := &modelTimeSeriesDataStore{
t: t,
pruneSeenStartKeys: make(map[string]struct{}),
pruneSeenEndKeys: make(map[string]struct{}),
}
manual := hlc.NewManualClock(1)
cfg := storage.TestStoreConfig(hlc.NewClock(manual.UnixNano, time.Nanosecond))
cfg.TimeSeriesDataStore = model
cfg.TestingKnobs.DisableScanner = true
cfg.TestingKnobs.DisableSplitQueue = true
stopper := stop.NewStopper()
defer stopper.Stop()
store := createTestStoreWithConfig(t, stopper, cfg)
// Generate several splits.
splitKeys := []roachpb.Key{roachpb.Key("c"), roachpb.Key("b"), roachpb.Key("a")}
for _, k := range splitKeys {
repl := store.LookupReplica(roachpb.RKey(k), nil)
args := adminSplitArgs(k, k)
if _, pErr := client.SendWrappedWith(context.Background(), store, roachpb.Header{
RangeID: repl.RangeID,
}, args); pErr != nil {
t.Fatal(pErr)
}
}
// Generate a list of start/end keys the model should have been passed by
// the queue. This consists of all split keys, with KeyMin as an additional
// start and KeyMax as an additional end.
expectedStartKeys := make(map[string]struct{})
expectedEndKeys := make(map[string]struct{})
expectedStartKeys[roachpb.KeyMin.String()] = struct{}{}
expectedEndKeys[roachpb.KeyMax.String()] = struct{}{}
for _, expected := range splitKeys {
expectedStartKeys[expected.String()] = struct{}{}
expectedEndKeys[expected.String()] = struct{}{}
}
// Wait for splits to complete and system config to be available.
util.SucceedsSoon(t, func() error {
if a, e := store.ReplicaCount(), len(expectedEndKeys); a != e {
return fmt.Errorf("expected %d replicas in store; found %d", a, e)
}
if _, ok := store.Gossip().GetSystemConfig(); !ok {
return fmt.Errorf("system config not yet available")
}
return nil
})
// Force replica scan to run, which will populate the model.
now := store.Clock().Now()
store.ForceTimeSeriesMaintenanceQueueProcess()
// Wait for processing to complete.
util.SucceedsSoon(t, func() error {
model.Lock()
defer model.Unlock()
if a, e := model.containsCalled, len(expectedStartKeys); a != e {
return fmt.Errorf("ContainsTimeSeries called %d times; expected %d", a, e)
}
if a, e := model.pruneCalled, len(expectedStartKeys); a != e {
return fmt.Errorf("PruneTimeSeries called %d times; expected %d", a, e)
}
return nil
})
model.Lock()
if a, e := model.pruneSeenStartKeys, expectedStartKeys; !reflect.DeepEqual(a, e) {
t.Errorf("start keys seen by PruneTimeSeries did not match expectation: %s", pretty.Diff(a, e))
}
if a, e := model.pruneSeenEndKeys, expectedEndKeys; !reflect.DeepEqual(a, e) {
t.Errorf("end keys seen by PruneTimeSeries did not match expectation: %s", pretty.Diff(a, e))
}
model.Unlock()
util.SucceedsSoon(t, func() error {
keys := []roachpb.RKey{roachpb.RKeyMin}
for _, k := range splitKeys {
keys = append(keys, roachpb.RKey(k))
}
for _, key := range keys {
repl := store.LookupReplica(key, nil)
ts, err := repl.GetQueueLastProcessed(context.TODO(), "timeSeriesMaintenance")
if err != nil {
return err
}
if ts.Less(now) {
return errors.Errorf("expected last processed %s > %s", ts, now)
}
}
return nil
})
// Force replica scan to run. But because we haven't moved the
// clock forward, no pruning will take place on second invocation.
store.ForceTimeSeriesMaintenanceQueueProcess()
model.Lock()
if a, e := model.containsCalled, len(expectedStartKeys); a != e {
t.Errorf("ContainsTimeSeries called %d times; expected %d", a, e)
}
if a, e := model.pruneCalled, len(expectedStartKeys); a != e {
t.Errorf("PruneTimeSeries called %d times; expected %d", a, e)
}
model.Unlock()
// Move clock forward and force to scan again.
manual.Increment(storage.TimeSeriesMaintenanceInterval.Nanoseconds())
store.ForceTimeSeriesMaintenanceQueueProcess()
util.SucceedsSoon(t, func() error {
model.Lock()
defer model.Unlock()
if a, e := model.containsCalled, len(expectedStartKeys)*2; a != e {
return errors.Errorf("ContainsTimeSeries called %d times; expected %d", a, e)
}
if a, e := model.pruneCalled, len(expectedStartKeys)*2; a != e {
return errors.Errorf("PruneTimeSeries called %d times; expected %d", a, e)
}
return nil
})
}
// TestStoreRangeMergeMetadataCleanup tests that all metadata of a
// subsumed range is cleaned up on merge.
func TestStoreRangeMergeMetadataCleanup(t *testing.T) {
defer leaktest.AfterTest(t)()
storeCfg := storage.TestStoreConfig(nil)
storeCfg.TestingKnobs.DisableSplitQueue = true
store, stopper := createTestStoreWithConfig(t, storeCfg)
defer stopper.Stop()
scan := func(f func(roachpb.KeyValue) (bool, error)) {
if _, err := engine.MVCCIterate(context.Background(), store.Engine(), roachpb.KeyMin, roachpb.KeyMax, hlc.ZeroTimestamp, true, nil, false, f); err != nil {
t.Fatal(err)
}
}
content := roachpb.Key("testing!")
// Write some values left of the proposed split key.
pArgs := putArgs([]byte("aaa"), content)
if _, err := client.SendWrapped(context.Background(), rg1(store), &pArgs); err != nil {
t.Fatal(err)
}
// Collect all the keys.
preKeys := make(map[string]struct{})
scan(func(kv roachpb.KeyValue) (bool, error) {
preKeys[string(kv.Key)] = struct{}{}
return false, nil
})
// Split the range.
_, bDesc, err := createSplitRanges(store)
if err != nil {
t.Fatal(err)
}
// Write some values right of the split key.
pArgs = putArgs([]byte("ccc"), content)
if _, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
RangeID: bDesc.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
if _, err := client.SendWrapped(context.Background(), rg1(store), &args); err != nil {
t.Fatal(err)
}
// Collect all the keys again.
postKeys := make(map[string]struct{})
scan(func(kv roachpb.KeyValue) (bool, error) {
postKeys[string(kv.Key)] = struct{}{}
return false, nil
})
// Compute the new keys.
for k := range preKeys {
delete(postKeys, k)
}
// Keep only the subsumed range's local keys.
localRangeKeyPrefix := string(keys.MakeRangeIDPrefix(bDesc.RangeID))
for k := range postKeys {
if !strings.HasPrefix(k, localRangeKeyPrefix) {
delete(postKeys, k)
}
}
if numKeys := len(postKeys); numKeys > 0 {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%d keys were not cleaned up:\n", numKeys)
for k := range postKeys {
fmt.Fprintf(&buf, "%q\n", k)
}
t.Fatal(buf.String())
}
}
// TestStoreRangeMergeWithData attempts to merge two collocate ranges
// each containing data.
func TestStoreRangeMergeWithData(t *testing.T) {
defer leaktest.AfterTest(t)()
storeCfg := storage.TestStoreConfig(nil)
storeCfg.TestingKnobs.DisableSplitQueue = true
store, stopper := createTestStoreWithConfig(t, storeCfg)
defer stopper.Stop()
content := roachpb.Key("testing!")
aDesc, bDesc, err := createSplitRanges(store)
if err != nil {
t.Fatal(err)
}
// Write some values left and right of the proposed split key.
pArgs := putArgs([]byte("aaa"), content)
if _, err := client.SendWrapped(context.Background(), rg1(store), &pArgs); err != nil {
t.Fatal(err)
}
pArgs = putArgs([]byte("ccc"), content)
if _, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
RangeID: bDesc.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Confirm the values are there.
gArgs := getArgs([]byte("aaa"))
if reply, err := client.SendWrapped(context.Background(), rg1(store), &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("ccc"))
if reply, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
RangeID: bDesc.RangeID,
}, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
if _, err := client.SendWrapped(context.Background(), rg1(store), &args); err != nil {
t.Fatal(err)
}
// Verify no intents remains on range descriptor keys.
for _, key := range []roachpb.Key{keys.RangeDescriptorKey(aDesc.StartKey), keys.RangeDescriptorKey(bDesc.StartKey)} {
if _, _, err := engine.MVCCGet(context.Background(), store.Engine(), key, store.Clock().Now(), true, nil); err != nil {
t.Fatal(err)
}
}
// Verify the merge by looking up keys from both ranges.
rangeA := store.LookupReplica([]byte("a"), nil)
rangeB := store.LookupReplica([]byte("c"), nil)
rangeADesc := rangeA.Desc()
rangeBDesc := rangeB.Desc()
if !reflect.DeepEqual(rangeA, rangeB) {
t.Fatalf("ranges were not merged %+v=%+v", rangeADesc, rangeBDesc)
}
if !bytes.Equal(rangeADesc.StartKey, roachpb.RKeyMin) {
t.Fatalf("The start key is not equal to KeyMin %q=%q", rangeADesc.StartKey, roachpb.RKeyMin)
}
if !bytes.Equal(rangeADesc.EndKey, roachpb.RKeyMax) {
t.Fatalf("The end key is not equal to KeyMax %q=%q", rangeADesc.EndKey, roachpb.RKeyMax)
}
// Try to get values from after the merge.
gArgs = getArgs([]byte("aaa"))
if reply, err := client.SendWrapped(context.Background(), rg1(store), &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("ccc"))
if reply, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
RangeID: rangeB.RangeID,
}, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Put new values after the merge on both sides.
pArgs = putArgs([]byte("aaaa"), content)
if _, err := client.SendWrapped(context.Background(), rg1(store), &pArgs); err != nil {
t.Fatal(err)
}
pArgs = putArgs([]byte("cccc"), content)
if _, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
RangeID: rangeB.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Try to get the newly placed values.
gArgs = getArgs([]byte("aaaa"))
if reply, err := client.SendWrapped(context.Background(), rg1(store), &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("cccc"))
if reply, err := client.SendWrapped(context.Background(), rg1(store), &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
}
// TestTxnPutOutOfOrder tests a case where a put operation of an older
// timestamp comes after a put operation of a newer timestamp in a
// txn. The test ensures such an out-of-order put succeeds and
// overrides an old value. The test uses a "Writer" and a "Reader"
// to reproduce an out-of-order put.
//
// 1) The Writer executes a put operation and writes a write intent with
// time T in a txn.
// 2) Before the Writer's txn is committed, the Reader sends a high priority
// get operation with time T+100. This pushes the Writer txn timestamp to
// T+100 and triggers the restart of the Writer's txn. The original
// write intent timestamp is also updated to T+100.
// 3) The Writer starts a new epoch of the txn, but before it writes, the
// Reader sends another high priority get operation with time T+200. This
// pushes the Writer txn timestamp to T+200 to trigger a restart of the
// Writer txn. The Writer will not actually restart until it tries to commit
// the current epoch of the transaction. The Reader updates the timestamp of
// the write intent to T+200. The test deliberately fails the Reader get
// operation, and cockroach doesn't update its read timestamp cache.
// 4) The Writer executes the put operation again. This put operation comes
// out-of-order since its timestamp is T+100, while the intent timestamp
// updated at Step 3 is T+200.
// 5) The put operation overrides the old value using timestamp T+100.
// 6) When the Writer attempts to commit its txn, the txn will be restarted
// again at a new epoch timestamp T+200, which will finally succeed.
func TestTxnPutOutOfOrder(t *testing.T) {
defer leaktest.AfterTest(t)()
const key = "key"
// Set up a filter to so that the get operation at Step 3 will return an error.
var numGets int32
stopper := stop.NewStopper()
defer stopper.Stop()
manual := hlc.NewManualClock(123)
cfg := storage.TestStoreConfig(hlc.NewClock(manual.UnixNano, time.Nanosecond))
cfg.TestingKnobs.TestingCommandFilter =
func(filterArgs storagebase.FilterArgs) *roachpb.Error {
if _, ok := filterArgs.Req.(*roachpb.GetRequest); ok &&
filterArgs.Req.Header().Key.Equal(roachpb.Key(key)) &&
filterArgs.Hdr.Txn == nil {
// The Reader executes two get operations, each of which triggers two get requests
// (the first request fails and triggers txn push, and then the second request
// succeeds). Returns an error for the fourth get request to avoid timestamp cache
// update after the third get operation pushes the txn timestamp.
if atomic.AddInt32(&numGets, 1) == 4 {
return roachpb.NewErrorWithTxn(errors.Errorf("Test"), filterArgs.Hdr.Txn)
}
}
return nil
}
eng := engine.NewInMem(roachpb.Attributes{}, 10<<20)
stopper.AddCloser(eng)
store := createTestStoreWithEngine(t,
eng,
true,
cfg,
stopper,
)
// Put an initial value.
initVal := []byte("initVal")
err := store.DB().Put(context.TODO(), key, initVal)
if err != nil {
t.Fatalf("failed to put: %s", err)
}
waitPut := make(chan struct{})
waitFirstGet := make(chan struct{})
waitTxnRestart := make(chan struct{})
waitSecondGet := make(chan struct{})
waitTxnComplete := make(chan struct{})
// Start the Writer.
go func() {
epoch := -1
// Start a txn that does read-after-write.
// The txn will be restarted twice, and the out-of-order put
// will happen in the second epoch.
if err := store.DB().Txn(context.TODO(), func(txn *client.Txn) error {
epoch++
if epoch == 1 {
// Wait until the second get operation is issued.
close(waitTxnRestart)
<-waitSecondGet
}
updatedVal := []byte("updatedVal")
if err := txn.Put(key, updatedVal); err != nil {
return err
}
// Make sure a get will return the value that was just written.
actual, err := txn.Get(key)
if err != nil {
return err
}
if !bytes.Equal(actual.ValueBytes(), updatedVal) {
t.Fatalf("unexpected get result: %s", actual)
}
if epoch == 0 {
// Wait until the first get operation will push the txn timestamp.
close(waitPut)
<-waitFirstGet
}
b := txn.NewBatch()
return txn.CommitInBatch(b)
}); err != nil {
t.Fatal(err)
}
if epoch != 2 {
t.Fatalf("unexpected number of txn retries: %d", epoch)
}
close(waitTxnComplete)
}()
<-waitPut
// Start the Reader.
// Advance the clock and send a get operation with higher
// priority to trigger the txn restart.
manual.Increment(100)
priority := roachpb.UserPriority(-math.MaxInt32)
requestHeader := roachpb.Span{
Key: roachpb.Key(key),
}
if _, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
Timestamp: cfg.Clock.Now(),
UserPriority: priority,
}, &roachpb.GetRequest{Span: requestHeader}); err != nil {
t.Fatalf("failed to get: %s", err)
}
// Wait until the writer restarts the txn.
close(waitFirstGet)
<-waitTxnRestart
// Advance the clock and send a get operation again. This time
// we use TestingCommandFilter so that a get operation is not
// processed after the write intent is resolved (to prevent the
// timestamp cache from being updated).
manual.Increment(100)
if _, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
Timestamp: cfg.Clock.Now(),
UserPriority: priority,
}, &roachpb.GetRequest{Span: requestHeader}); err == nil {
t.Fatal("unexpected success of get")
}
close(waitSecondGet)
<-waitTxnComplete
}
// TestMultiRangeScanWithMaxResults tests that commands which access multiple
// ranges with MaxResults parameter are carried out properly.
func TestMultiRangeScanWithMaxResults(t *testing.T) {
defer leaktest.AfterTest(t)()
testCases := []struct {
splitKeys []roachpb.Key
keys []roachpb.Key
}{
{[]roachpb.Key{roachpb.Key("m")},
[]roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}},
{[]roachpb.Key{roachpb.Key("h"), roachpb.Key("q")},
[]roachpb.Key{roachpb.Key("b"), roachpb.Key("f"), roachpb.Key("k"),
roachpb.Key("r"), roachpb.Key("w"), roachpb.Key("y")}},
}
for i, tc := range testCases {
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
ts := s.(*TestServer)
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = ts.stopper.ShouldQuiesce()
ds := kv.NewDistSender(kv.DistSenderConfig{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RPCRetryOptions: &retryOpts,
}, ts.Gossip())
ambient := log.AmbientContext{Tracer: tracing.NewTracer()}
tds := kv.NewTxnCoordSender(
ambient,
ds,
ts.Clock(),
ts.Cfg.Linearizable,
ts.stopper,
kv.MakeTxnMetrics(metric.TestSampleInterval),
)
for _, sk := range tc.splitKeys {
if err := ts.node.storeCfg.DB.AdminSplit(context.TODO(), sk); err != nil {
t.Fatal(err)
}
}
for _, k := range tc.keys {
put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k))
if _, err := client.SendWrapped(context.Background(), tds, put); err != nil {
t.Fatal(err)
}
}
// Try every possible ScanRequest startKey.
for start := 0; start < len(tc.keys); start++ {
// Try every possible maxResults, from 1 to beyond the size of key array.
for maxResults := 1; maxResults <= len(tc.keys)-start+1; maxResults++ {
scan := roachpb.NewScan(tc.keys[start], tc.keys[len(tc.keys)-1].Next())
reply, err := client.SendWrappedWith(
context.Background(), tds, roachpb.Header{MaxSpanRequestKeys: int64(maxResults)}, scan,
)
if err != nil {
t.Fatal(err)
}
rows := reply.(*roachpb.ScanResponse).Rows
if start+maxResults <= len(tc.keys) && len(rows) != maxResults {
t.Errorf("%d: start=%s: expected %d rows, but got %d", i, tc.keys[start], maxResults, len(rows))
} else if start+maxResults == len(tc.keys)+1 && len(rows) != maxResults-1 {
t.Errorf("%d: expected %d rows, but got %d", i, maxResults-1, len(rows))
}
}
}
}
}
// TestMultiRangeScanDeleteRange tests that commands which access multiple
// ranges are carried out properly.
func TestMultiRangeScanDeleteRange(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
ts := s.(*TestServer)
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = ts.stopper.ShouldQuiesce()
ds := kv.NewDistSender(kv.DistSenderConfig{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RPCRetryOptions: &retryOpts,
}, ts.Gossip())
ambient := log.AmbientContext{Tracer: tracing.NewTracer()}
tds := kv.NewTxnCoordSender(
ambient,
ds,
s.Clock(),
ts.Cfg.Linearizable,
ts.stopper,
kv.MakeTxnMetrics(metric.TestSampleInterval),
)
if err := ts.node.storeCfg.DB.AdminSplit(context.TODO(), "m"); err != nil {
t.Fatal(err)
}
writes := []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}
get := &roachpb.GetRequest{
Span: roachpb.Span{Key: writes[0]},
}
get.EndKey = writes[len(writes)-1]
if _, err := client.SendWrapped(context.Background(), tds, get); err == nil {
t.Errorf("able to call Get with a key range: %v", get)
}
var delTS hlc.Timestamp
for i, k := range writes {
put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k))
if _, err := client.SendWrapped(context.Background(), tds, put); err != nil {
t.Fatal(err)
}
scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next())
reply, err := client.SendWrapped(context.Background(), tds, scan)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if sr.Txn != nil {
// This was the other way around at some point in the past.
// Same below for Delete, etc.
t.Errorf("expected no transaction in response header")
}
if rows := sr.Rows; len(rows) != i+1 {
t.Fatalf("expected %d rows, but got %d", i+1, len(rows))
}
}
del := &roachpb.DeleteRangeRequest{
Span: roachpb.Span{
Key: writes[0],
EndKey: roachpb.Key(writes[len(writes)-1]).Next(),
},
ReturnKeys: true,
}
reply, err := client.SendWrappedWith(context.Background(), tds, roachpb.Header{Timestamp: delTS}, del)
if err != nil {
t.Fatal(err)
}
dr := reply.(*roachpb.DeleteRangeResponse)
if dr.Txn != nil {
t.Errorf("expected no transaction in response header")
}
if !reflect.DeepEqual(dr.Keys, writes) {
t.Errorf("expected %d keys to be deleted, but got %d instead", writes, dr.Keys)
}
scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next())
txn := &roachpb.Transaction{Name: "MyTxn"}
reply, err = client.SendWrappedWith(context.Background(), tds, roachpb.Header{Txn: txn}, scan)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if txn := sr.Txn; txn == nil || txn.Name != "MyTxn" {
t.Errorf("wanted Txn to persist, but it changed to %v", txn)
}
if rows := sr.Rows; len(rows) > 0 {
t.Fatalf("scan after delete returned rows: %v", rows)
}
}
func TestRangeInfo(t *testing.T) {
defer leaktest.AfterTest(t)()
mtc := startMultiTestContext(t, 2)
defer mtc.Stop()
// Up-replicate to two replicas.
mtc.replicateRange(mtc.stores[0].LookupReplica(roachpb.RKeyMin, nil).RangeID, 1)
// Split the key space at key "a".
splitKey := roachpb.RKey("a")
splitArgs := adminSplitArgs(splitKey.AsRawKey(), splitKey.AsRawKey())
if _, pErr := client.SendWrapped(
context.Background(), rg1(mtc.stores[0]), &splitArgs,
); pErr != nil {
t.Fatal(pErr)
}
// Get the replicas for each side of the split. This is done within
// a SucceedsSoon loop to ensure the split completes.
var lhsReplica0, lhsReplica1, rhsReplica0, rhsReplica1 *storage.Replica
util.SucceedsSoon(t, func() error {
lhsReplica0 = mtc.stores[0].LookupReplica(roachpb.RKeyMin, nil)
lhsReplica1 = mtc.stores[1].LookupReplica(roachpb.RKeyMin, nil)
rhsReplica0 = mtc.stores[0].LookupReplica(splitKey, nil)
rhsReplica1 = mtc.stores[1].LookupReplica(splitKey, nil)
if lhsReplica0 == rhsReplica0 || lhsReplica1 == rhsReplica1 {
return errors.Errorf("replicas not post-split %v, %v, %v, %v",
lhsReplica0, rhsReplica0, rhsReplica0, rhsReplica1)
}
return nil
})
lhsLease, _ := lhsReplica0.GetLease()
rhsLease, _ := rhsReplica0.GetLease()
// Verify range info is not set if unrequested.
getArgs := getArgs(splitKey.AsRawKey())
reply, pErr := client.SendWrapped(context.Background(), mtc.distSenders[0], &getArgs)
if pErr != nil {
t.Fatal(pErr)
}
if len(reply.Header().RangeInfos) > 0 {
t.Errorf("expected empty range infos if unrequested; got %v", reply.Header().RangeInfos)
}
// Verify range info on a get request.
h := roachpb.Header{
ReturnRangeInfo: true,
}
reply, pErr = client.SendWrappedWith(context.Background(), mtc.distSenders[0], h, &getArgs)
if pErr != nil {
t.Fatal(pErr)
}
expRangeInfos := []roachpb.RangeInfo{
{
Desc: *rhsReplica0.Desc(),
Lease: *rhsLease,
},
}
if !reflect.DeepEqual(reply.Header().RangeInfos, expRangeInfos) {
t.Errorf("on get reply, expected %+v; got %+v", expRangeInfos, reply.Header().RangeInfos)
}
// Verify range info on a put request.
putArgs := putArgs(splitKey.AsRawKey(), []byte("foo"))
reply, pErr = client.SendWrappedWith(context.Background(), mtc.distSenders[0], h, &putArgs)
if pErr != nil {
t.Fatal(pErr)
}
if !reflect.DeepEqual(reply.Header().RangeInfos, expRangeInfos) {
t.Errorf("on put reply, expected %+v; got %+v", expRangeInfos, reply.Header().RangeInfos)
}
// Verify multiple range infos on a scan request.
scanArgs := roachpb.ScanRequest{
Span: roachpb.Span{
Key: keys.SystemMax,
EndKey: roachpb.KeyMax,
},
}
h.Txn = roachpb.NewTransaction("test", roachpb.KeyMin, 1, enginepb.SERIALIZABLE, mtc.clock.Now(), 0)
reply, pErr = client.SendWrappedWith(context.Background(), mtc.distSenders[0], h, &scanArgs)
if pErr != nil {
t.Fatal(pErr)
}
expRangeInfos = []roachpb.RangeInfo{
{
Desc: *lhsReplica0.Desc(),
Lease: *lhsLease,
},
{
Desc: *rhsReplica0.Desc(),
Lease: *rhsLease,
},
}
if !reflect.DeepEqual(reply.Header().RangeInfos, expRangeInfos) {
t.Errorf("on scan reply, expected %+v; got %+v", expRangeInfos, reply.Header().RangeInfos)
}
// Verify multiple range infos and order on a reverse scan request.
revScanArgs := roachpb.ReverseScanRequest{
Span: roachpb.Span{
Key: keys.SystemMax,
EndKey: roachpb.KeyMax,
},
}
reply, pErr = client.SendWrappedWith(context.Background(), mtc.distSenders[0], h, &revScanArgs)
if pErr != nil {
t.Fatal(pErr)
}
expRangeInfos = []roachpb.RangeInfo{
{
Desc: *rhsReplica0.Desc(),
Lease: *rhsLease,
},
{
Desc: *lhsReplica0.Desc(),
Lease: *lhsLease,
},
}
if !reflect.DeepEqual(reply.Header().RangeInfos, expRangeInfos) {
t.Errorf("on reverse scan reply, expected %+v; got %+v", expRangeInfos, reply.Header().RangeInfos)
}
// Change lease holders for both ranges and re-scan.
for _, r := range []*storage.Replica{lhsReplica1, rhsReplica1} {
replDesc, err := r.GetReplicaDescriptor()
if err != nil {
t.Fatal(err)
}
if err = mtc.dbs[0].AdminTransferLease(context.TODO(),
r.Desc().StartKey.AsRawKey(), replDesc.StoreID); err != nil {
t.Fatalf("unable to transfer lease to replica %s: %s", r, err)
}
}
reply, pErr = client.SendWrappedWith(context.Background(), mtc.distSenders[0], h, &scanArgs)
if pErr != nil {
t.Fatal(pErr)
}
lhsLease, _ = lhsReplica1.GetLease()
rhsLease, _ = rhsReplica1.GetLease()
expRangeInfos = []roachpb.RangeInfo{
{
Desc: *lhsReplica1.Desc(),
Lease: *lhsLease,
},
{
Desc: *rhsReplica1.Desc(),
Lease: *rhsLease,
},
}
if !reflect.DeepEqual(reply.Header().RangeInfos, expRangeInfos) {
t.Errorf("on scan reply, expected %+v; got %+v", expRangeInfos, reply.Header().RangeInfos)
}
}
// TestRangeLookupUseReverse tests whether the results and the results count
// are correct when scanning in reverse order.
func TestRangeLookupUseReverse(t *testing.T) {
defer leaktest.AfterTest(t)()
storeCfg := storage.TestStoreConfig(nil)
storeCfg.TestingKnobs.DisableSplitQueue = true
store, stopper := createTestStoreWithConfig(t, storeCfg)
defer stopper.Stop()
// Init test ranges:
// ["","a"), ["a","c"), ["c","e"), ["e","g") and ["g","\xff\xff").
splits := []roachpb.AdminSplitRequest{
adminSplitArgs(roachpb.Key("g"), roachpb.Key("g")),
adminSplitArgs(roachpb.Key("e"), roachpb.Key("e")),
adminSplitArgs(roachpb.Key("c"), roachpb.Key("c")),
adminSplitArgs(roachpb.Key("a"), roachpb.Key("a")),
}
for _, split := range splits {
_, pErr := client.SendWrapped(context.Background(), rg1(store), &split)
if pErr != nil {
t.Fatalf("%q: split unexpected error: %s", split.SplitKey, pErr)
}
}
// Resolve the intents.
scanArgs := roachpb.ScanRequest{
Span: roachpb.Span{
Key: keys.RangeMetaKey(roachpb.RKeyMin.Next()),
EndKey: keys.RangeMetaKey(roachpb.RKeyMax),
},
}
util.SucceedsSoon(t, func() error {
_, pErr := client.SendWrapped(context.Background(), rg1(store), &scanArgs)
return pErr.GoError()
})
revScanArgs := func(key []byte, maxResults int32) *roachpb.RangeLookupRequest {
return &roachpb.RangeLookupRequest{
Span: roachpb.Span{
Key: key,
},
MaxRanges: maxResults,
Reverse: true,
}
}
// Test cases.
testCases := []struct {
request *roachpb.RangeLookupRequest
expected []roachpb.RangeDescriptor
expectedPre []roachpb.RangeDescriptor
}{
// Test key in the middle of the range.
{
request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("f")), 2),
// ["e","g") and ["c","e").
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")},
},
},
// Test key in the end key of the range.
{
request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("g")), 3),
// ["e","g"), ["c","e") and ["a","c").
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")},
{StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("c")},
},
},
{
request: revScanArgs(keys.RangeMetaKey(roachpb.RKey("e")), 2),
// ["c","e") and ["a","c").
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("c"), EndKey: roachpb.RKey("e")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("c")},
},
},
// Test Meta2KeyMax.
{
request: revScanArgs(keys.Meta2KeyMax, 2),
// ["e","g") and ["g","\xff\xff")
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("g"), EndKey: roachpb.RKey("\xff\xff")},
},
expectedPre: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKey("e"), EndKey: roachpb.RKey("g")},
},
},
// Test Meta1KeyMax.
{
request: revScanArgs(keys.Meta1KeyMax, 1),
// ["","a")
expected: []roachpb.RangeDescriptor{
{StartKey: roachpb.RKeyMin, EndKey: roachpb.RKey("a")},
},
},
}
for testIdx, test := range testCases {
resp, pErr := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
ReadConsistency: roachpb.INCONSISTENT,
}, test.request)
if pErr != nil {
t.Fatalf("%d: RangeLookup error: %s", testIdx, pErr)
}
rlReply := resp.(*roachpb.RangeLookupResponse)
// Checks the results count.
if rsCount, preRSCount := len(rlReply.Ranges), len(rlReply.PrefetchedRanges); int32(rsCount+preRSCount) != test.request.MaxRanges {
t.Fatalf("%d: returned results count, expected %d, but got %d+%d", testIdx, test.request.MaxRanges, rsCount, preRSCount)
}
// Checks the range descriptors.
for _, rngSlice := range []struct {
expect, reply []roachpb.RangeDescriptor
}{
{test.expected, rlReply.Ranges},
{test.expectedPre, rlReply.PrefetchedRanges},
} {
for i, rng := range rngSlice.expect {
if !(rng.StartKey.Equal(rngSlice.reply[i].StartKey) && rng.EndKey.Equal(rngSlice.reply[i].EndKey)) {
t.Fatalf("%d: returned range is not correct, expected %v, but got %v", testIdx, rng, rngSlice.reply[i])
}
}
}
}
}
// TestRequestToUninitializedRange tests the behavior when a request
// is sent to a node which should be a replica of the correct range
// but has not yet received its initial snapshot. This would
// previously panic due to a malformed error response from the server,
// as seen in https://github.com/cockroachdb/cockroach/issues/6027.
//
// Prior to the other changes in the commit that introduced it, this
// test would reliable trigger the panic from #6027. However, it
// relies on some hacky tricks to both trigger the panic and shut down
// cleanly. If this test needs a lot of maintenance in the future we
// should be willing to get rid of it.
func TestRequestToUninitializedRange(t *testing.T) {
defer leaktest.AfterTest(t)()
srv, _, _ := serverutils.StartServer(t, base.TestServerArgs{
StoreSpecs: []base.StoreSpec{
base.DefaultTestStoreSpec,
base.DefaultTestStoreSpec,
},
})
defer srv.Stopper().Stop()
s := srv.(*server.TestServer)
// Choose a range ID that is much larger than any that would be
// created by initial splits.
const rangeID = roachpb.RangeID(1000)
// Set up a range with replicas on two stores of the same node. This
// ensures that the DistSender will consider both replicas healthy
// and will try to talk to both (so we can get a non-retryable error
// from the second store).
replica1 := roachpb.ReplicaDescriptor{
NodeID: 1,
StoreID: 1,
ReplicaID: 1,
}
replica2 := roachpb.ReplicaDescriptor{
NodeID: 1,
StoreID: 2,
ReplicaID: 2,
}
// HACK: remove the second store from the node to generate a
// non-retryable error when we try to talk to it.
store2, err := s.Stores().GetStore(2)
if err != nil {
t.Fatal(err)
}
s.Stores().RemoveStore(store2)
// Create the uninitialized range by sending an isolated raft
// message to the first store.
conn, err := s.RPCContext().GRPCDial(s.ServingAddr())
if err != nil {
t.Fatal(err)
}
raftClient := storage.NewMultiRaftClient(conn)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stream, err := raftClient.RaftMessageBatch(ctx)
if err != nil {
t.Fatal(err)
}
msg := storage.RaftMessageRequestBatch{
Requests: []storage.RaftMessageRequest{
{
RangeID: rangeID,
ToReplica: replica1,
FromReplica: replica2,
Message: raftpb.Message{
Type: raftpb.MsgApp,
To: 1,
},
},
},
}
if err := stream.Send(&msg); err != nil {
t.Fatal(err)
}
// Make sure the replica was created.
store1, err := s.Stores().GetStore(1)
if err != nil {
t.Fatal(err)
}
util.SucceedsSoon(t, func() error {
if replica, err := store1.GetReplica(rangeID); err != nil {
return errors.Errorf("failed to look up replica: %s", err)
} else if replica.IsInitialized() {
return errors.Errorf("expected replica to be uninitialized")
}
return nil
})
// Create our own DistSender so we can force some requests to the
// bogus range. The DistSender needs to be in scope for its own
// MockRangeDescriptorDB closure.
var sender *kv.DistSender
sender = kv.NewDistSender(kv.DistSenderConfig{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RangeDescriptorDB: kv.MockRangeDescriptorDB(
func(key roachpb.RKey, useReverseScan bool,
) ([]roachpb.RangeDescriptor, []roachpb.RangeDescriptor, *roachpb.Error) {
if key.Equal(roachpb.RKeyMin) {
// Pass through requests for the first range to the real sender.
desc, err := sender.FirstRange()
if err != nil {
return nil, nil, roachpb.NewError(err)
}
return []roachpb.RangeDescriptor{*desc}, nil, nil
}
return []roachpb.RangeDescriptor{{
RangeID: rangeID,
StartKey: roachpb.RKey(keys.Meta2Prefix),
EndKey: roachpb.RKeyMax,
Replicas: []roachpb.ReplicaDescriptor{replica1, replica2},
}}, nil, nil
}),
}, s.Gossip())
// Only inconsistent reads triggered the panic in #6027.
hdr := roachpb.Header{
ReadConsistency: roachpb.INCONSISTENT,
}
req := roachpb.NewGet(roachpb.Key("asdf"))
// Repeat the test a few times: due to the randomization between the
// two replicas, each attempt only had a 50% chance of triggering
// the panic.
for i := 0; i < 5; i++ {
_, pErr := client.SendWrappedWith(context.Background(), sender, hdr, req)
// Each attempt fails with "store 2 not found" because that is the
// non-retryable error.
if !testutils.IsPError(pErr, "store 2 not found") {
t.Fatal(pErr)
}
}
}
func TestRangeTransferLease(t *testing.T) {
defer leaktest.AfterTest(t)()
cfg := storage.TestStoreConfig(nil)
var filterMu syncutil.Mutex
var filter func(filterArgs storagebase.FilterArgs) *roachpb.Error
cfg.TestingKnobs.TestingCommandFilter =
func(filterArgs storagebase.FilterArgs) *roachpb.Error {
filterMu.Lock()
filterCopy := filter
filterMu.Unlock()
if filterCopy != nil {
return filterCopy(filterArgs)
}
return nil
}
var waitForTransferBlocked atomic.Value
waitForTransferBlocked.Store(false)
transferBlocked := make(chan struct{})
cfg.TestingKnobs.LeaseTransferBlockedOnExtensionEvent = func(
_ roachpb.ReplicaDescriptor) {
if waitForTransferBlocked.Load().(bool) {
transferBlocked <- struct{}{}
waitForTransferBlocked.Store(false)
}
}
mtc := &multiTestContext{}
mtc.storeConfig = &cfg
mtc.Start(t, 2)
defer mtc.Stop()
// First, do a write; we'll use it to determine when the dust has settled.
leftKey := roachpb.Key("a")
incArgs := incrementArgs(leftKey, 1)
if _, pErr := client.SendWrapped(context.Background(), mtc.distSenders[0], &incArgs); pErr != nil {
t.Fatal(pErr)
}
// Get the left range's ID.
rangeID := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil).RangeID
// Replicate the left range onto node 1.
mtc.replicateRange(rangeID, 1)
replica0 := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil)
replica1 := mtc.stores[1].LookupReplica(roachpb.RKey("a"), nil)
gArgs := getArgs(leftKey)
replica0Desc, err := replica0.GetReplicaDescriptor()
if err != nil {
t.Fatal(err)
}
// Check that replica0 can serve reads OK.
if _, pErr := client.SendWrappedWith(
context.Background(),
mtc.senders[0],
roachpb.Header{Replica: replica0Desc},
&gArgs,
); pErr != nil {
t.Fatal(pErr)
}
{
// Transferring the lease to ourself should be a no-op.
origLeasePtr, _ := replica0.GetLease()
origLease := *origLeasePtr
if err := replica0.AdminTransferLease(replica0Desc.StoreID); err != nil {
t.Fatal(err)
}
newLeasePtr, _ := replica0.GetLease()
if origLeasePtr != newLeasePtr || origLease != *newLeasePtr {
t.Fatalf("expected %+v, but found %+v", origLeasePtr, newLeasePtr)
}
}
{
// An invalid target should result in an error.
const expected = "unable to find store .* in range"
if err := replica0.AdminTransferLease(1000); !testutils.IsError(err, expected) {
t.Fatalf("expected %s, but found %v", expected, err)
}
}
// Move the lease to store 1.
var newHolderDesc roachpb.ReplicaDescriptor
util.SucceedsSoon(t, func() error {
var err error
newHolderDesc, err = replica1.GetReplicaDescriptor()
return err
})
if err := replica0.AdminTransferLease(newHolderDesc.StoreID); err != nil {
t.Fatal(err)
}
// Check that replica0 doesn't serve reads any more.
replica0Desc, err = replica0.GetReplicaDescriptor()
if err != nil {
t.Fatal(err)
}
_, pErr := client.SendWrappedWith(
context.Background(),
mtc.senders[0],
roachpb.Header{Replica: replica0Desc},
&gArgs,
)
nlhe, ok := pErr.GetDetail().(*roachpb.NotLeaseHolderError)
if !ok {
t.Fatalf("expected %T, got %s", &roachpb.NotLeaseHolderError{}, pErr)
}
if *(nlhe.LeaseHolder) != newHolderDesc {
t.Fatalf("expected lease holder %+v, got %+v",
newHolderDesc, nlhe.LeaseHolder)
}
// Check that replica1 now has the lease (or gets it soon).
util.SucceedsSoon(t, func() error {
if _, pErr := client.SendWrappedWith(
context.Background(),
mtc.senders[1],
roachpb.Header{Replica: replica0Desc},
&gArgs,
); pErr != nil {
return pErr.GoError()
}
return nil
})
replica1Lease, _ := replica1.GetLease()
// Verify the timestamp cache low water. Because we executed a transfer lease
// request, the low water should be set to the new lease start time which is
// less than the previous lease's expiration time.
if lowWater := replica1.GetTimestampCacheLowWater(); lowWater != replica1Lease.Start {
t.Fatalf("expected timestamp cache low water %s, but found %s",
replica1Lease.Start, lowWater)
}
// Make replica1 extend its lease and transfer the lease immediately after
// that. Test that the transfer still happens (it'll wait until the extension
// is done).
extensionSem := make(chan struct{})
filterMu.Lock()
filter = func(filterArgs storagebase.FilterArgs) *roachpb.Error {
if filterArgs.Sid != mtc.stores[1].Ident.StoreID {
return nil
}
llReq, ok := filterArgs.Req.(*roachpb.RequestLeaseRequest)
if !ok {
return nil
}
if llReq.Lease.Replica == newHolderDesc {
// Notify the main thread that the extension is in progress and wait for
// the signal to proceed.
filterMu.Lock()
filter = nil
filterMu.Unlock()
extensionSem <- struct{}{}
<-extensionSem
}
return nil
}
filterMu.Unlock()
// Initiate an extension.
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
shouldRenewTS := replica1Lease.StartStasis.Add(-1, 0)
mtc.manualClock.Set(shouldRenewTS.WallTime + 1)
if _, pErr := client.SendWrappedWith(
context.Background(),
mtc.senders[1],
roachpb.Header{Replica: replica0Desc},
&gArgs,
); pErr != nil {
panic(pErr)
}
}()
<-extensionSem
waitForTransferBlocked.Store(true)
// Initiate a transfer.
wg.Add(1)
go func() {
defer wg.Done()
// Transfer back from replica1 to replica0.
if err := replica1.AdminTransferLease(replica0Desc.StoreID); err != nil {
panic(err)
}
}()
// Wait for the transfer to be blocked by the extension.
<-transferBlocked
// Now unblock the extension.
extensionSem <- struct{}{}
// Check that the transfer to replica1 eventually happens.
util.SucceedsSoon(t, func() error {
if _, pErr := client.SendWrappedWith(
context.Background(),
mtc.senders[0],
roachpb.Header{Replica: replica0Desc},
&gArgs,
); pErr != nil {
return pErr.GoError()
}
return nil
})
filterMu.Lock()
filter = nil
filterMu.Unlock()
wg.Wait()
}
// TestMultiRangeScanReverseScanInconsistent verifies that a Scan/ReverseScan
// across ranges that doesn't require read consistency will set a timestamp
// using the clock local to the distributed sender.
func TestMultiRangeScanReverseScanInconsistent(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
db := setupMultipleRanges(t, s, "b")
// Write keys "a" and "b", the latter of which is the first key in the
// second range.
keys := [2]string{"a", "b"}
ts := [2]hlc.Timestamp{}
for i, key := range keys {
b := &client.Batch{}
b.Put(key, "value")
if err := db.Run(context.TODO(), b); err != nil {
t.Fatal(err)
}
ts[i] = s.Clock().Now()
log.Infof(context.TODO(), "%d: %s %d", i, key, ts[i])
if i == 0 {
util.SucceedsSoon(t, func() error {
// Enforce that when we write the second key, it's written
// with a strictly higher timestamp. We're dropping logical
// ticks and the clock may just have been pushed into the
// future, so that's necessary. See #3122.
if ts[0].WallTime >= s.Clock().Now().WallTime {
return errors.New("time stands still")
}
return nil
})
}
}
// Do an inconsistent Scan/ReverseScan from a new DistSender and verify
// it does the read at its local clock and doesn't receive an
// OpRequiresTxnError. We set the local clock to the timestamp of
// just above the first key to verify it's used to read only key "a".
for i, request := range []roachpb.Request{
roachpb.NewScan(roachpb.Key("a"), roachpb.Key("c")),
roachpb.NewReverseScan(roachpb.Key("a"), roachpb.Key("c")),
} {
manual := hlc.NewManualClock(ts[0].WallTime + 1)
clock := hlc.NewClock(manual.UnixNano)
ds := kv.NewDistSender(
kv.DistSenderConfig{Clock: clock, RPCContext: s.RPCContext()},
s.(*server.TestServer).Gossip(),
)
reply, err := client.SendWrappedWith(context.Background(), ds, roachpb.Header{
ReadConsistency: roachpb.INCONSISTENT,
}, request)
if err != nil {
t.Fatal(err)
}
var rows []roachpb.KeyValue
switch r := reply.(type) {
case *roachpb.ScanResponse:
rows = r.Rows
case *roachpb.ReverseScanResponse:
rows = r.Rows
default:
t.Fatalf("unexpected response %T: %v", reply, reply)
}
if l := len(rows); l != 1 {
t.Fatalf("%d: expected 1 row; got %d\n%s", i, l, rows)
}
if key := string(rows[0].Key); keys[0] != key {
t.Errorf("expected key %q; got %q", keys[0], key)
}
}
}
// TestTimeSeriesMaintenanceQueue verifies shouldQueue and process method
// pass the correct data to the store's TimeSeriesData
func TestTimeSeriesMaintenanceQueue(t *testing.T) {
defer leaktest.AfterTest(t)()
model := &modelTimeSeriesDataStore{
t: t,
pruneSeenStartKeys: make(map[string]struct{}),
pruneSeenEndKeys: make(map[string]struct{}),
}
cfg := storage.TestStoreConfig(nil)
cfg.TimeSeriesDataStore = model
cfg.TestingKnobs.DisableScanner = true
cfg.TestingKnobs.DisableSplitQueue = true
store, stopper := createTestStoreWithConfig(t, cfg)
defer stopper.Stop()
// Generate several splits.
splitKeys := []roachpb.Key{roachpb.Key("c"), roachpb.Key("b"), roachpb.Key("a")}
for _, k := range splitKeys {
repl := store.LookupReplica(roachpb.RKey(k), nil)
args := adminSplitArgs(k, k)
if _, pErr := client.SendWrappedWith(context.Background(), store, roachpb.Header{
RangeID: repl.RangeID,
}, &args); pErr != nil {
t.Fatal(pErr)
}
}
// Generate a list of start/end keys the model should have been passed by
// the queue. This consists of all split keys, with KeyMin as an additional
// start and KeyMax as an additional end.
expectedStartKeys := make(map[string]struct{})
expectedEndKeys := make(map[string]struct{})
expectedStartKeys[roachpb.KeyMin.String()] = struct{}{}
expectedEndKeys[roachpb.KeyMax.String()] = struct{}{}
for _, expected := range splitKeys {
expectedStartKeys[expected.String()] = struct{}{}
expectedEndKeys[expected.String()] = struct{}{}
}
// Wait for splits to complete and system config to be available.
util.SucceedsSoon(t, func() error {
if a, e := store.ReplicaCount(), len(expectedEndKeys); a != e {
return fmt.Errorf("expected %d replicas in store; found %d", a, e)
}
if _, ok := store.Gossip().GetSystemConfig(); !ok {
return fmt.Errorf("system config not yet available")
}
return nil
})
// Force replica scan to run, which will populate the model.
store.ForceTimeSeriesMaintenanceQueueProcess()
// Wait for processing to complete.
util.SucceedsSoon(t, func() error {
model.Lock()
defer model.Unlock()
if a, e := model.containsCalled, len(expectedStartKeys); a != e {
return fmt.Errorf("ContainsTimeSeries called %d times; expected %d", a, e)
}
if a, e := model.pruneCalled, len(expectedStartKeys); a != e {
return fmt.Errorf("PruneTimeSeries called %d times; expected %d", a, e)
}
return nil
})
model.Lock()
defer model.Unlock()
if a, e := model.pruneSeenStartKeys, expectedStartKeys; !reflect.DeepEqual(a, e) {
t.Errorf("start keys seen by PruneTimeSeries did not match expectation: %s", pretty.Diff(a, e))
}
if a, e := model.pruneSeenEndKeys, expectedEndKeys; !reflect.DeepEqual(a, e) {
t.Errorf("end keys seen by PruneTimeSeries did not match expectation: %s", pretty.Diff(a, e))
}
}