// TestTimestampSelectionInOptions verifies that a client can set the
// Txn timestamp using client.TxnExecOptions.
func TestTimestampSelectionInOptions(t *testing.T) {
defer leaktest.AfterTest(t)()
db := NewDB(newTestSender(nil, nil))
txn := NewTxn(context.Background(), *db)
mc := hlc.NewManualClock(100)
clock := hlc.NewClock(mc.UnixNano, time.Nanosecond)
execOpt := TxnExecOptions{
Clock: clock,
}
refTimestamp := clock.Now()
txnClosure := func(txn *Txn, opt *TxnExecOptions) error {
// Ensure the KV transaction is created.
return txn.Put("a", "b")
}
if err := txn.Exec(execOpt, txnClosure); err != nil {
t.Fatal(err)
}
// Check the timestamp was initialized.
if txn.Proto.OrigTimestamp.WallTime != refTimestamp.WallTime {
t.Errorf("expected txn orig ts to be %s; got %s", refTimestamp, txn.Proto.OrigTimestamp)
}
}
func TestClockOffsetMetrics(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
clock := hlc.NewClock(hlc.NewManualClock(123).UnixNano, 20*time.Nanosecond)
monitor := newRemoteClockMonitor(clock, time.Hour)
monitor.mu.offsets = map[string]RemoteOffset{
"0": {
Offset: 13,
Uncertainty: 7,
MeasuredAt: 6,
},
}
if err := monitor.VerifyClockOffset(context.TODO()); err != nil {
t.Fatal(err)
}
if a, e := monitor.Metrics().ClockOffsetMeanNanos.Value(), int64(13); a != e {
t.Errorf("mean %d != expected %d", a, e)
}
if a, e := monitor.Metrics().ClockOffsetStdDevNanos.Value(), int64(7); a != e {
t.Errorf("stdDev %d != expected %d", a, e)
}
}
func TestReacquireLease(t *testing.T) {
defer leaktest.AfterTest(t)()
s, db := setup(t)
defer s.Stopper().Stop()
ctx := context.Background()
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, time.Nanosecond)
lm := client.NewLeaseManager(db, clock, client.LeaseManagerOptions{ClientID: clientID1})
l, err := lm.AcquireLease(ctx, leaseKey)
if err != nil {
t.Fatal(err)
}
// We allow re-acquiring the same lease as long as the client ID is
// the same to allow a client to reacquire its own leases rather than
// having to wait them out if it crashes and restarts.
l, err = lm.AcquireLease(ctx, leaseKey)
if err != nil {
t.Fatal(err)
}
if err := lm.ReleaseLease(ctx, l); err != nil {
t.Fatal(err)
}
}
// TestTimestampCacheNoEviction verifies that even after
// the MinTSCacheWindow interval, if the cache has not hit
// its size threshold, it will not evict entries.
func TestTimestampCacheNoEviction(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, time.Nanosecond)
tc := newTimestampCache(clock)
// Increment time to the low water mark + 1.
manual.Increment(1)
aTS := clock.Now()
tc.add(roachpb.Key("a"), nil, aTS, nil, true)
tc.AddRequest(cacheRequest{
reads: []roachpb.Span{{Key: roachpb.Key("c")}},
timestamp: aTS,
})
// Increment time by the MinTSCacheWindow and add another key.
manual.Increment(MinTSCacheWindow.Nanoseconds())
tc.add(roachpb.Key("b"), nil, clock.Now(), nil, true)
tc.AddRequest(cacheRequest{
reads: []roachpb.Span{{Key: roachpb.Key("d")}},
timestamp: clock.Now(),
})
// Verify that the cache still has 4 entries in it
if l, want := tc.len(), 4; l != want {
t.Errorf("expected %d entries to remain, got %d", want, l)
}
}
func TestTimestampCacheClear(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, time.Nanosecond)
tc := newTimestampCache(clock)
key := roachpb.Key("a")
ts := clock.Now()
tc.add(key, nil, ts, nil, true)
manual.Increment(5000000)
expTS := clock.Now()
// Clear the cache, which will reset the low water mark to
// the current time.
tc.Clear(expTS)
// Fetching any keys should give current time.
if rTS, _, ok := tc.GetMaxRead(key, nil); ok {
t.Errorf("expected %s to have cleared timestamp", key)
} else if !rTS.Equal(expTS) {
t.Errorf("expected %s, got %s", rTS, expTS)
}
}
// TestStoreRangeMergeStats starts by splitting a range, then writing random data
// to both sides of the split. It then merges the ranges and verifies the merged
// range has stats consistent with recomputations.
func TestStoreRangeMergeStats(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(123)
storeCfg := storage.TestStoreConfig(hlc.NewClock(manual.UnixNano, time.Nanosecond))
storeCfg.TestingKnobs.DisableSplitQueue = true
store, stopper := createTestStoreWithConfig(t, storeCfg)
defer stopper.Stop()
// Split the range.
aDesc, bDesc, pErr := createSplitRanges(store)
if pErr != nil {
t.Fatal(pErr)
}
// Write some values left and right of the proposed split key.
writeRandomDataToRange(t, store, aDesc.RangeID, []byte("aaa"))
writeRandomDataToRange(t, store, bDesc.RangeID, []byte("ccc"))
// Get the range stats for both ranges now that we have data.
snap := store.Engine().NewSnapshot()
defer snap.Close()
msA, err := engine.MVCCGetRangeStats(context.Background(), snap, aDesc.RangeID)
if err != nil {
t.Fatal(err)
}
msB, err := engine.MVCCGetRangeStats(context.Background(), snap, bDesc.RangeID)
if err != nil {
t.Fatal(err)
}
// Stats should agree with recomputation.
if err := verifyRecomputedStats(snap, aDesc, msA, manual.UnixNano()); err != nil {
t.Fatalf("failed to verify range A's stats before split: %v", err)
}
if err := verifyRecomputedStats(snap, bDesc, msB, manual.UnixNano()); err != nil {
t.Fatalf("failed to verify range B's stats before split: %v", err)
}
manual.Increment(100)
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
if _, err := client.SendWrapped(context.Background(), rg1(store), &args); err != nil {
t.Fatal(err)
}
replMerged := store.LookupReplica(aDesc.StartKey, nil)
// Get the range stats for the merged range and verify.
snap = store.Engine().NewSnapshot()
defer snap.Close()
msMerged, err := engine.MVCCGetRangeStats(context.Background(), snap, replMerged.RangeID)
if err != nil {
t.Fatal(err)
}
// Merged stats should agree with recomputation.
if err := verifyRecomputedStats(snap, replMerged.Desc(), msMerged, manual.UnixNano()); err != nil {
t.Errorf("failed to verify range's stats after merge: %v", err)
}
}
func TestVerifyClockOffset(t *testing.T) {
defer leaktest.AfterTest(t)()
clock := hlc.NewClock(hlc.NewManualClock(123).UnixNano, 50*time.Nanosecond)
monitor := newRemoteClockMonitor(clock, time.Hour)
for idx, tc := range []struct {
offsets []RemoteOffset
expectedError bool
}{
// no error if no offsets.
{[]RemoteOffset{}, false},
// no error when a majority of offsets are under the maximum tolerated offset.
{[]RemoteOffset{{Offset: 20, Uncertainty: 10}, {Offset: 48, Uncertainty: 20}, {Offset: 61, Uncertainty: 25}, {Offset: 91, Uncertainty: 31}}, false},
// error when less than a majority of offsets are under the maximum tolerated offset.
{[]RemoteOffset{{Offset: 20, Uncertainty: 10}, {Offset: 58, Uncertainty: 20}, {Offset: 85, Uncertainty: 25}, {Offset: 91, Uncertainty: 31}}, true},
} {
monitor.mu.offsets = make(map[string]RemoteOffset)
for i, offset := range tc.offsets {
monitor.mu.offsets[strconv.Itoa(i)] = offset
}
if tc.expectedError {
if err := monitor.VerifyClockOffset(context.TODO()); !testutils.IsError(err, errOffsetGreaterThanMaxOffset) {
t.Errorf("%d: unexpected error %v", idx, err)
}
} else {
if err := monitor.VerifyClockOffset(context.TODO()); err != nil {
t.Errorf("%d: unexpected error %s", idx, err)
}
}
}
}
func TestHeartbeatReply(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(5)
clock := hlc.NewClock(manual.UnixNano, time.Nanosecond)
heartbeat := &HeartbeatService{
clock: clock,
remoteClockMonitor: newRemoteClockMonitor(clock, time.Hour),
}
request := &PingRequest{
Ping: "testPing",
}
response, err := heartbeat.Ping(context.Background(), request)
if err != nil {
t.Fatal(err)
}
if response.Pong != request.Ping {
t.Errorf("expected %s to be equal to %s", response.Pong, request.Ping)
}
if response.ServerTime != 5 {
t.Errorf("expected server time 5, instead %d", response.ServerTime)
}
}
// TestScannerTiming verifies that ranges are scanned, regardless
// of how many, to match scanInterval.
func TestScannerTiming(t *testing.T) {
defer leaktest.AfterTest(t)()
const count = 3
const runTime = 100 * time.Millisecond
const maxError = 7500 * time.Microsecond
durations := []time.Duration{
15 * time.Millisecond,
25 * time.Millisecond,
}
for i, duration := range durations {
testutils.SucceedsSoon(t, func() error {
ranges := newTestRangeSet(count, t)
q := &testQueue{}
s := newReplicaScanner(log.AmbientContext{}, duration, 0, ranges)
s.AddQueues(q)
mc := hlc.NewManualClock(123)
clock := hlc.NewClock(mc.UnixNano, time.Nanosecond)
stopper := stop.NewStopper()
s.Start(clock, stopper)
time.Sleep(runTime)
stopper.Stop()
avg := s.avgScan()
log.Infof(context.Background(), "%d: average scan: %s", i, avg)
if avg.Nanoseconds()-duration.Nanoseconds() > maxError.Nanoseconds() ||
duration.Nanoseconds()-avg.Nanoseconds() > maxError.Nanoseconds() {
return errors.Errorf("expected %s, got %s: exceeds max error of %s", duration, avg, maxError)
}
return nil
})
}
}
func TestAcquireAndRelease(t *testing.T) {
defer leaktest.AfterTest(t)()
s, db := setup(t)
defer s.Stopper().Stop()
ctx := context.Background()
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, time.Nanosecond)
lm := client.NewLeaseManager(db, clock, client.LeaseManagerOptions{ClientID: clientID1})
l, err := lm.AcquireLease(ctx, leaseKey)
if err != nil {
t.Fatal(err)
}
if err := lm.ReleaseLease(ctx, l); err != nil {
t.Fatal(err)
}
if err := lm.ReleaseLease(ctx, l); !testutils.IsError(err, "unexpected value") {
t.Fatal(err)
}
l, err = lm.AcquireLease(ctx, leaseKey)
if err != nil {
t.Fatal(err)
}
if err := lm.ReleaseLease(ctx, l); err != nil {
t.Fatal(err)
}
}
// TestTxnCoordSenderSingleRoundtripTxn checks that a batch which completely
// holds the writing portion of a Txn (including EndTransaction) does not
// launch a heartbeat goroutine at all.
func TestTxnCoordSenderSingleRoundtripTxn(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, 20*time.Nanosecond)
senderFunc := func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
br := ba.CreateReply()
txnClone := ba.Txn.Clone()
br.Txn = &txnClone
br.Txn.Writing = true
return br, nil
}
ambient := log.AmbientContext{Tracer: tracing.NewTracer()}
ts := NewTxnCoordSender(
ambient, senderFn(senderFunc), clock, false, stopper, MakeTxnMetrics(metric.TestSampleInterval),
)
// Stop the stopper manually, prior to trying the transaction. This has the
// effect of returning a NodeUnavailableError for any attempts at launching
// a heartbeat goroutine.
stopper.Stop()
var ba roachpb.BatchRequest
key := roachpb.Key("test")
ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.EndTransactionRequest{})
ba.Txn = &roachpb.Transaction{Name: "test"}
_, pErr := ts.Send(context.Background(), ba)
if pErr != nil {
t.Fatal(pErr)
}
}
// TestTimestampCacheEqualTimestamp verifies that in the event of two
// non-overlapping transactions with equal timestamps, the returned
// timestamp is not owned by either one.
func TestTimestampCacheEqualTimestamps(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, time.Nanosecond)
tc := newTimestampCache(clock)
txn1 := uuid.MakeV4()
txn2 := uuid.MakeV4()
// Add two non-overlapping transactions at the same timestamp.
ts1 := clock.Now()
tc.add(roachpb.Key("a"), roachpb.Key("b"), ts1, &txn1, true)
tc.add(roachpb.Key("b"), roachpb.Key("c"), ts1, &txn2, true)
// When querying either side separately, the transaction ID is returned.
if ts, txn, _ := tc.GetMaxRead(roachpb.Key("a"), roachpb.Key("b")); !ts.Equal(ts1) {
t.Errorf("expected 'a'-'b' to have timestamp %s, but found %s", ts1, ts)
} else if *txn != txn1 {
t.Errorf("expected 'a'-'b' to have txn id %s, but found %s", txn1, txn)
}
if ts, txn, _ := tc.GetMaxRead(roachpb.Key("b"), roachpb.Key("c")); !ts.Equal(ts1) {
t.Errorf("expected 'b'-'c' to have timestamp %s, but found %s", ts1, ts)
} else if *txn != txn2 {
t.Errorf("expected 'b'-'c' to have txn id %s, but found %s", txn2, txn)
}
// Querying a span that overlaps both returns a nil txn ID; neither
// can proceed here.
if ts, txn, _ := tc.GetMaxRead(roachpb.Key("a"), roachpb.Key("c")); !ts.Equal(ts1) {
t.Errorf("expected 'a'-'c' to have timestamp %s, but found %s", ts1, ts)
} else if txn != nil {
t.Errorf("expected 'a'-'c' to have nil txn id, but found %s", txn)
}
}
// TestTxnCoordSenderErrorWithIntent validates that if a transactional request
// returns an error but also indicates a Writing transaction, the coordinator
// tracks it just like a successful request.
func TestTxnCoordSenderErrorWithIntent(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(20)
testCases := []struct {
roachpb.Error
errMsg string
}{
{*roachpb.NewError(roachpb.NewTransactionRetryError()), "retry txn"},
{*roachpb.NewError(roachpb.NewTransactionPushError(roachpb.Transaction{
TxnMeta: enginepb.TxnMeta{
ID: uuid.NewV4(),
}})), "failed to push"},
{*roachpb.NewErrorf("testError"), "testError"},
}
for i, test := range testCases {
func() {
senderFunc := func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
txn := ba.Txn.Clone()
txn.Writing = true
pErr := &roachpb.Error{}
*pErr = test.Error
pErr.SetTxn(&txn)
return nil, pErr
}
ambient := log.AmbientContext{Tracer: tracing.NewTracer()}
ts := NewTxnCoordSender(
ambient,
senderFn(senderFunc),
clock,
false,
stopper,
MakeTxnMetrics(metric.TestSampleInterval),
)
var ba roachpb.BatchRequest
key := roachpb.Key("test")
ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.EndTransactionRequest{})
ba.Txn = &roachpb.Transaction{Name: "test"}
_, pErr := ts.Send(context.Background(), ba)
if !testutils.IsPError(pErr, test.errMsg) {
t.Errorf("%d: error did not match %s: %v", i, test.errMsg, pErr)
}
defer teardownHeartbeats(ts)
ts.Lock()
defer ts.Unlock()
if len(ts.txns) != 1 {
t.Errorf("%d: expected transaction to be tracked", i)
}
}()
}
}
func TestLeasesMultipleClients(t *testing.T) {
defer leaktest.AfterTest(t)()
s, db := setup(t)
defer s.Stopper().Stop()
ctx := context.Background()
manual1 := hlc.NewManualClock(123)
clock1 := hlc.NewClock(manual1.UnixNano, time.Nanosecond)
manual2 := hlc.NewManualClock(123)
clock2 := hlc.NewClock(manual2.UnixNano, time.Nanosecond)
lm1 := client.NewLeaseManager(db, clock1, client.LeaseManagerOptions{ClientID: clientID1})
lm2 := client.NewLeaseManager(db, clock2, client.LeaseManagerOptions{ClientID: clientID2})
l1, err := lm1.AcquireLease(ctx, leaseKey)
if err != nil {
t.Fatal(err)
}
_, err = lm2.AcquireLease(ctx, leaseKey)
if !testutils.IsError(err, "is not available until") {
t.Fatalf("didn't get expected error trying to acquire already held lease: %v", err)
}
if _, ok := err.(*client.LeaseNotAvailableError); !ok {
t.Fatalf("expected LeaseNotAvailableError, got %v", err)
}
// Ensure a lease can be "stolen" after it's expired.
manual2.Increment(int64(client.DefaultLeaseDuration) + 1)
l2, err := lm2.AcquireLease(ctx, leaseKey)
if err != nil {
t.Fatal(err)
}
// lm1's clock indicates that its lease should still be valid, but it doesn't
// own it anymore.
manual1.Increment(int64(client.DefaultLeaseDuration) / 2)
if err := lm1.ExtendLease(ctx, l1); !testutils.IsError(err, "out of sync with DB state") {
t.Fatalf("didn't get expected error trying to extend expired lease: %v", err)
}
if err := lm1.ReleaseLease(ctx, l1); !testutils.IsError(err, "unexpected value") {
t.Fatalf("didn't get expected error trying to release stolen lease: %v", err)
}
if err := lm2.ReleaseLease(ctx, l2); err != nil {
t.Fatal(err)
}
}
// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester, baseCtx *base.Config, initSender InitSenderFn) {
ambient := log.AmbientContext{Tracer: tracing.NewTracer()}
nc := &base.NodeIDContainer{}
ambient.AddLogTag("n", nc)
nodeID := roachpb.NodeID(1)
nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID}
ltc.tester = t
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = stop.NewStopper()
rpcContext := rpc.NewContext(ambient, baseCtx, ltc.Clock, ltc.Stopper)
server := rpc.NewServer(rpcContext) // never started
ltc.Gossip = gossip.New(ambient, nc, rpcContext, server, nil, ltc.Stopper, metric.NewRegistry())
ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20)
ltc.Stopper.AddCloser(ltc.Eng)
ltc.Stores = storage.NewStores(ambient, ltc.Clock)
ltc.Sender = initSender(nodeDesc, ambient.Tracer, ltc.Clock, ltc.Latency, ltc.Stores, ltc.Stopper,
ltc.Gossip)
if ltc.DBContext == nil {
dbCtx := client.DefaultDBContext()
ltc.DBContext = &dbCtx
}
ltc.DB = client.NewDBWithContext(ltc.Sender, *ltc.DBContext)
transport := storage.NewDummyRaftTransport()
cfg := storage.TestStoreConfig()
if ltc.RangeRetryOptions != nil {
cfg.RangeRetryOptions = *ltc.RangeRetryOptions
}
cfg.AmbientCtx = ambient
cfg.Clock = ltc.Clock
cfg.DB = ltc.DB
cfg.Gossip = ltc.Gossip
cfg.Transport = transport
cfg.MetricsSampleInterval = metric.TestSampleInterval
ltc.Store = storage.NewStore(cfg, ltc.Eng, nodeDesc)
if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.Stores.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(nil); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(context.Background(), ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
nc.Set(context.TODO(), nodeDesc.NodeID)
if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil {
t.Fatalf("unable to set node descriptor: %s", err)
}
}
// TestScannerDisabled verifies that disabling a scanner prevents
// replicas from being added to queues.
func TestScannerDisabled(t *testing.T) {
defer leaktest.AfterTest(t)()
const count = 3
ranges := newTestRangeSet(count, t)
q := &testQueue{}
s := newReplicaScanner(log.AmbientContext{}, 1*time.Millisecond, 0, ranges)
s.AddQueues(q)
mc := hlc.NewManualClock(123)
clock := hlc.NewClock(mc.UnixNano, time.Nanosecond)
stopper := stop.NewStopper()
s.Start(clock, stopper)
defer stopper.Stop()
// Verify queue gets all ranges.
testutils.SucceedsSoon(t, func() error {
if q.count() != count {
return errors.Errorf("expected %d replicas; have %d", count, q.count())
}
if s.scanCount() == 0 {
return errors.Errorf("expected scanner count to increment")
}
return nil
})
lastWaitEnabledCount := s.waitEnabledCount()
// Now, disable the scanner.
s.SetDisabled(true)
testutils.SucceedsSoon(t, func() error {
if s.waitEnabledCount() == lastWaitEnabledCount {
return errors.Errorf("expected scanner to stop when disabled")
}
return nil
})
lastScannerCount := s.scanCount()
// Remove the replicas and verify the scanner still removes them while disabled.
ranges.Visit(func(repl *Replica) bool {
s.RemoveReplica(repl)
return true
})
testutils.SucceedsSoon(t, func() error {
if qc := q.count(); qc != 0 {
return errors.Errorf("expected queue to be empty after replicas removed from scanner; got %d", qc)
}
return nil
})
if sc := s.scanCount(); sc != lastScannerCount {
t.Errorf("expected scanner count to not increment: %d != %d", sc, lastScannerCount)
}
}
// TestRangeCommandClockUpdate verifies that followers update their
// clocks when executing a command, even if the lease holder's clock is far
// in the future.
func TestRangeCommandClockUpdate(t *testing.T) {
defer leaktest.AfterTest(t)()
const numNodes = 3
var manuals []*hlc.ManualClock
var clocks []*hlc.Clock
for i := 0; i < numNodes; i++ {
manuals = append(manuals, hlc.NewManualClock(1))
clocks = append(clocks, hlc.NewClock(manuals[i].UnixNano))
clocks[i].SetMaxOffset(100 * time.Millisecond)
}
mtc := &multiTestContext{clocks: clocks}
mtc.Start(t, numNodes)
defer mtc.Stop()
mtc.replicateRange(1, 1, 2)
// Advance the lease holder's clock ahead of the followers (by more than
// MaxOffset but less than the range lease) and execute a command.
manuals[0].Increment(int64(500 * time.Millisecond))
incArgs := incrementArgs([]byte("a"), 5)
ts := clocks[0].Now()
if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err != nil {
t.Fatal(err)
}
// Wait for that command to execute on all the followers.
util.SucceedsSoon(t, func() error {
values := []int64{}
for _, eng := range mtc.engines {
val, _, err := engine.MVCCGet(context.Background(), eng, roachpb.Key("a"), clocks[0].Now(), true, nil)
if err != nil {
return err
}
values = append(values, mustGetInt(val))
}
if !reflect.DeepEqual(values, []int64{5, 5, 5}) {
return errors.Errorf("expected (5, 5, 5), got %v", values)
}
return nil
})
// Verify that all the followers have accepted the clock update from
// node 0 even though it comes from outside the usual max offset.
now := clocks[0].Now()
for i, clock := range clocks {
// Only compare the WallTimes: it's normal for clock 0 to be a few logical ticks ahead.
if clock.Now().WallTime < now.WallTime {
t.Errorf("clock %d is behind clock 0: %s vs %s", i, clock.Now(), now)
}
}
}
// TestRejectFutureCommand verifies that lease holders reject commands that
// would cause a large time jump.
func TestRejectFutureCommand(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, 100*time.Millisecond)
mtc := &multiTestContext{clock: clock}
mtc.Start(t, 1)
defer mtc.Stop()
ts1 := clock.Now()
key := roachpb.Key("a")
incArgs := incrementArgs(key, 5)
// Commands with a future timestamp that is within the MaxOffset
// bound will be accepted and will cause the clock to advance.
const numCmds = 3
clockOffset := clock.MaxOffset() / numCmds
for i := int64(1); i <= numCmds; i++ {
ts := ts1.Add(i*clockOffset.Nanoseconds(), 0)
if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err != nil {
t.Fatal(err)
}
}
ts2 := clock.Now()
if expAdvance, advance := ts2.GoTime().Sub(ts1.GoTime()), numCmds*clockOffset; advance != expAdvance {
t.Fatalf("expected clock to advance %s; got %s", expAdvance, advance)
}
// Once the accumulated offset reaches MaxOffset, commands will be rejected.
_, pErr := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts1.Add(clock.MaxOffset().Nanoseconds()+1, 0)}, &incArgs)
if !testutils.IsPError(pErr, "rejecting command with timestamp in the future") {
t.Fatalf("unexpected error %v", pErr)
}
// The clock did not advance and the final command was not executed.
ts3 := clock.Now()
if advance := ts3.GoTime().Sub(ts2.GoTime()); advance != 0 {
t.Fatalf("expected clock not to advance, but it advanced by %s", advance)
}
val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], key, ts3, true, nil)
if err != nil {
t.Fatal(err)
}
if a, e := mustGetInt(val), incArgs.Increment*numCmds; a != e {
t.Errorf("expected %d, got %d", e, a)
}
}
func TestRemoveLeaseIfExpiring(t *testing.T) {
defer leaktest.AfterTest(t)()
p := planner{session: &Session{context: context.Background()}}
mc := hlc.NewManualClock(123)
p.leaseMgr = &LeaseManager{LeaseStore: LeaseStore{clock: hlc.NewClock(mc.UnixNano, time.Nanosecond)}}
p.leases = make([]*LeaseState, 0)
txn := client.Txn{Context: context.Background()}
p.setTxn(&txn)
if p.removeLeaseIfExpiring(nil) {
t.Error("expected false with nil input")
}
// Add a lease to the planner.
d := int64(LeaseDuration)
l1 := &LeaseState{expiration: parser.DTimestamp{Time: time.Unix(0, mc.UnixNano()+d+1)}}
p.leases = append(p.leases, l1)
et := hlc.Timestamp{WallTime: l1.Expiration().UnixNano()}
txn.UpdateDeadlineMaybe(et)
if p.removeLeaseIfExpiring(l1) {
t.Error("expected false with a non-expiring lease")
}
if !p.txn.GetDeadline().Equal(et) {
t.Errorf("expected deadline %s but got %s", et, p.txn.GetDeadline())
}
// Advance the clock so that l1 will be expired.
mc.Increment(d + 1)
// Add another lease.
l2 := &LeaseState{expiration: parser.DTimestamp{Time: time.Unix(0, mc.UnixNano()+d+1)}}
p.leases = append(p.leases, l2)
if !p.removeLeaseIfExpiring(l1) {
t.Error("expected true with an expiring lease")
}
et = hlc.Timestamp{WallTime: l2.Expiration().UnixNano()}
txn.UpdateDeadlineMaybe(et)
if !(len(p.leases) == 1 && p.leases[0] == l2) {
t.Errorf("expected leases to contain %s but has %s", l2, p.leases)
}
if !p.txn.GetDeadline().Equal(et) {
t.Errorf("expected deadline %s, but got %s", et, p.txn.GetDeadline())
}
}
// TestScannerEmptyRangeSet verifies that an empty range set doesn't busy loop.
func TestScannerEmptyRangeSet(t *testing.T) {
defer leaktest.AfterTest(t)()
ranges := newTestRangeSet(0, t)
q := &testQueue{}
s := newReplicaScanner(log.AmbientContext{}, time.Hour, 0, ranges)
s.AddQueues(q)
mc := hlc.NewManualClock(123)
clock := hlc.NewClock(mc.UnixNano, time.Nanosecond)
stopper := stop.NewStopper()
defer stopper.Stop()
s.Start(clock, stopper)
time.Sleep(time.Millisecond) // give it some time to (not) busy loop
if count := s.scanCount(); count > 1 {
t.Errorf("expected at most one loop, but got %d", count)
}
}
// createTestBookie creates a new bookie, stopper and manual clock for testing.
func createTestBookie(
reservationTimeout time.Duration, maxReservations int, maxReservedBytes int64,
) (*stop.Stopper, *hlc.ManualClock, *bookie) {
stopper := stop.NewStopper()
mc := hlc.NewManualClock(0)
clock := hlc.NewClock(mc.UnixNano)
b := newBookie(clock, stopper, newStoreMetrics(time.Hour), reservationTimeout)
// Lock the bookie to prevent the main loop from running as we change some
// of the bookie's state.
b.mu.Lock()
defer b.mu.Unlock()
b.maxReservations = maxReservations
b.maxReservedBytes = maxReservedBytes
// Set a high number for a mocked total available space.
b.metrics.Available.Update(defaultMaxReservedBytes * 10)
return stopper, mc, b
}
// TestTimestampCacheSetLowWater verifies that setting the low
// water mark moves max timestamps forward as appropriate.
func TestTimestampCacheSetLowWater(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, time.Nanosecond)
tc := newTimestampCache(clock)
// Increment time to the low water mark + 10.
manual.Increment(10)
aTS := clock.Now()
tc.add(roachpb.Key("a"), nil, aTS, nil, true)
// Increment time by 10ns and add another key.
manual.Increment(10)
bTS := clock.Now()
tc.add(roachpb.Key("b"), nil, bTS, nil, true)
// Increment time by 10ns and add another key.
manual.Increment(10)
cTS := clock.Now()
tc.add(roachpb.Key("c"), nil, cTS, nil, true)
// Set low water mark.
tc.SetLowWater(bTS)
// Verify looking up key "a" returns the new low water mark ("a"'s timestamp).
for i, test := range []struct {
key roachpb.Key
expTS hlc.Timestamp
expOK bool
}{
{roachpb.Key("a"), bTS, false},
{roachpb.Key("b"), bTS, false},
{roachpb.Key("c"), cTS, true},
{roachpb.Key("d"), bTS, false},
} {
if rTS, _, ok := tc.GetMaxRead(test.key, nil); !rTS.Equal(test.expTS) || ok != test.expOK {
t.Errorf("%d: expected ts %s, got %s; exp ok=%t; got %t", i, test.expTS, rTS, test.expOK, ok)
}
}
// Try setting a lower low water mark than the previous value.
tc.SetLowWater(aTS)
if rTS, _, ok := tc.GetMaxRead(roachpb.Key("d"), nil); !rTS.Equal(bTS) || ok {
t.Errorf("setting lower low water mark should not be allowed; expected %s; got %s; ok=%t", bTS, rTS, ok)
}
}
// TestRejectFutureCommand verifies that lease holders reject commands that
// would cause a large time jump.
func TestRejectFutureCommand(t *testing.T) {
defer leaktest.AfterTest(t)()
const maxOffset = 100 * time.Millisecond
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(maxOffset)
mtc := &multiTestContext{clock: clock}
mtc.Start(t, 1)
defer mtc.Stop()
startTime := manual.UnixNano()
// Commands with a future timestamp that is within the MaxOffset
// bound will be accepted and will cause the clock to advance.
for i := int64(0); i < 3; i++ {
incArgs := incrementArgs([]byte("a"), 5)
ts := hlc.ZeroTimestamp.Add(startTime+((i+1)*30)*int64(time.Millisecond), 0)
if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err != nil {
t.Fatal(err)
}
}
if now := clock.Now(); now.WallTime != int64(90*time.Millisecond) {
t.Fatalf("expected clock to advance to 90ms; got %s", now)
}
// Once the accumulated offset reaches MaxOffset, commands will be rejected.
incArgs := incrementArgs([]byte("a"), 11)
ts := hlc.ZeroTimestamp.Add(int64((time.Duration(startTime)+maxOffset+1)*time.Millisecond), 0)
if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err == nil {
t.Fatalf("expected clock offset error but got nil")
}
// The clock remained at 90ms and the final command was not executed.
if now := clock.Now(); now.WallTime != int64(90*time.Millisecond) {
t.Errorf("expected clock to stay at 90ms; got %s", now)
}
val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], roachpb.Key("a"), clock.Now(), true, nil)
if err != nil {
t.Fatal(err)
}
if v := mustGetInt(val); v != 15 {
t.Errorf("expected 15, got %v", v)
}
}
// TestScannerAddToQueues verifies that ranges are added to and
// removed from multiple queues.
func TestScannerAddToQueues(t *testing.T) {
defer leaktest.AfterTest(t)()
const count = 3
ranges := newTestRangeSet(count, t)
q1, q2 := &testQueue{}, &testQueue{}
// We don't want to actually consume entries from the queues during this test.
q1.setDisabled(true)
q2.setDisabled(true)
s := newReplicaScanner(log.AmbientContext{}, 1*time.Millisecond, 0, ranges)
s.AddQueues(q1, q2)
mc := hlc.NewManualClock(123)
clock := hlc.NewClock(mc.UnixNano, time.Nanosecond)
stopper := stop.NewStopper()
// Start scanner and verify that all ranges are added to both queues.
s.Start(clock, stopper)
testutils.SucceedsSoon(t, func() error {
if q1.count() != count || q2.count() != count {
return errors.Errorf("q1 or q2 count != %d; got %d, %d", count, q1.count(), q2.count())
}
return nil
})
// Remove first range and verify it does not exist in either range.
rng := ranges.remove(0, t)
testutils.SucceedsSoon(t, func() error {
// This is intentionally inside the loop, otherwise this test races as
// our removal of the range may be processed before a stray re-queue.
// Removing on each attempt makes sure we clean this up as we retry.
s.RemoveReplica(rng)
c1 := q1.count()
c2 := q2.count()
if c1 != count-1 || c2 != count-1 {
return errors.Errorf("q1 or q2 count != %d; got %d, %d", count-1, c1, c2)
}
return nil
})
// Stop scanner and verify both queues are stopped.
stopper.Stop()
if !q1.isDone() || !q2.isDone() {
t.Errorf("expected all queues to stop; got %t, %t", q1.isDone(), q2.isDone())
}
}
// createTestStorePool creates a stopper, gossip and storePool for use in
// tests. Stopper must be stopped by the caller.
func createTestStorePool(
timeUntilStoreDead time.Duration,
) (*stop.Stopper, *gossip.Gossip, *hlc.ManualClock, *StorePool) {
stopper := stop.NewStopper()
mc := hlc.NewManualClock(0)
clock := hlc.NewClock(mc.UnixNano)
rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, clock, stopper)
server := rpc.NewServer(rpcContext) // never started
g := gossip.NewTest(1, rpcContext, server, nil, stopper, metric.NewRegistry())
storePool := NewStorePool(
log.AmbientContext{},
g,
clock,
rpcContext,
timeUntilStoreDead,
stopper,
)
return stopper, g, mc, storePool
}
// Test that the a txn gets a fresh OrigTimestamp with every retry.
func TestAbortedRetryRenewsTimestamp(t *testing.T) {
defer leaktest.AfterTest(t)()
// Create a TestSender that aborts a transaction 2 times before succeeding.
mc := hlc.NewManualClock(123)
clock := hlc.NewClock(mc.UnixNano, time.Nanosecond)
count := 0
db := NewDB(newTestSender(func(ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
if _, ok := ba.GetArg(roachpb.Put); ok {
mc.Increment(1)
count++
if count < 3 {
return nil, roachpb.NewError(&roachpb.TransactionAbortedError{})
}
}
return ba.CreateReply(), nil
}, nil))
txnClosure := func(txn *Txn, opt *TxnExecOptions) error {
// Ensure the KV transaction is created.
return txn.Put("a", "b")
}
txn := NewTxn(context.Background(), *db)
// Request a client-defined timestamp.
refTimestamp := clock.Now()
execOpt := TxnExecOptions{
AutoRetry: true,
AutoCommit: true,
Clock: clock,
}
// Perform the transaction.
if err := txn.Exec(execOpt, txnClosure); err != nil {
t.Fatal(err)
}
// Check the timestamp was preserved.
if txn.Proto.OrigTimestamp.WallTime == refTimestamp.WallTime {
t.Errorf("expected txn orig ts to be different than %s", refTimestamp)
}
}
func TestTimestampCacheClear(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
tc := newTimestampCache(clock)
manual.Set(5000000)
ts := clock.Now()
tc.add(roachpb.Key("a"), nil, ts, nil, true)
// Clear the cache, which will reset the low water mark to
// the current time.
tc.Clear(clock.Now())
// Fetching any keys should give current time.
expTS := clock.Timestamp()
if rTS, _, ok := tc.GetMaxRead(roachpb.Key("a"), nil); !rTS.Equal(expTS) || ok {
t.Errorf("expected \"a\" to have cleared timestamp; exp ok=false; got %t", ok)
}
}
// createStores creates a slice of count stores.
func createStores(count int, t *testing.T) (*hlc.ManualClock, []*Store, *Stores, *stop.Stopper) {
stopper := stop.NewStopper()
manual := hlc.NewManualClock(123)
cfg := TestStoreConfig(hlc.NewClock(manual.UnixNano, time.Nanosecond))
ls := NewStores(log.AmbientContext{}, cfg.Clock)
// Create two stores with ranges we care about.
stores := []*Store{}
for i := 0; i < 2; i++ {
cfg.Transport = NewDummyRaftTransport()
eng := engine.NewInMem(roachpb.Attributes{}, 1<<20)
stopper.AddCloser(eng)
s := NewStore(cfg, eng, &roachpb.NodeDescriptor{NodeID: 1})
storeIDAlloc++
s.Ident.StoreID = storeIDAlloc
stores = append(stores, s)
}
return manual, stores, ls, stopper
}
// createTestStorePool creates a stopper, gossip and storePool for use in
// tests. Stopper must be stopped by the caller.
func createTestStorePool(
timeUntilStoreDead time.Duration, deterministic bool, defaultNodeLiveness bool,
) (*stop.Stopper, *gossip.Gossip, *hlc.ManualClock, *StorePool, *mockNodeLiveness) {
stopper := stop.NewStopper()
mc := hlc.NewManualClock(123)
clock := hlc.NewClock(mc.UnixNano, time.Nanosecond)
rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, clock, stopper)
server := rpc.NewServer(rpcContext) // never started
g := gossip.NewTest(1, rpcContext, server, nil, stopper, metric.NewRegistry())
mnl := newMockNodeLiveness(defaultNodeLiveness)
storePool := NewStorePool(
log.AmbientContext{},
g,
clock,
mnl.nodeLivenessFunc,
timeUntilStoreDead,
deterministic,
)
return stopper, g, mc, storePool, mnl
}
func BenchmarkTimestampCacheInsertion(b *testing.B) {
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, time.Nanosecond)
tc := newTimestampCache(clock)
for i := 0; i < b.N; i++ {
tc.Clear(clock.Now())
cdTS := clock.Now()
tc.add(roachpb.Key("c"), roachpb.Key("d"), cdTS, nil, true)
beTS := clock.Now()
tc.add(roachpb.Key("b"), roachpb.Key("e"), beTS, nil, true)
adTS := clock.Now()
tc.add(roachpb.Key("a"), roachpb.Key("d"), adTS, nil, true)
cfTS := clock.Now()
tc.add(roachpb.Key("c"), roachpb.Key("f"), cfTS, nil, true)
}
}
