// TestTimestampSelectionInOptions verifies that a client can set the // Txn timestamp using client.TxnExecOptions. func TestTimestampSelectionInOptions(t *testing.T) { defer leaktest.AfterTest(t)() db := NewDB(newTestSender(nil, nil)) txn := NewTxn(context.Background(), *db) mc := hlc.NewManualClock(100) clock := hlc.NewClock(mc.UnixNano, time.Nanosecond) execOpt := TxnExecOptions{ Clock: clock, } refTimestamp := clock.Now() txnClosure := func(txn *Txn, opt *TxnExecOptions) error { // Ensure the KV transaction is created. return txn.Put("a", "b") } if err := txn.Exec(execOpt, txnClosure); err != nil { t.Fatal(err) } // Check the timestamp was initialized. if txn.Proto.OrigTimestamp.WallTime != refTimestamp.WallTime { t.Errorf("expected txn orig ts to be %s; got %s", refTimestamp, txn.Proto.OrigTimestamp) } }
func TestClockOffsetMetrics(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() clock := hlc.NewClock(hlc.NewManualClock(123).UnixNano, 20*time.Nanosecond) monitor := newRemoteClockMonitor(clock, time.Hour) monitor.mu.offsets = map[string]RemoteOffset{ "0": { Offset: 13, Uncertainty: 7, MeasuredAt: 6, }, } if err := monitor.VerifyClockOffset(context.TODO()); err != nil { t.Fatal(err) } if a, e := monitor.Metrics().ClockOffsetMeanNanos.Value(), int64(13); a != e { t.Errorf("mean %d != expected %d", a, e) } if a, e := monitor.Metrics().ClockOffsetStdDevNanos.Value(), int64(7); a != e { t.Errorf("stdDev %d != expected %d", a, e) } }
func TestReacquireLease(t *testing.T) { defer leaktest.AfterTest(t)() s, db := setup(t) defer s.Stopper().Stop() ctx := context.Background() manual := hlc.NewManualClock(123) clock := hlc.NewClock(manual.UnixNano, time.Nanosecond) lm := client.NewLeaseManager(db, clock, client.LeaseManagerOptions{ClientID: clientID1}) l, err := lm.AcquireLease(ctx, leaseKey) if err != nil { t.Fatal(err) } // We allow re-acquiring the same lease as long as the client ID is // the same to allow a client to reacquire its own leases rather than // having to wait them out if it crashes and restarts. l, err = lm.AcquireLease(ctx, leaseKey) if err != nil { t.Fatal(err) } if err := lm.ReleaseLease(ctx, l); err != nil { t.Fatal(err) } }
// TestTimestampCacheNoEviction verifies that even after // the MinTSCacheWindow interval, if the cache has not hit // its size threshold, it will not evict entries. func TestTimestampCacheNoEviction(t *testing.T) { defer leaktest.AfterTest(t)() manual := hlc.NewManualClock(123) clock := hlc.NewClock(manual.UnixNano, time.Nanosecond) tc := newTimestampCache(clock) // Increment time to the low water mark + 1. manual.Increment(1) aTS := clock.Now() tc.add(roachpb.Key("a"), nil, aTS, nil, true) tc.AddRequest(cacheRequest{ reads: []roachpb.Span{{Key: roachpb.Key("c")}}, timestamp: aTS, }) // Increment time by the MinTSCacheWindow and add another key. manual.Increment(MinTSCacheWindow.Nanoseconds()) tc.add(roachpb.Key("b"), nil, clock.Now(), nil, true) tc.AddRequest(cacheRequest{ reads: []roachpb.Span{{Key: roachpb.Key("d")}}, timestamp: clock.Now(), }) // Verify that the cache still has 4 entries in it if l, want := tc.len(), 4; l != want { t.Errorf("expected %d entries to remain, got %d", want, l) } }
func TestTimestampCacheClear(t *testing.T) { defer leaktest.AfterTest(t)() manual := hlc.NewManualClock(123) clock := hlc.NewClock(manual.UnixNano, time.Nanosecond) tc := newTimestampCache(clock) key := roachpb.Key("a") ts := clock.Now() tc.add(key, nil, ts, nil, true) manual.Increment(5000000) expTS := clock.Now() // Clear the cache, which will reset the low water mark to // the current time. tc.Clear(expTS) // Fetching any keys should give current time. if rTS, _, ok := tc.GetMaxRead(key, nil); ok { t.Errorf("expected %s to have cleared timestamp", key) } else if !rTS.Equal(expTS) { t.Errorf("expected %s, got %s", rTS, expTS) } }
// TestStoreRangeMergeStats starts by splitting a range, then writing random data // to both sides of the split. It then merges the ranges and verifies the merged // range has stats consistent with recomputations. func TestStoreRangeMergeStats(t *testing.T) { defer leaktest.AfterTest(t)() manual := hlc.NewManualClock(123) storeCfg := storage.TestStoreConfig(hlc.NewClock(manual.UnixNano, time.Nanosecond)) storeCfg.TestingKnobs.DisableSplitQueue = true store, stopper := createTestStoreWithConfig(t, storeCfg) defer stopper.Stop() // Split the range. aDesc, bDesc, pErr := createSplitRanges(store) if pErr != nil { t.Fatal(pErr) } // Write some values left and right of the proposed split key. writeRandomDataToRange(t, store, aDesc.RangeID, []byte("aaa")) writeRandomDataToRange(t, store, bDesc.RangeID, []byte("ccc")) // Get the range stats for both ranges now that we have data. snap := store.Engine().NewSnapshot() defer snap.Close() msA, err := engine.MVCCGetRangeStats(context.Background(), snap, aDesc.RangeID) if err != nil { t.Fatal(err) } msB, err := engine.MVCCGetRangeStats(context.Background(), snap, bDesc.RangeID) if err != nil { t.Fatal(err) } // Stats should agree with recomputation. if err := verifyRecomputedStats(snap, aDesc, msA, manual.UnixNano()); err != nil { t.Fatalf("failed to verify range A's stats before split: %v", err) } if err := verifyRecomputedStats(snap, bDesc, msB, manual.UnixNano()); err != nil { t.Fatalf("failed to verify range B's stats before split: %v", err) } manual.Increment(100) // Merge the b range back into the a range. args := adminMergeArgs(roachpb.KeyMin) if _, err := client.SendWrapped(context.Background(), rg1(store), &args); err != nil { t.Fatal(err) } replMerged := store.LookupReplica(aDesc.StartKey, nil) // Get the range stats for the merged range and verify. snap = store.Engine().NewSnapshot() defer snap.Close() msMerged, err := engine.MVCCGetRangeStats(context.Background(), snap, replMerged.RangeID) if err != nil { t.Fatal(err) } // Merged stats should agree with recomputation. if err := verifyRecomputedStats(snap, replMerged.Desc(), msMerged, manual.UnixNano()); err != nil { t.Errorf("failed to verify range's stats after merge: %v", err) } }
func TestVerifyClockOffset(t *testing.T) { defer leaktest.AfterTest(t)() clock := hlc.NewClock(hlc.NewManualClock(123).UnixNano, 50*time.Nanosecond) monitor := newRemoteClockMonitor(clock, time.Hour) for idx, tc := range []struct { offsets []RemoteOffset expectedError bool }{ // no error if no offsets. {[]RemoteOffset{}, false}, // no error when a majority of offsets are under the maximum tolerated offset. {[]RemoteOffset{{Offset: 20, Uncertainty: 10}, {Offset: 48, Uncertainty: 20}, {Offset: 61, Uncertainty: 25}, {Offset: 91, Uncertainty: 31}}, false}, // error when less than a majority of offsets are under the maximum tolerated offset. {[]RemoteOffset{{Offset: 20, Uncertainty: 10}, {Offset: 58, Uncertainty: 20}, {Offset: 85, Uncertainty: 25}, {Offset: 91, Uncertainty: 31}}, true}, } { monitor.mu.offsets = make(map[string]RemoteOffset) for i, offset := range tc.offsets { monitor.mu.offsets[strconv.Itoa(i)] = offset } if tc.expectedError { if err := monitor.VerifyClockOffset(context.TODO()); !testutils.IsError(err, errOffsetGreaterThanMaxOffset) { t.Errorf("%d: unexpected error %v", idx, err) } } else { if err := monitor.VerifyClockOffset(context.TODO()); err != nil { t.Errorf("%d: unexpected error %s", idx, err) } } } }
func TestHeartbeatReply(t *testing.T) { defer leaktest.AfterTest(t)() manual := hlc.NewManualClock(5) clock := hlc.NewClock(manual.UnixNano, time.Nanosecond) heartbeat := &HeartbeatService{ clock: clock, remoteClockMonitor: newRemoteClockMonitor(clock, time.Hour), } request := &PingRequest{ Ping: "testPing", } response, err := heartbeat.Ping(context.Background(), request) if err != nil { t.Fatal(err) } if response.Pong != request.Ping { t.Errorf("expected %s to be equal to %s", response.Pong, request.Ping) } if response.ServerTime != 5 { t.Errorf("expected server time 5, instead %d", response.ServerTime) } }
// TestScannerTiming verifies that ranges are scanned, regardless // of how many, to match scanInterval. func TestScannerTiming(t *testing.T) { defer leaktest.AfterTest(t)() const count = 3 const runTime = 100 * time.Millisecond const maxError = 7500 * time.Microsecond durations := []time.Duration{ 15 * time.Millisecond, 25 * time.Millisecond, } for i, duration := range durations { testutils.SucceedsSoon(t, func() error { ranges := newTestRangeSet(count, t) q := &testQueue{} s := newReplicaScanner(log.AmbientContext{}, duration, 0, ranges) s.AddQueues(q) mc := hlc.NewManualClock(123) clock := hlc.NewClock(mc.UnixNano, time.Nanosecond) stopper := stop.NewStopper() s.Start(clock, stopper) time.Sleep(runTime) stopper.Stop() avg := s.avgScan() log.Infof(context.Background(), "%d: average scan: %s", i, avg) if avg.Nanoseconds()-duration.Nanoseconds() > maxError.Nanoseconds() || duration.Nanoseconds()-avg.Nanoseconds() > maxError.Nanoseconds() { return errors.Errorf("expected %s, got %s: exceeds max error of %s", duration, avg, maxError) } return nil }) } }
func TestAcquireAndRelease(t *testing.T) { defer leaktest.AfterTest(t)() s, db := setup(t) defer s.Stopper().Stop() ctx := context.Background() manual := hlc.NewManualClock(123) clock := hlc.NewClock(manual.UnixNano, time.Nanosecond) lm := client.NewLeaseManager(db, clock, client.LeaseManagerOptions{ClientID: clientID1}) l, err := lm.AcquireLease(ctx, leaseKey) if err != nil { t.Fatal(err) } if err := lm.ReleaseLease(ctx, l); err != nil { t.Fatal(err) } if err := lm.ReleaseLease(ctx, l); !testutils.IsError(err, "unexpected value") { t.Fatal(err) } l, err = lm.AcquireLease(ctx, leaseKey) if err != nil { t.Fatal(err) } if err := lm.ReleaseLease(ctx, l); err != nil { t.Fatal(err) } }
// TestTxnCoordSenderSingleRoundtripTxn checks that a batch which completely // holds the writing portion of a Txn (including EndTransaction) does not // launch a heartbeat goroutine at all. func TestTxnCoordSenderSingleRoundtripTxn(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() manual := hlc.NewManualClock(123) clock := hlc.NewClock(manual.UnixNano, 20*time.Nanosecond) senderFunc := func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { br := ba.CreateReply() txnClone := ba.Txn.Clone() br.Txn = &txnClone br.Txn.Writing = true return br, nil } ambient := log.AmbientContext{Tracer: tracing.NewTracer()} ts := NewTxnCoordSender( ambient, senderFn(senderFunc), clock, false, stopper, MakeTxnMetrics(metric.TestSampleInterval), ) // Stop the stopper manually, prior to trying the transaction. This has the // effect of returning a NodeUnavailableError for any attempts at launching // a heartbeat goroutine. stopper.Stop() var ba roachpb.BatchRequest key := roachpb.Key("test") ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}}) ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}}) ba.Add(&roachpb.EndTransactionRequest{}) ba.Txn = &roachpb.Transaction{Name: "test"} _, pErr := ts.Send(context.Background(), ba) if pErr != nil { t.Fatal(pErr) } }
// TestTimestampCacheEqualTimestamp verifies that in the event of two // non-overlapping transactions with equal timestamps, the returned // timestamp is not owned by either one. func TestTimestampCacheEqualTimestamps(t *testing.T) { defer leaktest.AfterTest(t)() manual := hlc.NewManualClock(123) clock := hlc.NewClock(manual.UnixNano, time.Nanosecond) tc := newTimestampCache(clock) txn1 := uuid.MakeV4() txn2 := uuid.MakeV4() // Add two non-overlapping transactions at the same timestamp. ts1 := clock.Now() tc.add(roachpb.Key("a"), roachpb.Key("b"), ts1, &txn1, true) tc.add(roachpb.Key("b"), roachpb.Key("c"), ts1, &txn2, true) // When querying either side separately, the transaction ID is returned. if ts, txn, _ := tc.GetMaxRead(roachpb.Key("a"), roachpb.Key("b")); !ts.Equal(ts1) { t.Errorf("expected 'a'-'b' to have timestamp %s, but found %s", ts1, ts) } else if *txn != txn1 { t.Errorf("expected 'a'-'b' to have txn id %s, but found %s", txn1, txn) } if ts, txn, _ := tc.GetMaxRead(roachpb.Key("b"), roachpb.Key("c")); !ts.Equal(ts1) { t.Errorf("expected 'b'-'c' to have timestamp %s, but found %s", ts1, ts) } else if *txn != txn2 { t.Errorf("expected 'b'-'c' to have txn id %s, but found %s", txn2, txn) } // Querying a span that overlaps both returns a nil txn ID; neither // can proceed here. if ts, txn, _ := tc.GetMaxRead(roachpb.Key("a"), roachpb.Key("c")); !ts.Equal(ts1) { t.Errorf("expected 'a'-'c' to have timestamp %s, but found %s", ts1, ts) } else if txn != nil { t.Errorf("expected 'a'-'c' to have nil txn id, but found %s", txn) } }
// TestTxnCoordSenderErrorWithIntent validates that if a transactional request // returns an error but also indicates a Writing transaction, the coordinator // tracks it just like a successful request. func TestTxnCoordSenderErrorWithIntent(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() manual := hlc.NewManualClock(0) clock := hlc.NewClock(manual.UnixNano) clock.SetMaxOffset(20) testCases := []struct { roachpb.Error errMsg string }{ {*roachpb.NewError(roachpb.NewTransactionRetryError()), "retry txn"}, {*roachpb.NewError(roachpb.NewTransactionPushError(roachpb.Transaction{ TxnMeta: enginepb.TxnMeta{ ID: uuid.NewV4(), }})), "failed to push"}, {*roachpb.NewErrorf("testError"), "testError"}, } for i, test := range testCases { func() { senderFunc := func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { txn := ba.Txn.Clone() txn.Writing = true pErr := &roachpb.Error{} *pErr = test.Error pErr.SetTxn(&txn) return nil, pErr } ambient := log.AmbientContext{Tracer: tracing.NewTracer()} ts := NewTxnCoordSender( ambient, senderFn(senderFunc), clock, false, stopper, MakeTxnMetrics(metric.TestSampleInterval), ) var ba roachpb.BatchRequest key := roachpb.Key("test") ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}}) ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}}) ba.Add(&roachpb.EndTransactionRequest{}) ba.Txn = &roachpb.Transaction{Name: "test"} _, pErr := ts.Send(context.Background(), ba) if !testutils.IsPError(pErr, test.errMsg) { t.Errorf("%d: error did not match %s: %v", i, test.errMsg, pErr) } defer teardownHeartbeats(ts) ts.Lock() defer ts.Unlock() if len(ts.txns) != 1 { t.Errorf("%d: expected transaction to be tracked", i) } }() } }
func TestLeasesMultipleClients(t *testing.T) { defer leaktest.AfterTest(t)() s, db := setup(t) defer s.Stopper().Stop() ctx := context.Background() manual1 := hlc.NewManualClock(123) clock1 := hlc.NewClock(manual1.UnixNano, time.Nanosecond) manual2 := hlc.NewManualClock(123) clock2 := hlc.NewClock(manual2.UnixNano, time.Nanosecond) lm1 := client.NewLeaseManager(db, clock1, client.LeaseManagerOptions{ClientID: clientID1}) lm2 := client.NewLeaseManager(db, clock2, client.LeaseManagerOptions{ClientID: clientID2}) l1, err := lm1.AcquireLease(ctx, leaseKey) if err != nil { t.Fatal(err) } _, err = lm2.AcquireLease(ctx, leaseKey) if !testutils.IsError(err, "is not available until") { t.Fatalf("didn't get expected error trying to acquire already held lease: %v", err) } if _, ok := err.(*client.LeaseNotAvailableError); !ok { t.Fatalf("expected LeaseNotAvailableError, got %v", err) } // Ensure a lease can be "stolen" after it's expired. manual2.Increment(int64(client.DefaultLeaseDuration) + 1) l2, err := lm2.AcquireLease(ctx, leaseKey) if err != nil { t.Fatal(err) } // lm1's clock indicates that its lease should still be valid, but it doesn't // own it anymore. manual1.Increment(int64(client.DefaultLeaseDuration) / 2) if err := lm1.ExtendLease(ctx, l1); !testutils.IsError(err, "out of sync with DB state") { t.Fatalf("didn't get expected error trying to extend expired lease: %v", err) } if err := lm1.ReleaseLease(ctx, l1); !testutils.IsError(err, "unexpected value") { t.Fatalf("didn't get expected error trying to release stolen lease: %v", err) } if err := lm2.ReleaseLease(ctx, l2); err != nil { t.Fatal(err) } }
// Start starts the test cluster by bootstrapping an in-memory store // (defaults to maximum of 50M). The server is started, launching the // node RPC server and all HTTP endpoints. Use the value of // TestServer.Addr after Start() for client connections. Use Stop() // to shutdown the server after the test completes. func (ltc *LocalTestCluster) Start(t util.Tester, baseCtx *base.Config, initSender InitSenderFn) { ambient := log.AmbientContext{Tracer: tracing.NewTracer()} nc := &base.NodeIDContainer{} ambient.AddLogTag("n", nc) nodeID := roachpb.NodeID(1) nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID} ltc.tester = t ltc.Manual = hlc.NewManualClock(0) ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano) ltc.Stopper = stop.NewStopper() rpcContext := rpc.NewContext(ambient, baseCtx, ltc.Clock, ltc.Stopper) server := rpc.NewServer(rpcContext) // never started ltc.Gossip = gossip.New(ambient, nc, rpcContext, server, nil, ltc.Stopper, metric.NewRegistry()) ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20) ltc.Stopper.AddCloser(ltc.Eng) ltc.Stores = storage.NewStores(ambient, ltc.Clock) ltc.Sender = initSender(nodeDesc, ambient.Tracer, ltc.Clock, ltc.Latency, ltc.Stores, ltc.Stopper, ltc.Gossip) if ltc.DBContext == nil { dbCtx := client.DefaultDBContext() ltc.DBContext = &dbCtx } ltc.DB = client.NewDBWithContext(ltc.Sender, *ltc.DBContext) transport := storage.NewDummyRaftTransport() cfg := storage.TestStoreConfig() if ltc.RangeRetryOptions != nil { cfg.RangeRetryOptions = *ltc.RangeRetryOptions } cfg.AmbientCtx = ambient cfg.Clock = ltc.Clock cfg.DB = ltc.DB cfg.Gossip = ltc.Gossip cfg.Transport = transport cfg.MetricsSampleInterval = metric.TestSampleInterval ltc.Store = storage.NewStore(cfg, ltc.Eng, nodeDesc) if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } ltc.Stores.AddStore(ltc.Store) if err := ltc.Store.BootstrapRange(nil); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } if err := ltc.Store.Start(context.Background(), ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } nc.Set(context.TODO(), nodeDesc.NodeID) if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil { t.Fatalf("unable to set node descriptor: %s", err) } }
// TestScannerDisabled verifies that disabling a scanner prevents // replicas from being added to queues. func TestScannerDisabled(t *testing.T) { defer leaktest.AfterTest(t)() const count = 3 ranges := newTestRangeSet(count, t) q := &testQueue{} s := newReplicaScanner(log.AmbientContext{}, 1*time.Millisecond, 0, ranges) s.AddQueues(q) mc := hlc.NewManualClock(123) clock := hlc.NewClock(mc.UnixNano, time.Nanosecond) stopper := stop.NewStopper() s.Start(clock, stopper) defer stopper.Stop() // Verify queue gets all ranges. testutils.SucceedsSoon(t, func() error { if q.count() != count { return errors.Errorf("expected %d replicas; have %d", count, q.count()) } if s.scanCount() == 0 { return errors.Errorf("expected scanner count to increment") } return nil }) lastWaitEnabledCount := s.waitEnabledCount() // Now, disable the scanner. s.SetDisabled(true) testutils.SucceedsSoon(t, func() error { if s.waitEnabledCount() == lastWaitEnabledCount { return errors.Errorf("expected scanner to stop when disabled") } return nil }) lastScannerCount := s.scanCount() // Remove the replicas and verify the scanner still removes them while disabled. ranges.Visit(func(repl *Replica) bool { s.RemoveReplica(repl) return true }) testutils.SucceedsSoon(t, func() error { if qc := q.count(); qc != 0 { return errors.Errorf("expected queue to be empty after replicas removed from scanner; got %d", qc) } return nil }) if sc := s.scanCount(); sc != lastScannerCount { t.Errorf("expected scanner count to not increment: %d != %d", sc, lastScannerCount) } }
// TestRangeCommandClockUpdate verifies that followers update their // clocks when executing a command, even if the lease holder's clock is far // in the future. func TestRangeCommandClockUpdate(t *testing.T) { defer leaktest.AfterTest(t)() const numNodes = 3 var manuals []*hlc.ManualClock var clocks []*hlc.Clock for i := 0; i < numNodes; i++ { manuals = append(manuals, hlc.NewManualClock(1)) clocks = append(clocks, hlc.NewClock(manuals[i].UnixNano)) clocks[i].SetMaxOffset(100 * time.Millisecond) } mtc := &multiTestContext{clocks: clocks} mtc.Start(t, numNodes) defer mtc.Stop() mtc.replicateRange(1, 1, 2) // Advance the lease holder's clock ahead of the followers (by more than // MaxOffset but less than the range lease) and execute a command. manuals[0].Increment(int64(500 * time.Millisecond)) incArgs := incrementArgs([]byte("a"), 5) ts := clocks[0].Now() if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err != nil { t.Fatal(err) } // Wait for that command to execute on all the followers. util.SucceedsSoon(t, func() error { values := []int64{} for _, eng := range mtc.engines { val, _, err := engine.MVCCGet(context.Background(), eng, roachpb.Key("a"), clocks[0].Now(), true, nil) if err != nil { return err } values = append(values, mustGetInt(val)) } if !reflect.DeepEqual(values, []int64{5, 5, 5}) { return errors.Errorf("expected (5, 5, 5), got %v", values) } return nil }) // Verify that all the followers have accepted the clock update from // node 0 even though it comes from outside the usual max offset. now := clocks[0].Now() for i, clock := range clocks { // Only compare the WallTimes: it's normal for clock 0 to be a few logical ticks ahead. if clock.Now().WallTime < now.WallTime { t.Errorf("clock %d is behind clock 0: %s vs %s", i, clock.Now(), now) } } }
// TestRejectFutureCommand verifies that lease holders reject commands that // would cause a large time jump. func TestRejectFutureCommand(t *testing.T) { defer leaktest.AfterTest(t)() manual := hlc.NewManualClock(123) clock := hlc.NewClock(manual.UnixNano, 100*time.Millisecond) mtc := &multiTestContext{clock: clock} mtc.Start(t, 1) defer mtc.Stop() ts1 := clock.Now() key := roachpb.Key("a") incArgs := incrementArgs(key, 5) // Commands with a future timestamp that is within the MaxOffset // bound will be accepted and will cause the clock to advance. const numCmds = 3 clockOffset := clock.MaxOffset() / numCmds for i := int64(1); i <= numCmds; i++ { ts := ts1.Add(i*clockOffset.Nanoseconds(), 0) if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err != nil { t.Fatal(err) } } ts2 := clock.Now() if expAdvance, advance := ts2.GoTime().Sub(ts1.GoTime()), numCmds*clockOffset; advance != expAdvance { t.Fatalf("expected clock to advance %s; got %s", expAdvance, advance) } // Once the accumulated offset reaches MaxOffset, commands will be rejected. _, pErr := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts1.Add(clock.MaxOffset().Nanoseconds()+1, 0)}, &incArgs) if !testutils.IsPError(pErr, "rejecting command with timestamp in the future") { t.Fatalf("unexpected error %v", pErr) } // The clock did not advance and the final command was not executed. ts3 := clock.Now() if advance := ts3.GoTime().Sub(ts2.GoTime()); advance != 0 { t.Fatalf("expected clock not to advance, but it advanced by %s", advance) } val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], key, ts3, true, nil) if err != nil { t.Fatal(err) } if a, e := mustGetInt(val), incArgs.Increment*numCmds; a != e { t.Errorf("expected %d, got %d", e, a) } }
func TestRemoveLeaseIfExpiring(t *testing.T) { defer leaktest.AfterTest(t)() p := planner{session: &Session{context: context.Background()}} mc := hlc.NewManualClock(123) p.leaseMgr = &LeaseManager{LeaseStore: LeaseStore{clock: hlc.NewClock(mc.UnixNano, time.Nanosecond)}} p.leases = make([]*LeaseState, 0) txn := client.Txn{Context: context.Background()} p.setTxn(&txn) if p.removeLeaseIfExpiring(nil) { t.Error("expected false with nil input") } // Add a lease to the planner. d := int64(LeaseDuration) l1 := &LeaseState{expiration: parser.DTimestamp{Time: time.Unix(0, mc.UnixNano()+d+1)}} p.leases = append(p.leases, l1) et := hlc.Timestamp{WallTime: l1.Expiration().UnixNano()} txn.UpdateDeadlineMaybe(et) if p.removeLeaseIfExpiring(l1) { t.Error("expected false with a non-expiring lease") } if !p.txn.GetDeadline().Equal(et) { t.Errorf("expected deadline %s but got %s", et, p.txn.GetDeadline()) } // Advance the clock so that l1 will be expired. mc.Increment(d + 1) // Add another lease. l2 := &LeaseState{expiration: parser.DTimestamp{Time: time.Unix(0, mc.UnixNano()+d+1)}} p.leases = append(p.leases, l2) if !p.removeLeaseIfExpiring(l1) { t.Error("expected true with an expiring lease") } et = hlc.Timestamp{WallTime: l2.Expiration().UnixNano()} txn.UpdateDeadlineMaybe(et) if !(len(p.leases) == 1 && p.leases[0] == l2) { t.Errorf("expected leases to contain %s but has %s", l2, p.leases) } if !p.txn.GetDeadline().Equal(et) { t.Errorf("expected deadline %s, but got %s", et, p.txn.GetDeadline()) } }
// TestScannerEmptyRangeSet verifies that an empty range set doesn't busy loop. func TestScannerEmptyRangeSet(t *testing.T) { defer leaktest.AfterTest(t)() ranges := newTestRangeSet(0, t) q := &testQueue{} s := newReplicaScanner(log.AmbientContext{}, time.Hour, 0, ranges) s.AddQueues(q) mc := hlc.NewManualClock(123) clock := hlc.NewClock(mc.UnixNano, time.Nanosecond) stopper := stop.NewStopper() defer stopper.Stop() s.Start(clock, stopper) time.Sleep(time.Millisecond) // give it some time to (not) busy loop if count := s.scanCount(); count > 1 { t.Errorf("expected at most one loop, but got %d", count) } }
// createTestBookie creates a new bookie, stopper and manual clock for testing. func createTestBookie( reservationTimeout time.Duration, maxReservations int, maxReservedBytes int64, ) (*stop.Stopper, *hlc.ManualClock, *bookie) { stopper := stop.NewStopper() mc := hlc.NewManualClock(0) clock := hlc.NewClock(mc.UnixNano) b := newBookie(clock, stopper, newStoreMetrics(time.Hour), reservationTimeout) // Lock the bookie to prevent the main loop from running as we change some // of the bookie's state. b.mu.Lock() defer b.mu.Unlock() b.maxReservations = maxReservations b.maxReservedBytes = maxReservedBytes // Set a high number for a mocked total available space. b.metrics.Available.Update(defaultMaxReservedBytes * 10) return stopper, mc, b }
// TestTimestampCacheSetLowWater verifies that setting the low // water mark moves max timestamps forward as appropriate. func TestTimestampCacheSetLowWater(t *testing.T) { defer leaktest.AfterTest(t)() manual := hlc.NewManualClock(123) clock := hlc.NewClock(manual.UnixNano, time.Nanosecond) tc := newTimestampCache(clock) // Increment time to the low water mark + 10. manual.Increment(10) aTS := clock.Now() tc.add(roachpb.Key("a"), nil, aTS, nil, true) // Increment time by 10ns and add another key. manual.Increment(10) bTS := clock.Now() tc.add(roachpb.Key("b"), nil, bTS, nil, true) // Increment time by 10ns and add another key. manual.Increment(10) cTS := clock.Now() tc.add(roachpb.Key("c"), nil, cTS, nil, true) // Set low water mark. tc.SetLowWater(bTS) // Verify looking up key "a" returns the new low water mark ("a"'s timestamp). for i, test := range []struct { key roachpb.Key expTS hlc.Timestamp expOK bool }{ {roachpb.Key("a"), bTS, false}, {roachpb.Key("b"), bTS, false}, {roachpb.Key("c"), cTS, true}, {roachpb.Key("d"), bTS, false}, } { if rTS, _, ok := tc.GetMaxRead(test.key, nil); !rTS.Equal(test.expTS) || ok != test.expOK { t.Errorf("%d: expected ts %s, got %s; exp ok=%t; got %t", i, test.expTS, rTS, test.expOK, ok) } } // Try setting a lower low water mark than the previous value. tc.SetLowWater(aTS) if rTS, _, ok := tc.GetMaxRead(roachpb.Key("d"), nil); !rTS.Equal(bTS) || ok { t.Errorf("setting lower low water mark should not be allowed; expected %s; got %s; ok=%t", bTS, rTS, ok) } }
// TestRejectFutureCommand verifies that lease holders reject commands that // would cause a large time jump. func TestRejectFutureCommand(t *testing.T) { defer leaktest.AfterTest(t)() const maxOffset = 100 * time.Millisecond manual := hlc.NewManualClock(0) clock := hlc.NewClock(manual.UnixNano) clock.SetMaxOffset(maxOffset) mtc := &multiTestContext{clock: clock} mtc.Start(t, 1) defer mtc.Stop() startTime := manual.UnixNano() // Commands with a future timestamp that is within the MaxOffset // bound will be accepted and will cause the clock to advance. for i := int64(0); i < 3; i++ { incArgs := incrementArgs([]byte("a"), 5) ts := hlc.ZeroTimestamp.Add(startTime+((i+1)*30)*int64(time.Millisecond), 0) if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err != nil { t.Fatal(err) } } if now := clock.Now(); now.WallTime != int64(90*time.Millisecond) { t.Fatalf("expected clock to advance to 90ms; got %s", now) } // Once the accumulated offset reaches MaxOffset, commands will be rejected. incArgs := incrementArgs([]byte("a"), 11) ts := hlc.ZeroTimestamp.Add(int64((time.Duration(startTime)+maxOffset+1)*time.Millisecond), 0) if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{Timestamp: ts}, &incArgs); err == nil { t.Fatalf("expected clock offset error but got nil") } // The clock remained at 90ms and the final command was not executed. if now := clock.Now(); now.WallTime != int64(90*time.Millisecond) { t.Errorf("expected clock to stay at 90ms; got %s", now) } val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], roachpb.Key("a"), clock.Now(), true, nil) if err != nil { t.Fatal(err) } if v := mustGetInt(val); v != 15 { t.Errorf("expected 15, got %v", v) } }
// TestScannerAddToQueues verifies that ranges are added to and // removed from multiple queues. func TestScannerAddToQueues(t *testing.T) { defer leaktest.AfterTest(t)() const count = 3 ranges := newTestRangeSet(count, t) q1, q2 := &testQueue{}, &testQueue{} // We don't want to actually consume entries from the queues during this test. q1.setDisabled(true) q2.setDisabled(true) s := newReplicaScanner(log.AmbientContext{}, 1*time.Millisecond, 0, ranges) s.AddQueues(q1, q2) mc := hlc.NewManualClock(123) clock := hlc.NewClock(mc.UnixNano, time.Nanosecond) stopper := stop.NewStopper() // Start scanner and verify that all ranges are added to both queues. s.Start(clock, stopper) testutils.SucceedsSoon(t, func() error { if q1.count() != count || q2.count() != count { return errors.Errorf("q1 or q2 count != %d; got %d, %d", count, q1.count(), q2.count()) } return nil }) // Remove first range and verify it does not exist in either range. rng := ranges.remove(0, t) testutils.SucceedsSoon(t, func() error { // This is intentionally inside the loop, otherwise this test races as // our removal of the range may be processed before a stray re-queue. // Removing on each attempt makes sure we clean this up as we retry. s.RemoveReplica(rng) c1 := q1.count() c2 := q2.count() if c1 != count-1 || c2 != count-1 { return errors.Errorf("q1 or q2 count != %d; got %d, %d", count-1, c1, c2) } return nil }) // Stop scanner and verify both queues are stopped. stopper.Stop() if !q1.isDone() || !q2.isDone() { t.Errorf("expected all queues to stop; got %t, %t", q1.isDone(), q2.isDone()) } }
// createTestStorePool creates a stopper, gossip and storePool for use in // tests. Stopper must be stopped by the caller. func createTestStorePool( timeUntilStoreDead time.Duration, ) (*stop.Stopper, *gossip.Gossip, *hlc.ManualClock, *StorePool) { stopper := stop.NewStopper() mc := hlc.NewManualClock(0) clock := hlc.NewClock(mc.UnixNano) rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, clock, stopper) server := rpc.NewServer(rpcContext) // never started g := gossip.NewTest(1, rpcContext, server, nil, stopper, metric.NewRegistry()) storePool := NewStorePool( log.AmbientContext{}, g, clock, rpcContext, timeUntilStoreDead, stopper, ) return stopper, g, mc, storePool }
// Test that the a txn gets a fresh OrigTimestamp with every retry. func TestAbortedRetryRenewsTimestamp(t *testing.T) { defer leaktest.AfterTest(t)() // Create a TestSender that aborts a transaction 2 times before succeeding. mc := hlc.NewManualClock(123) clock := hlc.NewClock(mc.UnixNano, time.Nanosecond) count := 0 db := NewDB(newTestSender(func(ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { if _, ok := ba.GetArg(roachpb.Put); ok { mc.Increment(1) count++ if count < 3 { return nil, roachpb.NewError(&roachpb.TransactionAbortedError{}) } } return ba.CreateReply(), nil }, nil)) txnClosure := func(txn *Txn, opt *TxnExecOptions) error { // Ensure the KV transaction is created. return txn.Put("a", "b") } txn := NewTxn(context.Background(), *db) // Request a client-defined timestamp. refTimestamp := clock.Now() execOpt := TxnExecOptions{ AutoRetry: true, AutoCommit: true, Clock: clock, } // Perform the transaction. if err := txn.Exec(execOpt, txnClosure); err != nil { t.Fatal(err) } // Check the timestamp was preserved. if txn.Proto.OrigTimestamp.WallTime == refTimestamp.WallTime { t.Errorf("expected txn orig ts to be different than %s", refTimestamp) } }
func TestTimestampCacheClear(t *testing.T) { defer leaktest.AfterTest(t)() manual := hlc.NewManualClock(0) clock := hlc.NewClock(manual.UnixNano) tc := newTimestampCache(clock) manual.Set(5000000) ts := clock.Now() tc.add(roachpb.Key("a"), nil, ts, nil, true) // Clear the cache, which will reset the low water mark to // the current time. tc.Clear(clock.Now()) // Fetching any keys should give current time. expTS := clock.Timestamp() if rTS, _, ok := tc.GetMaxRead(roachpb.Key("a"), nil); !rTS.Equal(expTS) || ok { t.Errorf("expected \"a\" to have cleared timestamp; exp ok=false; got %t", ok) } }
// createStores creates a slice of count stores. func createStores(count int, t *testing.T) (*hlc.ManualClock, []*Store, *Stores, *stop.Stopper) { stopper := stop.NewStopper() manual := hlc.NewManualClock(123) cfg := TestStoreConfig(hlc.NewClock(manual.UnixNano, time.Nanosecond)) ls := NewStores(log.AmbientContext{}, cfg.Clock) // Create two stores with ranges we care about. stores := []*Store{} for i := 0; i < 2; i++ { cfg.Transport = NewDummyRaftTransport() eng := engine.NewInMem(roachpb.Attributes{}, 1<<20) stopper.AddCloser(eng) s := NewStore(cfg, eng, &roachpb.NodeDescriptor{NodeID: 1}) storeIDAlloc++ s.Ident.StoreID = storeIDAlloc stores = append(stores, s) } return manual, stores, ls, stopper }
// createTestStorePool creates a stopper, gossip and storePool for use in // tests. Stopper must be stopped by the caller. func createTestStorePool( timeUntilStoreDead time.Duration, deterministic bool, defaultNodeLiveness bool, ) (*stop.Stopper, *gossip.Gossip, *hlc.ManualClock, *StorePool, *mockNodeLiveness) { stopper := stop.NewStopper() mc := hlc.NewManualClock(123) clock := hlc.NewClock(mc.UnixNano, time.Nanosecond) rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, clock, stopper) server := rpc.NewServer(rpcContext) // never started g := gossip.NewTest(1, rpcContext, server, nil, stopper, metric.NewRegistry()) mnl := newMockNodeLiveness(defaultNodeLiveness) storePool := NewStorePool( log.AmbientContext{}, g, clock, mnl.nodeLivenessFunc, timeUntilStoreDead, deterministic, ) return stopper, g, mc, storePool, mnl }
func BenchmarkTimestampCacheInsertion(b *testing.B) { manual := hlc.NewManualClock(123) clock := hlc.NewClock(manual.UnixNano, time.Nanosecond) tc := newTimestampCache(clock) for i := 0; i < b.N; i++ { tc.Clear(clock.Now()) cdTS := clock.Now() tc.add(roachpb.Key("c"), roachpb.Key("d"), cdTS, nil, true) beTS := clock.Now() tc.add(roachpb.Key("b"), roachpb.Key("e"), beTS, nil, true) adTS := clock.Now() tc.add(roachpb.Key("a"), roachpb.Key("d"), adTS, nil, true) cfTS := clock.Now() tc.add(roachpb.Key("c"), roachpb.Key("f"), cfTS, nil, true) } }