// startFakeServerGossip creates local gossip instances and remote faked gossip instance.
// The remote gossip instance launches its faked gossip service just for
// check the client message.
func startFakeServerGossip(t *testing.T) (local *Gossip, remote *fakeGossipServer, stopper *stop.Stopper) {
lclock := hlc.NewClock(hlc.UnixNano)
stopper = stop.NewStopper()
lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper)
laddr := util.CreateTestAddr("tcp")
lserver := rpc.NewServer(laddr, lRPCContext)
if err := lserver.Start(); err != nil {
t.Fatal(err)
}
local = New(lRPCContext, TestBootstrap)
local.start(lserver, stopper)
rclock := hlc.NewClock(hlc.UnixNano)
raddr := util.CreateTestAddr("tcp")
rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper)
rserver := rpc.NewServer(raddr, rRPCContext)
if err := rserver.Start(); err != nil {
t.Fatal(err)
}
remote, err := newFakeGossipServer(rserver, stopper)
if err != nil {
t.Fatal(err)
}
addr := rserver.Addr()
remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String())
time.Sleep(time.Millisecond)
return
}
// createTestStoreWithoutStart creates a test store using an in-memory
// engine without starting the store. It returns the store, the store
// clock's manual unix nanos time and a stopper. The caller is
// responsible for stopping the stopper upon completion.
func createTestStoreWithoutStart(t *testing.T) (*Store, *hlc.ManualClock, *stop.Stopper) {
stopper := stop.NewStopper()
// Setup fake zone config handler.
config.TestingSetupZoneConfigHook(stopper)
rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper)
ctx := TestStoreContext
ctx.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap)
ctx.StorePool = NewStorePool(ctx.Gossip, TestTimeUntilStoreDeadOff, stopper)
manual := hlc.NewManualClock(0)
ctx.Clock = hlc.NewClock(manual.UnixNano)
eng := engine.NewInMem(roachpb.Attributes{}, 10<<20, stopper)
ctx.Transport = multiraft.NewLocalRPCTransport(stopper)
stopper.AddCloser(ctx.Transport)
sender := &testSender{}
ctx.DB = client.NewDB(sender)
store := NewStore(ctx, eng, &roachpb.NodeDescriptor{NodeID: 1})
sender.store = store
if err := store.Bootstrap(roachpb.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil {
t.Fatal(err)
}
if err := store.BootstrapRange(nil); err != nil {
t.Fatal(err)
}
return store, manual, stopper
}
// createTestStoreWithoutStart creates a test store using an in-memory
// engine without starting the store. It returns the store, the store
// clock's manual unix nanos time and a stopper. The caller is
// responsible for stopping the stopper upon completion.
func createTestStoreWithoutStart(t *testing.T) (*Store, *hlc.ManualClock, *stop.Stopper) {
stopper := stop.NewStopper()
rpcContext := rpc.NewContext(rootTestBaseContext, hlc.NewClock(hlc.UnixNano), stopper)
ctx := TestStoreContext
ctx.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap)
manual := hlc.NewManualClock(0)
ctx.Clock = hlc.NewClock(manual.UnixNano)
eng := engine.NewInMem(proto.Attributes{}, 10<<20)
ctx.Transport = multiraft.NewLocalRPCTransport()
stopper.AddCloser(ctx.Transport)
sender := &testSender{}
var err error
if ctx.DB, err = client.Open("//root@", client.SenderOpt(sender)); err != nil {
t.Fatal(err)
}
store := NewStore(ctx, eng, &proto.NodeDescriptor{NodeID: 1})
sender.store = store
if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil {
t.Fatal(err)
}
if err := store.BootstrapRange(); err != nil {
t.Fatal(err)
}
return store, manual, stopper
}
func TestOffsetMeasurement(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
serverTime := time.Unix(0, 20)
serverClock := hlc.NewClock(serverTime.UnixNano)
serverCtx := newNodeTestContext(serverClock, stopper)
s, ln := newTestServer(t, serverCtx, true)
remoteAddr := ln.Addr().String()
RegisterHeartbeatServer(s, &HeartbeatService{
clock: serverClock,
remoteClockMonitor: serverCtx.RemoteClocks,
})
// Create a client clock that is behind the server clock.
clientAdvancing := AdvancingClock{time: time.Unix(0, 10)}
clientClock := hlc.NewClock(clientAdvancing.UnixNano)
clientClock.SetMaxOffset(time.Millisecond)
clientCtx := newNodeTestContext(clientClock, stopper)
clientCtx.RemoteClocks.offsetTTL = 5 * clientAdvancing.advancementInterval
if _, err := clientCtx.GRPCDial(remoteAddr); err != nil {
t.Fatal(err)
}
expectedOffset := RemoteOffset{Offset: 10, Uncertainty: 0, MeasuredAt: 10}
util.SucceedsSoon(t, func() error {
clientCtx.RemoteClocks.mu.Lock()
defer clientCtx.RemoteClocks.mu.Unlock()
if o, ok := clientCtx.RemoteClocks.mu.offsets[remoteAddr]; !ok {
return util.Errorf("expected offset of %s to be initialized, but it was not", remoteAddr)
} else if o != expectedOffset {
return util.Errorf("expected:\n%v\nactual:\n%v", expectedOffset, o)
}
return nil
})
// Change the client such that it receives a heartbeat right after the
// maximum clock reading delay.
clientAdvancing.Lock()
clientAdvancing.advancementInterval = maximumPingDurationMult*clientClock.MaxOffset() + 1*time.Nanosecond
clientAdvancing.Unlock()
util.SucceedsSoon(t, func() error {
clientCtx.RemoteClocks.mu.Lock()
defer clientCtx.RemoteClocks.mu.Unlock()
if o, ok := clientCtx.RemoteClocks.mu.offsets[remoteAddr]; ok {
return util.Errorf("expected offset to have been cleared, but found %s", o)
}
return nil
})
}
// startGossip creates local and remote gossip instances.
// Both remote and local instances launch the gossip service.
func startGossip(t *testing.T) (local, remote *Gossip, stopper *stop.Stopper) {
stopper = stop.NewStopper()
lclock := hlc.NewClock(hlc.UnixNano)
lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper)
laddr := util.CreateTestAddr("tcp")
lserver := rpc.NewServer(lRPCContext)
lTLSConfig, err := lRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
lln, err := util.ListenAndServe(stopper, lserver, laddr, lTLSConfig)
if err != nil {
t.Fatal(err)
}
local = New(lRPCContext, TestBootstrap)
local.SetNodeID(1)
if err := local.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: 1,
Address: util.MakeUnresolvedAddr(laddr.Network(), laddr.String()),
}); err != nil {
t.Fatal(err)
}
rclock := hlc.NewClock(hlc.UnixNano)
rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper)
raddr := util.CreateTestAddr("tcp")
rserver := rpc.NewServer(rRPCContext)
rTLSConfig, err := rRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
rln, err := util.ListenAndServe(stopper, rserver, raddr, rTLSConfig)
if err != nil {
t.Fatal(err)
}
remote = New(rRPCContext, TestBootstrap)
remote.SetNodeID(2)
if err := remote.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: 2,
Address: util.MakeUnresolvedAddr(raddr.Network(), raddr.String()),
}); err != nil {
t.Fatal(err)
}
local.start(lserver, lln.Addr(), stopper)
remote.start(rserver, rln.Addr(), stopper)
time.Sleep(time.Millisecond)
return
}
// TestDelayedOffsetMeasurement tests that the client will record a
// zero offset if the heartbeat reply exceeds the
// maximumClockReadingDelay, but not the heartbeat timeout.
func TestDelayedOffsetMeasurement(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := util.NewStopper()
defer stopper.Stop()
serverManual := hlc.NewManualClock(10)
serverClock := hlc.NewClock(serverManual.UnixNano)
s := createTestServer(serverClock, stopper, t)
heartbeat := &HeartbeatService{
clock: serverClock,
remoteClockMonitor: newRemoteClockMonitor(serverClock),
}
if err := s.RegisterName("Heartbeat", heartbeat); err != nil {
t.Fatalf("Unable to register heartbeat service: %s", err)
}
// Create a client that receives a heartbeat right after the
// maximumClockReadingDelay.
advancing := AdvancingClock{
time: 0,
advancementInterval: maximumClockReadingDelay.Nanoseconds() + 1,
}
clientClock := hlc.NewClock(advancing.UnixNano)
context := NewServerTestContext(clientClock, stopper)
c := NewClient(s.Addr(), nil, context)
<-c.Ready
// Ensure we get a good heartbeat before continuing.
if err := util.IsTrueWithin(c.IsHealthy, heartbeatInterval*10); err != nil {
t.Fatal(err)
}
// Since the reply took too long, we should have a zero offset, even
// though the client is still healthy because it received a heartbeat
// reply.
if o := c.RemoteOffset(); !o.Equal(proto.RemoteOffset{}) {
t.Errorf("expected offset %v, actual %v", proto.RemoteOffset{}, o)
}
// Ensure the general offsets map was updated properly too.
context.RemoteClocks.mu.Lock()
if o, ok := context.RemoteClocks.offsets[c.addr.String()]; ok {
t.Errorf("expected offset to not exist, but found %v", o)
}
context.RemoteClocks.mu.Unlock()
}
// TestTxnCoordSenderSingleRoundtripTxn checks that a batch which completely
// holds the writing portion of a Txn (including EndTransaction) does not
// launch a heartbeat goroutine at all.
func TestTxnCoordSenderSingleRoundtripTxn(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(20)
ts := NewTxnCoordSender(senderFn(func(_ context.Context, ba proto.BatchRequest) (*proto.BatchResponse, *proto.Error) {
return ba.CreateReply().(*proto.BatchResponse), nil
}), clock, false, nil, stopper)
// Stop the stopper manually, prior to trying the transaction. This has the
// effect of returning a NodeUnavailableError for any attempts at launching
// a heartbeat goroutine.
stopper.Stop()
var ba proto.BatchRequest
put := &proto.PutRequest{}
put.Key = proto.Key("test")
ba.Add(put)
ba.Add(&proto.EndTransactionRequest{})
ba.Txn = &proto.Transaction{Name: "test"}
_, pErr := ts.Send(context.Background(), ba)
if pErr != nil {
t.Fatal(pErr)
}
}
// TestTimestampCacheReadVsWrite verifies that the timestamp cache
// can differentiate between read and write timestamp.
func TestTimestampCacheReadVsWrite(t *testing.T) {
defer leaktest.AfterTest(t)
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
tc := NewTimestampCache(clock)
// Add read-only non-txn entry at current time.
ts1 := clock.Now()
tc.Add(roachpb.Key("a"), roachpb.Key("b"), ts1, nil, true)
// Add two successive txn entries; one read-only and one read-write.
txn1ID := uuid.NewUUID4()
txn2ID := uuid.NewUUID4()
ts2 := clock.Now()
tc.Add(roachpb.Key("a"), nil, ts2, txn1ID, true)
ts3 := clock.Now()
tc.Add(roachpb.Key("a"), nil, ts3, txn2ID, false)
// Fetching with no transaction gets latest values.
if rTS, wTS := tc.GetMax(roachpb.Key("a"), nil, nil); !rTS.Equal(ts2) || !wTS.Equal(ts3) {
t.Errorf("expected %s %s; got %s %s", ts2, ts3, rTS, wTS)
}
// Fetching with txn ID "1" gets original for read and most recent for write.
if rTS, wTS := tc.GetMax(roachpb.Key("a"), nil, txn1ID); !rTS.Equal(ts1) || !wTS.Equal(ts3) {
t.Errorf("expected %s %s; got %s %s", ts1, ts3, rTS, wTS)
}
// Fetching with txn ID "2" gets ts2 for read and low water mark for write.
if rTS, wTS := tc.GetMax(roachpb.Key("a"), nil, txn2ID); !rTS.Equal(ts2) || !wTS.Equal(tc.lowWater) {
t.Errorf("expected %s %s; got %s %s", ts2, tc.lowWater, rTS, wTS)
}
}
// TestTimestampCacheWithTxnID verifies that timestamps matching
// the specified txn ID are ignored.
func TestTimestampCacheWithTxnID(t *testing.T) {
defer leaktest.AfterTest(t)
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
tc := NewTimestampCache(clock)
// Add two successive txn entries.
txn1ID := uuid.NewUUID4()
txn2ID := uuid.NewUUID4()
ts1 := clock.Now()
tc.Add(roachpb.Key("a"), roachpb.Key("c"), ts1, txn1ID, true)
ts2 := clock.Now()
// This entry will remove "a"-"b" from the cache.
tc.Add(roachpb.Key("b"), roachpb.Key("d"), ts2, txn2ID, true)
// Fetching with no transaction gets latest value.
if ts, _ := tc.GetMax(roachpb.Key("b"), nil, nil); !ts.Equal(ts2) {
t.Errorf("expected %s; got %s", ts2, ts)
}
// Fetching with txn ID "1" gets most recent.
if ts, _ := tc.GetMax(roachpb.Key("b"), nil, txn1ID); !ts.Equal(ts2) {
t.Errorf("expected %s; got %s", ts2, ts)
}
// Fetching with txn ID "2" skips most recent.
if ts, _ := tc.GetMax(roachpb.Key("b"), nil, txn2ID); !ts.Equal(ts1) {
t.Errorf("expected %s; got %s", ts1, ts)
}
}
// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester) {
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = stop.NewStopper()
rpcContext := rpc.NewContext(testutils.NewRootTestBaseContext(), ltc.Clock, ltc.Stopper)
ltc.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap)
ltc.Eng = engine.NewInMem(proto.Attributes{}, 50<<20)
ltc.lSender = newRetryableLocalSender(NewLocalSender())
ltc.Sender = NewTxnCoordSender(ltc.lSender, ltc.Clock, false, nil, ltc.Stopper)
var err error
if ltc.DB, err = client.Open("//root@", client.SenderOpt(ltc.Sender)); err != nil {
t.Fatal(err)
}
transport := multiraft.NewLocalRPCTransport(ltc.Stopper)
ltc.Stopper.AddCloser(transport)
ctx := storage.TestStoreContext
ctx.Clock = ltc.Clock
ctx.DB = ltc.DB
ctx.Gossip = ltc.Gossip
ctx.Transport = transport
ltc.Store = storage.NewStore(ctx, ltc.Eng, &proto.NodeDescriptor{NodeID: 1})
if err := ltc.Store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.lSender.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(nil); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
}
func TestStoresVisitStores(t *testing.T) {
defer leaktest.AfterTest(t)()
ls := NewStores(hlc.NewClock(hlc.UnixNano))
numStores := 10
for i := 0; i < numStores; i++ {
ls.AddStore(&Store{Ident: roachpb.StoreIdent{StoreID: roachpb.StoreID(i)}})
}
visit := make([]bool, numStores)
err := ls.VisitStores(func(s *Store) error { visit[s.Ident.StoreID] = true; return nil })
if err != nil {
t.Errorf("unexpected error on visit: %s", err.Error())
}
for i, visited := range visit {
if !visited {
t.Errorf("store %d was not visited", i)
}
}
err = ls.VisitStores(func(s *Store) error { return errors.New("") })
if err == nil {
t.Errorf("expected visit error")
}
}
func TestHeartbeatReply(t *testing.T) {
defer leaktest.AfterTest(t)
manual := hlc.NewManualClock(5)
clock := hlc.NewClock(manual.UnixNano)
heartbeat := &HeartbeatService{
clock: clock,
remoteClockMonitor: newRemoteClockMonitor(clock),
}
request := &PingRequest{
Ping: "testPing",
}
var response *PingResponse
if responseI, err := heartbeat.Ping(request); err != nil {
t.Fatal(err)
} else {
response = responseI.(*PingResponse)
}
if response.Pong != request.Ping {
t.Errorf("expected %s to be equal to %s", response.Pong, request.Ping)
}
if response.ServerTime != 5 {
t.Errorf("expected server time 5, instead %d", response.ServerTime)
}
}
// TestTxnCoordSenderSingleRoundtripTxn checks that a batch which completely
// holds the writing portion of a Txn (including EndTransaction) does not
// launch a heartbeat goroutine at all.
func TestTxnCoordSenderSingleRoundtripTxn(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(20)
ts := NewTxnCoordSender(senderFn(func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
br := ba.CreateReply()
br.Txn = ba.Txn.Clone()
br.Txn.Writing = true
return br, nil
}), clock, false, nil, stopper)
// Stop the stopper manually, prior to trying the transaction. This has the
// effect of returning a NodeUnavailableError for any attempts at launching
// a heartbeat goroutine.
stopper.Stop()
var ba roachpb.BatchRequest
key := roachpb.Key("test")
ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.EndTransactionRequest{})
ba.Txn = &roachpb.Transaction{Name: "test"}
_, pErr := ts.Send(context.Background(), ba)
if pErr != nil {
t.Fatal(pErr)
}
}
// TestMultiRangeScanReverseScanInconsistent verifies that a Scan/ReverseScan
// across ranges that doesn't require read consistency will set a timestamp
// using the clock local to the distributed sender.
func TestMultiRangeScanReverseScanInconsistent(t *testing.T) {
defer leaktest.AfterTest(t)
s, db := setupMultipleRanges(t, "b")
defer s.Stop()
// Write keys "a" and "b", the latter of which is the first key in the
// second range.
keys := []string{"a", "b"}
ts := []time.Time{}
for i, key := range keys {
b := &client.Batch{}
b.Put(key, "value")
if err := db.Run(b); err != nil {
t.Fatal(err)
}
ts = append(ts, b.Results[0].Rows[0].Timestamp())
log.Infof("%d: %s", i, b.Results[0].Rows[0].Timestamp())
}
// Do an inconsistent Scan/ReverseScan from a new DistSender and verify
// it does the read at its local clock and doesn't receive an
// OpRequiresTxnError. We set the local clock to the timestamp of
// the first key to verify it's used to read only key "a".
manual := hlc.NewManualClock(ts[1].UnixNano() - 1)
clock := hlc.NewClock(manual.UnixNano)
ds := kv.NewDistSender(&kv.DistSenderContext{Clock: clock}, s.Gossip())
// Scan.
sa := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("c"), 0).(*roachpb.ScanRequest)
reply, err := client.SendWrappedWith(ds, nil, roachpb.BatchRequest_Header{
ReadConsistency: roachpb.INCONSISTENT,
}, sa)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if l := len(sr.Rows); l != 1 {
t.Fatalf("expected 1 row; got %d", l)
}
if key := string(sr.Rows[0].Key); keys[0] != key {
t.Errorf("expected key %q; got %q", keys[0], key)
}
// ReverseScan.
rsa := roachpb.NewReverseScan(roachpb.Key("a"), roachpb.Key("c"), 0).(*roachpb.ReverseScanRequest)
reply, err = client.SendWrappedWith(ds, nil, roachpb.BatchRequest_Header{
ReadConsistency: roachpb.INCONSISTENT,
}, rsa)
if err != nil {
t.Fatal(err)
}
rsr := reply.(*roachpb.ReverseScanResponse)
if l := len(rsr.Rows); l != 1 {
t.Fatalf("expected 1 row; got %d", l)
}
if key := string(rsr.Rows[0].Key); keys[0] != key {
t.Errorf("expected key %q; got %q", keys[0], key)
}
}
func gossipForTest(t *testing.T) (*gossip.Gossip, *stop.Stopper) {
stopper := stop.NewStopper()
// Setup fake zone config handler.
config.TestingSetupZoneConfigHook(stopper)
rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper)
g := gossip.New(rpcContext, gossip.TestBootstrap, stopper)
// Have to call g.SetNodeID before call g.AddInfo
g.SetNodeID(roachpb.NodeID(1))
// Put an empty system config into gossip.
if err := g.AddInfoProto(gossip.KeySystemConfig,
&config.SystemConfig{}, 0); err != nil {
t.Fatal(err)
}
// Wait for SystemConfig.
util.SucceedsSoon(t, func() error {
if g.GetSystemConfig() == nil {
return util.Errorf("expected non-nil system config")
}
return nil
})
return g, stopper
}
func TestTimestampCacheMergeInto(t *testing.T) {
defer leaktest.AfterTest(t)
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
testCases := []struct {
useClear bool
expLen int
}{
{true, 3},
{false, 5},
}
for _, test := range testCases {
tc1 := NewTimestampCache(clock)
tc2 := NewTimestampCache(clock)
bfTS := clock.Now()
tc2.Add(roachpb.Key("b"), roachpb.Key("f"), bfTS, nil, true)
adTS := clock.Now()
tc1.Add(roachpb.Key("a"), roachpb.Key("d"), adTS, nil, true)
beTS := clock.Now()
tc1.Add(roachpb.Key("b"), roachpb.Key("e"), beTS, nil, true)
aaTS := clock.Now()
tc2.Add(roachpb.Key("aa"), nil, aaTS, nil, true)
cTS := clock.Now()
tc1.Add(roachpb.Key("c"), nil, cTS, nil, true)
tc1.MergeInto(tc2, test.useClear)
if tc2.cache.Len() != test.expLen {
t.Errorf("expected merged length of %d; got %d", test.expLen, tc2.cache.Len())
}
if !tc2.latest.Equal(tc1.latest) {
t.Errorf("expected latest to be updated to %s; got %s", tc1.latest, tc2.latest)
}
if rTS, _ := tc2.GetMax(roachpb.Key("a"), nil, nil); !rTS.Equal(adTS) {
t.Error("expected \"a\" to have adTS timestamp")
}
if rTS, _ := tc2.GetMax(roachpb.Key("b"), nil, nil); !rTS.Equal(beTS) {
t.Error("expected \"b\" to have beTS timestamp")
}
if test.useClear {
if rTS, _ := tc2.GetMax(roachpb.Key("aa"), nil, nil); !rTS.Equal(adTS) {
t.Error("expected \"aa\" to have adTS timestamp")
}
} else {
if rTS, _ := tc2.GetMax(roachpb.Key("aa"), nil, nil); !rTS.Equal(aaTS) {
t.Error("expected \"aa\" to have aaTS timestamp")
}
if rTS, _ := tc2.GetMax(roachpb.Key("a"), roachpb.Key("c"), nil); !rTS.Equal(aaTS) {
t.Error("expected \"a\"-\"c\" to have aaTS timestamp")
}
}
}
}
// TestClientHeartbeatBadServer verifies that the client is not marked
// as "ready" until a heartbeat request succeeds.
func TestClientHeartbeatBadServer(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
// Create a server without registering a heartbeat service.
serverClock := hlc.NewClock(hlc.UnixNano)
s := createTestServer(serverClock, stopper, t)
// Create a client. It should attempt a heartbeat and fail.
c := NewClient(s.Addr(), s.context)
// Register a heartbeat service.
heartbeat := &HeartbeatService{
clock: serverClock,
remoteClockMonitor: newRemoteClockMonitor(serverClock),
}
select {
case <-c.Healthy():
t.Error("client became healthy before a successful heartbeat")
case <-time.After(10 * time.Millisecond):
}
if err := heartbeat.Register(s); err != nil {
t.Fatalf("Unable to register heartbeat service: %s", err)
}
// A heartbeat should succeed and the client should become ready.
<-c.Healthy()
}
// TestScannerStats verifies that stats accumulate from all ranges.
func TestScannerStats(t *testing.T) {
defer leaktest.AfterTest(t)
const count = 3
ranges := newTestRangeSet(count, t)
q := &testQueue{}
stopper := util.NewStopper()
defer stopper.Stop()
s := newRangeScanner(1*time.Millisecond, 0, ranges, nil)
s.AddQueues(q)
mc := hlc.NewManualClock(0)
clock := hlc.NewClock(mc.UnixNano)
// At start, scanner stats should be blank for MVCC, but have accurate number of ranges.
if rc := s.Stats().RangeCount; rc != count {
t.Errorf("range count expected %d; got %d", count, rc)
}
if vb := s.Stats().MVCC.ValBytes; vb != 0 {
t.Errorf("value bytes expected %d; got %d", 0, vb)
}
s.Start(clock, stopper)
// We expect a full run to accumulate stats from all ranges.
if err := util.IsTrueWithin(func() bool {
if rc := s.Stats().RangeCount; rc != count {
return false
}
if vb := s.Stats().MVCC.ValBytes; vb != count*2 {
return false
}
return true
}, 100*time.Millisecond); err != nil {
t.Error(err)
}
}
// TestClientGossip verifies a client can gossip a delta to the server.
func TestClientGossip(t *testing.T) {
defer leaktest.AfterTest(t)
local, remote, stopper := startGossip(t)
disconnected := make(chan *client, 1)
client := newClient(remote.is.NodeAddr)
defer func() {
stopper.Stop()
if client != <-disconnected {
t.Errorf("expected client disconnect after remote close")
}
}()
if err := local.AddInfo("local-key", nil, time.Second); err != nil {
t.Fatal(err)
}
if err := remote.AddInfo("remote-key", nil, time.Second); err != nil {
t.Fatal(err)
}
// Use an insecure context. We're talking to tcp socket which are not in the certs.
lclock := hlc.NewClock(hlc.UnixNano)
rpcContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper)
client.start(local, disconnected, rpcContext, stopper)
util.SucceedsWithin(t, 500*time.Millisecond, func() error {
if _, err := remote.GetInfo("local-key"); err != nil {
return err
}
if _, err := local.GetInfo("remote-key"); err != nil {
return err
}
return nil
})
}
// createCluster generates a new cluster using the provided stopper and the
// number of nodes supplied. Each node will have one store to start.
func createCluster(stopper *stop.Stopper, nodeCount int) *Cluster {
rand, seed := randutil.NewPseudoRand()
clock := hlc.NewClock(hlc.UnixNano)
rpcContext := rpc.NewContext(&base.Context{}, clock, stopper)
g := gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap)
storePool := storage.NewStorePool(g, storage.TestTimeUntilStoreDeadOff, stopper)
c := &Cluster{
stopper: stopper,
clock: clock,
rpc: rpcContext,
gossip: g,
storePool: storePool,
allocator: storage.MakeAllocator(storePool, storage.RebalancingOptions{}),
storeGossiper: gossiputil.NewStoreGossiper(g),
nodes: make(map[proto.NodeID]*Node),
stores: make(map[proto.StoreID]*Store),
ranges: make(map[proto.RangeID]*Range),
rand: rand,
seed: seed,
}
// Add the nodes.
for i := 0; i < nodeCount; i++ {
c.addNewNodeWithStore()
}
// Add a single range and add to this first node's first store.
firstRange := c.addRange()
firstRange.attachRangeToStore(c.stores[proto.StoreID(0)])
return c
}
// startGossip creates and starts a gossip instance.
func startGossip(nodeID roachpb.NodeID, stopper *stop.Stopper, t *testing.T) *Gossip {
clock := hlc.NewClock(hlc.UnixNano)
rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper)
addr := util.CreateTestAddr("tcp")
server := rpc.NewServer(rpcContext)
tlsConfig, err := rpcContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
ln, err := util.ListenAndServe(stopper, server, addr, tlsConfig)
if err != nil {
t.Fatal(err)
}
g := New(rpcContext, TestBootstrap, stopper)
g.SetNodeID(nodeID)
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: nodeID,
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}); err != nil {
t.Fatal(err)
}
g.start(server, ln.Addr())
time.Sleep(time.Millisecond)
return g
}
// createTestNode creates an rpc server using the specified address,
// gossip instance, KV database and a node using the specified slice
// of engines. The server, clock and node are returned. If gossipBS is
// not nil, the gossip bootstrap address is set to gossipBS.
func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) (
*rpc.Server, *hlc.Clock, *Node, *stop.Stopper) {
ctx := storage.StoreContext{}
stopper := stop.NewStopper()
ctx.Clock = hlc.NewClock(hlc.UnixNano)
nodeRPCContext := rpc.NewContext(nodeTestBaseContext, ctx.Clock, stopper)
ctx.ScanInterval = 10 * time.Hour
rpcServer := rpc.NewServer(addr, nodeRPCContext)
if err := rpcServer.Start(); err != nil {
t.Fatal(err)
}
g := gossip.New(nodeRPCContext, testContext.GossipInterval, testContext.GossipBootstrapResolvers)
if gossipBS != nil {
// Handle possibility of a :0 port specification.
if gossipBS == addr {
gossipBS = rpcServer.Addr()
}
g.SetResolvers([]resolver.Resolver{resolver.NewResolverFromAddress(gossipBS)})
g.Start(rpcServer, stopper)
}
ctx.Gossip = g
sender := kv.NewDistSender(&kv.DistSenderContext{Clock: ctx.Clock}, g)
ctx.DB = client.NewDB(sender)
// TODO(bdarnell): arrange to have the transport closed.
// (or attach LocalRPCTransport.Close to the stopper)
ctx.Transport = multiraft.NewLocalRPCTransport(stopper)
ctx.EventFeed = util.NewFeed(stopper)
node := NewNode(ctx)
return rpcServer, ctx.Clock, node, stopper
}
// TestTxnCoordSenderErrorWithIntent validates that if a transactional request
// returns an error but also indicates a Writing transaction, the coordinator
// tracks it just like a successful request.
func TestTxnCoordSenderErrorWithIntent(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(20)
ts := NewTxnCoordSender(senderFn(func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
txn := ba.Txn.Clone()
txn.Writing = true
pErr := roachpb.NewError(roachpb.NewTransactionRetryError())
pErr.SetTxn(txn)
return nil, pErr
}), clock, false, nil, stopper)
defer stopper.Stop()
var ba roachpb.BatchRequest
key := roachpb.Key("test")
ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.EndTransactionRequest{})
ba.Txn = &roachpb.Transaction{Name: "test"}
if _, pErr := ts.Send(context.Background(), ba); !testutils.IsPError(pErr, "retry txn") {
t.Fatalf("unexpected error: %v", pErr)
}
defer teardownHeartbeats(ts)
ts.Lock()
defer ts.Unlock()
if len(ts.txns) != 1 {
t.Fatalf("expected transaction to be tracked")
}
}
// TestFindOffsetWithLargeError tests a case where offset errors are
// bigger than the max offset (e.g., a case where heartbeat messages
// to the node are having high latency).
func TestFindOffsetWithLargeError(t *testing.T) {
defer leaktest.AfterTest(t)()
maxOffset := 100 * time.Nanosecond
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(maxOffset)
offsets := map[string]RemoteOffset{}
// Offsets are bigger than maxOffset, but Errors are also bigger than Offset.
offsets["0"] = RemoteOffset{Offset: 110, Uncertainty: 300}
offsets["1"] = RemoteOffset{Offset: 120, Uncertainty: 300}
offsets["2"] = RemoteOffset{Offset: 130, Uncertainty: 300}
remoteClocks := newRemoteClockMonitor(clock)
remoteClocks.mu.offsets = offsets
interval, err := remoteClocks.findOffsetInterval()
if err != nil {
t.Fatal(err)
}
expectedInterval := clusterOffsetInterval{lowerbound: -270, upperbound: 510}
if interval != expectedInterval {
t.Errorf("expected interval %v, instead %v", expectedInterval, interval)
}
// The interval is still considered healthy.
assertIntervalHealth(true, interval, maxOffset, t)
}
// BootstrapCluster bootstraps a multiple stores using the provided engines and
// cluster ID. The first bootstrapped store contains a single range spanning
// all keys. Initial range lookup metadata is populated for the range.
//
// Returns a KV client for unittest purposes. Caller should close the returned
// client.
func BootstrapCluster(clusterID string, engines []engine.Engine, stopper *stop.Stopper) (*client.DB, error) {
ctx := storage.StoreContext{}
ctx.ScanInterval = 10 * time.Minute
ctx.Clock = hlc.NewClock(hlc.UnixNano)
// Create a KV DB with a local sender.
lSender := kv.NewLocalSender()
sender := kv.NewTxnCoordSender(lSender, ctx.Clock, false, nil, stopper)
ctx.DB = client.NewDB(sender)
ctx.Transport = multiraft.NewLocalRPCTransport(stopper)
for i, eng := range engines {
sIdent := roachpb.StoreIdent{
ClusterID: clusterID,
NodeID: 1,
StoreID: roachpb.StoreID(i + 1),
}
// The bootstrapping store will not connect to other nodes so its
// StoreConfig doesn't really matter.
s := storage.NewStore(ctx, eng, &roachpb.NodeDescriptor{NodeID: 1})
// Verify the store isn't already part of a cluster.
if len(s.Ident.ClusterID) > 0 {
return nil, util.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID)
}
// Bootstrap store to persist the store ident.
if err := s.Bootstrap(sIdent, stopper); err != nil {
return nil, err
}
// Create first range, writing directly to engine. Note this does
// not create the range, just its data. Only do this if this is the
// first store.
if i == 0 {
// TODO(marc): this is better than having storage/ import sql, but still
// not great. Find a better place to keep those.
initialValues := sql.GetInitialSystemValues()
if err := s.BootstrapRange(initialValues); err != nil {
return nil, err
}
}
if err := s.Start(stopper); err != nil {
return nil, err
}
lSender.AddStore(s)
// Initialize node and store ids. Only initialize the node once.
if i == 0 {
if nodeID, err := allocateNodeID(ctx.DB); nodeID != sIdent.NodeID || err != nil {
return nil, util.Errorf("expected to initialize node id allocator to %d, got %d: %s",
sIdent.NodeID, nodeID, err)
}
}
if storeID, err := allocateStoreIDs(sIdent.NodeID, 1, ctx.DB); storeID != sIdent.StoreID || err != nil {
return nil, util.Errorf("expected to initialize store id allocator to %d, got %d: %s",
sIdent.StoreID, storeID, err)
}
}
return ctx.DB, nil
}
// TestTimestampCacheNoEviction verifies that even after
// the MinTSCacheWindow interval, if the cache has not hit
// its size threshold, it will not evict entries.
func TestTimestampCacheNoEviction(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(maxClockOffset)
tc := newTimestampCache(clock)
// Increment time to the maxClockOffset low water mark + 1.
manual.Set(maxClockOffset.Nanoseconds() + 1)
aTS := clock.Now()
tc.add(roachpb.Key("a"), nil, aTS, nil, true)
tc.AddRequest(cacheRequest{
reads: []roachpb.Span{{Key: roachpb.Key("c")}},
timestamp: aTS,
})
// Increment time by the MinTSCacheWindow and add another key.
manual.Increment(MinTSCacheWindow.Nanoseconds())
tc.add(roachpb.Key("b"), nil, clock.Now(), nil, true)
tc.AddRequest(cacheRequest{
reads: []roachpb.Span{{Key: roachpb.Key("d")}},
timestamp: clock.Now(),
})
// Verify that the cache still has 4 entries in it
if l, want := tc.len(), 4; l != want {
t.Errorf("expected %d entries to remain, got %d", want, l)
}
}
// TestBuildEndpointListRemoveStagnantClocks tests the side effect of removing
// older offsets when we build an endpoint list.
func TestBuildEndpointListRemoveStagnantClocks(t *testing.T) {
defer leaktest.AfterTest(t)()
offsets := map[string]RemoteOffset{
"0": {Offset: 0, Uncertainty: 10, MeasuredAt: 11},
"stagnant0": {Offset: 1, Uncertainty: 10, MeasuredAt: 0},
"1": {Offset: 2, Uncertainty: 10, MeasuredAt: 20},
"stagnant1": {Offset: 3, Uncertainty: 10, MeasuredAt: 9},
}
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(5 * time.Nanosecond)
remoteClocks := newRemoteClockMonitor(clock)
// The stagnant offsets older than this will be removed.
remoteClocks.monitorInterval = 10 * time.Nanosecond
remoteClocks.mu.offsets = offsets
remoteClocks.mu.lastMonitoredAt = time.Unix(0, 10) // offsets measured before this will be removed.
remoteClocks.buildEndpointList()
_, ok0 := offsets["stagnant0"]
_, ok1 := offsets["stagnant1"]
if ok0 || ok1 {
t.Errorf("expected stagant offsets removed, instead offsets: %v", offsets)
}
}
// TestBootstrapOfNonEmptyStore verifies bootstrap failure if engine
// is not empty.
func TestBootstrapOfNonEmptyStore(t *testing.T) {
defer leaktest.AfterTest(t)
eng := engine.NewInMem(proto.Attributes{}, 1<<20)
// Put some random garbage into the engine.
if err := eng.Put(proto.EncodedKey("foo"), []byte("bar")); err != nil {
t.Errorf("failure putting key foo into engine: %s", err)
}
ctx := TestStoreContext
manual := hlc.NewManualClock(0)
ctx.Clock = hlc.NewClock(manual.UnixNano)
ctx.Transport = multiraft.NewLocalRPCTransport()
stopper := stop.NewStopper()
stopper.AddCloser(ctx.Transport)
defer stopper.Stop()
store := NewStore(ctx, eng, &proto.NodeDescriptor{NodeID: 1})
// Can't init as haven't bootstrapped.
if err := store.Start(stopper); err == nil {
t.Error("expected failure init'ing un-bootstrapped store")
}
// Bootstrap should fail on non-empty engine.
if err := store.Bootstrap(testIdent, stopper); err == nil {
t.Error("expected bootstrap error on non-empty store")
}
}
// TestRejectFutureCommand verifies that lease holders reject commands that
// would cause a large time jump.
func TestRejectFutureCommand(t *testing.T) {
defer leaktest.AfterTest(t)()
const maxOffset = 100 * time.Millisecond
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(maxOffset)
mtc := multiTestContext{
clock: clock,
}
mtc.Start(t, 1)
defer mtc.Stop()
// First do a write. The first write will advance the clock by MaxOffset
// because of the read cache's low water mark.
getArgs := putArgs([]byte("b"), []byte("b"))
if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &getArgs); err != nil {
t.Fatal(err)
}
if now := clock.Now(); now.WallTime != int64(maxOffset) {
t.Fatalf("expected clock to advance to 100ms; got %s", now)
}
// The logical clock has advanced past the physical clock; increment
// the "physical" clock to catch up.
manual.Increment(int64(maxOffset))
startTime := manual.UnixNano()
// Commands with a future timestamp that is within the MaxOffset
// bound will be accepted and will cause the clock to advance.
for i := int64(0); i < 3; i++ {
incArgs := incrementArgs([]byte("a"), 5)
ts := hlc.ZeroTimestamp.Add(startTime+((i+1)*30)*int64(time.Millisecond), 0)
if _, err := client.SendWrappedWith(rg1(mtc.stores[0]), nil, roachpb.Header{Timestamp: ts}, &incArgs); err != nil {
t.Fatal(err)
}
}
if now := clock.Now(); now.WallTime != int64(190*time.Millisecond) {
t.Fatalf("expected clock to advance to 190ms; got %s", now)
}
// Once the accumulated offset reaches MaxOffset, commands will be rejected.
incArgs := incrementArgs([]byte("a"), 11)
ts := hlc.ZeroTimestamp.Add(int64((time.Duration(startTime)+maxOffset+1)*time.Millisecond), 0)
if _, err := client.SendWrappedWith(rg1(mtc.stores[0]), nil, roachpb.Header{Timestamp: ts}, &incArgs); err == nil {
t.Fatalf("expected clock offset error but got nil")
}
// The clock remained at 190ms and the final command was not executed.
if now := clock.Now(); now.WallTime != int64(190*time.Millisecond) {
t.Errorf("expected clock to advance to 190ms; got %s", now)
}
val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], roachpb.Key("a"), clock.Now(), true, nil)
if err != nil {
t.Fatal(err)
}
if v := mustGetInt(val); v != 15 {
t.Errorf("expected 15, got %v", v)
}
}
// TestScannerTiming verifies that ranges are scanned, regardless
// of how many, to match scanInterval.
func TestScannerTiming(t *testing.T) {
defer leaktest.AfterTest(t)()
const count = 3
const runTime = 100 * time.Millisecond
const maxError = 7500 * time.Microsecond
durations := []time.Duration{
15 * time.Millisecond,
25 * time.Millisecond,
}
for i, duration := range durations {
util.SucceedsSoon(t, func() error {
ranges := newTestRangeSet(count, t)
q := &testQueue{}
s := newReplicaScanner(duration, 0, ranges)
s.AddQueues(q)
mc := hlc.NewManualClock(0)
clock := hlc.NewClock(mc.UnixNano)
stopper := stop.NewStopper()
s.Start(clock, stopper)
time.Sleep(runTime)
stopper.Stop()
avg := s.avgScan()
log.Infof("%d: average scan: %s", i, avg)
if avg.Nanoseconds()-duration.Nanoseconds() > maxError.Nanoseconds() ||
duration.Nanoseconds()-avg.Nanoseconds() > maxError.Nanoseconds() {
return errors.Errorf("expected %s, got %s: exceeds max error of %s", duration, avg, maxError)
}
return nil
})
}
}