// StartWithStopper is the same as Start, but allows passing a stopper
// explicitly.
func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error {
if ts.Ctx == nil {
ts.Ctx = NewTestContext()
}
if stopper == nil {
stopper = stop.NewStopper()
}
// Change the replication requirements so we don't get log spam
// about ranges not being replicated enough.
// TODO(marc): set this in the zones table when we have an entry
// for the default cluster-wide zone config and remove these
// shenanigans about mutating the global default.
oldDefaultZC := proto.Clone(config.DefaultZoneConfig).(*config.ZoneConfig)
config.DefaultZoneConfig.ReplicaAttrs = []roachpb.Attributes{{}}
stopper.AddCloser(stop.CloserFn(func() {
config.DefaultZoneConfig = oldDefaultZC
}))
var err error
ts.Server, err = NewServer(ts.Ctx, stopper)
if err != nil {
return err
}
// Ensure we have the correct number of engines. Add in in-memory ones where
// needed. There must be at least one store/engine.
if ts.StoresPerNode < 1 {
ts.StoresPerNode = 1
}
for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ {
ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, ts.Server.stopper))
}
if !ts.SkipBootstrap {
stopper := stop.NewStopper()
_, err := BootstrapCluster("cluster-1", ts.Ctx.Engines, stopper)
if err != nil {
return util.Errorf("could not bootstrap cluster: %s", err)
}
stopper.Stop()
}
if err := ts.Server.Start(true); err != nil {
return err
}
// If enabled, wait for initial splits to complete before returning control.
// If initial splits do not complete, the server is stopped before
// returning.
if config.TestingTableSplitsDisabled() {
return nil
}
if err := ts.WaitForInitialSplits(); err != nil {
ts.Stop()
return err
}
return nil
}
// TestCorruptedClusterID verifies that a node fails to start when a
// store's cluster ID is empty.
func TestCorruptedClusterID(t *testing.T) {
defer leaktest.AfterTest(t)
engineStopper := stop.NewStopper()
e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)
defer engineStopper.Stop()
eagerStopper := stop.NewStopper()
_, err := BootstrapCluster("cluster-1", []engine.Engine{e}, eagerStopper)
if err != nil {
t.Fatal(err)
}
eagerStopper.Stop()
// Set the cluster ID to an empty string.
sIdent := roachpb.StoreIdent{
ClusterID: "",
NodeID: 1,
StoreID: 1,
}
if err = engine.MVCCPutProto(e, nil, keys.StoreIdentKey(), roachpb.ZeroTimestamp, nil, &sIdent); err != nil {
t.Fatal(err)
}
engines := []engine.Engine{e}
server, serverAddr, _, node, stopper := createTestNode(util.CreateTestAddr("tcp"), engines, nil, t)
if err := node.start(server, serverAddr, engines, roachpb.Attributes{}, stopper); err == nil {
t.Errorf("unexpected success")
}
stopper.Stop()
}
func TestGossipOrphanedStallDetection(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t, metric.NewRegistry())
local.SetStallInterval(5 * time.Millisecond)
// Make sure we have the sentinel to ensure that its absence is not the
// cause of stall detection.
if err := local.AddInfo(KeySentinel, nil, time.Hour); err != nil {
t.Fatal(err)
}
peerStopper := stop.NewStopper()
peer := startGossip(2, peerStopper, t, metric.NewRegistry())
peerNodeID := peer.GetNodeID()
peerAddr := peer.GetNodeAddr()
local.startClient(peerAddr, peerNodeID)
util.SucceedsSoon(t, func() error {
for _, peerID := range local.Outgoing() {
if peerID == peerNodeID {
return nil
}
}
return errors.Errorf("%d not yet connected", peerNodeID)
})
local.bootstrap()
local.manage()
peerStopper.Stop()
util.SucceedsSoon(t, func() error {
for _, peerID := range local.Outgoing() {
if peerID == peerNodeID {
return errors.Errorf("%d still connected", peerNodeID)
}
}
return nil
})
peerStopper = stop.NewStopper()
defer peerStopper.Stop()
peer = startGossipAtAddr(peerNodeID, peerAddr, peerStopper, t, metric.NewRegistry())
util.SucceedsSoon(t, func() error {
for _, peerID := range local.Outgoing() {
if peerID == peerNodeID {
return nil
}
}
return errors.Errorf("%d not yet connected", peerNodeID)
})
}
func (m *multiTestContext) Start(t *testing.T, numStores int) {
m.t = t
if m.manualClock == nil {
m.manualClock = hlc.NewManualClock(0)
}
if m.clock == nil {
m.clock = hlc.NewClock(m.manualClock.UnixNano)
}
if m.gossip == nil {
rpcContext := rpc.NewContext(&base.Context{}, m.clock, nil)
m.gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap)
}
if m.clientStopper == nil {
m.clientStopper = stop.NewStopper()
}
if m.transport == nil {
m.transport = multiraft.NewLocalRPCTransport(m.clientStopper)
}
if m.storePool == nil {
if m.timeUntilStoreDead == 0 {
m.timeUntilStoreDead = storage.TestTimeUntilStoreDeadOff
}
m.storePool = storage.NewStorePool(m.gossip, m.timeUntilStoreDead, m.clientStopper)
}
// Always create the first sender.
m.senders = append(m.senders, kv.NewLocalSender())
rpcSend := func(_ rpc.Options, _ string, _ []net.Addr,
getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message,
_ *rpc.Context) ([]gogoproto.Message, error) {
call := proto.Call{
Args: getArgs(nil /* net.Addr */).(proto.Request),
Reply: getReply().(proto.Response),
}
m.senders[0].Send(context.Background(), call)
return []gogoproto.Message{call.Reply}, call.Reply.Header().GoError()
}
if m.db == nil {
distSender := kv.NewDistSender(&kv.DistSenderContext{
Clock: m.clock,
RangeDescriptorDB: m.senders[0],
RPCSend: rpcSend,
}, m.gossip)
sender := kv.NewTxnCoordSender(distSender, m.clock, false, nil, m.clientStopper)
m.db = client.NewDB(sender)
}
for i := 0; i < numStores; i++ {
m.addStore()
}
if m.transportStopper == nil {
m.transportStopper = stop.NewStopper()
}
m.transportStopper.AddCloser(m.transport)
}
// TestRetryableError verifies that Send returns a retryable error
// when it hits an RPC error.
func TestRetryableError(t *testing.T) {
defer leaktest.AfterTest(t)()
clientStopper := stop.NewStopper()
defer clientStopper.Stop()
clientContext := newNodeTestContext(nil, clientStopper)
serverStopper := stop.NewStopper()
serverContext := newNodeTestContext(nil, serverStopper)
s, ln := newTestServer(t, serverContext)
roachpb.RegisterInternalServer(s, Node(0))
conn, err := clientContext.GRPCDial(ln.Addr().String())
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
waitForConnState := func(desiredState grpc.ConnectivityState) {
clientState, err := conn.State()
for clientState != desiredState {
if err != nil {
t.Fatal(err)
}
if clientState == grpc.Shutdown {
t.Fatalf("%v has unexpectedly shut down", conn)
}
clientState, err = conn.WaitForStateChange(ctx, clientState)
}
}
// Wait until the client becomes healthy and shut down the server.
waitForConnState(grpc.Ready)
serverStopper.Stop()
// Wait until the client becomes unhealthy.
waitForConnState(grpc.TransientFailure)
sp := tracing.NewTracer().StartSpan("node test")
defer sp.Finish()
opts := SendOptions{
Ordering: orderStable,
SendNextTimeout: 100 * time.Millisecond,
Timeout: 100 * time.Millisecond,
Trace: sp,
}
if _, err := sendBatch(opts, []net.Addr{ln.Addr()}, clientContext); err != nil {
retryErr, ok := err.(retry.Retryable)
if !ok {
t.Fatalf("Unexpected error type: %v", err)
}
if !retryErr.CanRetry() {
t.Errorf("Expected retryable error: %v", retryErr)
}
} else {
t.Fatalf("Unexpected success")
}
}
// StartTestCluster starts up a TestCluster made up of `nodes` in-memory testing
// servers.
// The cluster should be stopped using cluster.Stop().
func StartTestCluster(t testing.TB, nodes int, args base.TestClusterArgs) *TestCluster {
if nodes < 1 {
t.Fatal("invalid cluster size: ", nodes)
}
if args.ServerArgs.JoinAddr != "" {
t.Fatal("can't specify a join addr when starting a cluster")
}
if args.ServerArgs.Stopper != nil {
t.Fatal("can't set individual server stoppers when starting a cluster")
}
storeKnobs := args.ServerArgs.Knobs.Store
if storeKnobs != nil &&
(storeKnobs.(*storage.StoreTestingKnobs).DisableSplitQueue ||
storeKnobs.(*storage.StoreTestingKnobs).DisableReplicateQueue) {
t.Fatal("can't disable an individual server's queues when starting a cluster; " +
"the cluster controls replication")
}
switch args.ReplicationMode {
case base.ReplicationAuto:
case base.ReplicationManual:
if args.ServerArgs.Knobs.Store == nil {
args.ServerArgs.Knobs.Store = &storage.StoreTestingKnobs{}
}
storeKnobs := args.ServerArgs.Knobs.Store.(*storage.StoreTestingKnobs)
storeKnobs.DisableSplitQueue = true
storeKnobs.DisableReplicateQueue = true
default:
t.Fatal("unexpected replication mode")
}
tc := &TestCluster{}
tc.stopper = stop.NewStopper()
args.ServerArgs.PartOfCluster = true
for i := 0; i < nodes; i++ {
serverArgs := args.ServerArgs
serverArgs.Stopper = stop.NewStopper()
if i > 0 {
serverArgs.JoinAddr = tc.Servers[0].ServingAddr()
}
s, conn, _ := serverutils.StartServer(t, serverArgs)
tc.Servers = append(tc.Servers, s.(*server.TestServer))
tc.Conns = append(tc.Conns, conn)
tc.mu.Lock()
tc.mu.serverStoppers = append(tc.mu.serverStoppers, serverArgs.Stopper)
tc.mu.Unlock()
}
// Create a closer that will stop the individual server stoppers when the
// cluster stopper is stopped.
tc.stopper.AddCloser(stop.CloserFn(tc.stopServers))
tc.waitForStores(t)
return tc
}
// TestStoreRecoverWithErrors verifies that even commands that fail are marked as
// applied so they are not retried after recovery.
func TestStoreRecoverWithErrors(t *testing.T) {
defer leaktest.AfterTest(t)
defer func() { storage.TestingCommandFilter = nil }()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
engineStopper := stop.NewStopper()
defer engineStopper.Stop()
eng := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)
numIncrements := 0
storage.TestingCommandFilter = func(_ roachpb.StoreID, args roachpb.Request, _ roachpb.Header) error {
if _, ok := args.(*roachpb.IncrementRequest); ok && args.Header().Key.Equal(roachpb.Key("a")) {
numIncrements++
}
return nil
}
func() {
stopper := stop.NewStopper()
defer stopper.Stop()
store := createTestStoreWithEngine(t, eng, clock, true, nil, stopper)
// Write a bytes value so the increment will fail.
putArgs := putArgs(roachpb.Key("a"), []byte("asdf"))
if _, err := client.SendWrapped(rg1(store), nil, &putArgs); err != nil {
t.Fatal(err)
}
// Try and fail to increment the key. It is important for this test that the
// failure be the last thing in the raft log when the store is stopped.
incArgs := incrementArgs(roachpb.Key("a"), 42)
if _, err := client.SendWrapped(rg1(store), nil, &incArgs); err == nil {
t.Fatal("did not get expected error")
}
}()
if numIncrements != 1 {
t.Fatalf("expected 1 increments; was %d", numIncrements)
}
// Recover from the engine.
store := createTestStoreWithEngine(t, eng, clock, false, nil, engineStopper)
// Issue a no-op write to lazily initialize raft on the range.
incArgs := incrementArgs(roachpb.Key("b"), 0)
if _, err := client.SendWrapped(rg1(store), nil, &incArgs); err != nil {
t.Fatal(err)
}
// No additional increments were performed on key A during recovery.
if numIncrements != 1 {
t.Fatalf("expected 1 increments; was %d", numIncrements)
}
}
// StartWithStopper is the same as Start, but allows passing a stopper
// explicitly.
func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error {
if ts.Ctx == nil {
ts.Ctx = NewTestContext()
}
// TODO(marc): set this in the zones table when we have an entry
// for the default cluster-wide zone config.
config.DefaultZoneConfig.ReplicaAttrs = []roachpb.Attributes{{}}
if stopper == nil {
stopper = stop.NewStopper()
}
var err error
ts.Server, err = NewServer(ts.Ctx, stopper)
if err != nil {
return err
}
// Ensure we have the correct number of engines. Add in in-memory ones where
// needed. There must be at least one store/engine.
if ts.StoresPerNode < 1 {
ts.StoresPerNode = 1
}
for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ {
ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, ts.Server.stopper))
}
if !ts.SkipBootstrap {
stopper := stop.NewStopper()
_, err := BootstrapCluster("cluster-1", ts.Ctx.Engines, stopper)
if err != nil {
return util.Errorf("could not bootstrap cluster: %s", err)
}
stopper.Stop()
}
if err := ts.Server.Start(true); err != nil {
return err
}
// If enabled, wait for initial splits to complete before returning control.
// If initial splits do not complete, the server is stopped before
// returning.
if config.TestingTableSplitsDisabled() {
return nil
}
if err := ts.WaitForInitialSplits(); err != nil {
ts.Stop()
return err
}
return nil
}
// TestNodeJoin verifies a new node is able to join a bootstrapped
// cluster consisting of one node.
func TestNodeJoin(t *testing.T) {
defer leaktest.AfterTest(t)
engineStopper := stop.NewStopper()
defer engineStopper.Stop()
e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)
stopper := stop.NewStopper()
_, err := BootstrapCluster("cluster-1", []engine.Engine{e}, stopper)
if err != nil {
t.Fatal(err)
}
stopper.Stop()
// Set an aggressive gossip interval to make sure information is exchanged tout de suite.
testContext.GossipInterval = gossip.TestInterval
// Start the bootstrap node.
engines1 := []engine.Engine{e}
addr1 := util.CreateTestAddr("tcp")
server1, node1, stopper1 := createAndStartTestNode(addr1, engines1, addr1, t)
defer stopper1.Stop()
// Create a new node.
engines2 := []engine.Engine{engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)}
server2, node2, stopper2 := createAndStartTestNode(util.CreateTestAddr("tcp"), engines2, server1.Addr(), t)
defer stopper2.Stop()
// Verify new node is able to bootstrap its store.
if err := util.IsTrueWithin(func() bool { return node2.lSender.GetStoreCount() == 1 }, 50*time.Millisecond); err != nil {
t.Fatal(err)
}
// Verify node1 sees node2 via gossip and vice versa.
node1Key := gossip.MakeNodeIDKey(node1.Descriptor.NodeID)
node2Key := gossip.MakeNodeIDKey(node2.Descriptor.NodeID)
if err := util.IsTrueWithin(func() bool {
nodeDesc1 := &roachpb.NodeDescriptor{}
if err := node1.ctx.Gossip.GetInfoProto(node2Key, nodeDesc1); err != nil {
return false
}
if addr2 := nodeDesc1.Address.AddressField; addr2 != server2.Addr().String() {
t.Errorf("addr2 gossip %s doesn't match addr2 address %s", addr2, server2.Addr().String())
}
nodeDesc2 := &roachpb.NodeDescriptor{}
if err := node2.ctx.Gossip.GetInfoProto(node1Key, nodeDesc2); err != nil {
return false
}
if addr1 := nodeDesc2.Address.AddressField; addr1 != server1.Addr().String() {
t.Errorf("addr1 gossip %s doesn't match addr1 address %s", addr1, server1.Addr().String())
}
return true
}, 50*time.Millisecond); err != nil {
t.Error(err)
}
}
// TestRetryableError verifies that Send returns a retryable error
// when it hits an RPC error.
func TestRetryableError(t *testing.T) {
defer leaktest.AfterTest(t)()
clientStopper := stop.NewStopper()
defer clientStopper.Stop()
clientContext := newNodeTestContext(nil, clientStopper)
clientContext.HeartbeatTimeout = 10 * clientContext.HeartbeatInterval
serverStopper := stop.NewStopper()
serverContext := newNodeTestContext(nil, serverStopper)
s, ln := newTestServer(t, serverContext)
registerBatch(t, s, 0)
c := rpc.NewClient(ln.Addr(), clientContext)
// Wait until the client becomes healthy and shut down the server.
<-c.Healthy()
serverStopper.Stop()
// Wait until the client becomes unhealthy.
func() {
for r := retry.Start(retry.Options{}); r.Next(); {
select {
case <-c.Healthy():
case <-time.After(1 * time.Nanosecond):
return
}
}
}()
sp := tracing.NewTracer().StartSpan("node test")
defer sp.Finish()
opts := SendOptions{
Ordering: orderStable,
SendNextTimeout: 100 * time.Millisecond,
Timeout: 100 * time.Millisecond,
Trace: sp,
}
if _, err := sendBatch(opts, []net.Addr{ln.Addr()}, clientContext); err != nil {
retryErr, ok := err.(retry.Retryable)
if !ok {
t.Fatalf("Unexpected error type: %v", err)
}
if !retryErr.CanRetry() {
t.Errorf("Expected retryable error: %v", retryErr)
}
} else {
t.Fatalf("Unexpected success")
}
}
func TestBatchDefer(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
e := NewInMem(roachpb.Attributes{}, 1<<20, stopper)
b := e.NewBatch()
defer b.Close()
list := []string{}
b.Defer(func() {
list = append(list, "one")
})
b.Defer(func() {
list = append(list, "two")
})
if err := b.Commit(); err != nil {
t.Fatal(err)
}
// Order was reversed when the defers were run.
if !reflect.DeepEqual(list, []string{"two", "one"}) {
t.Errorf("expected [two, one]; got %v", list)
}
}
// TestBatchScanMaxWithDeleted verifies that if a deletion
// in the updates map shadows an entry from the engine, the
// max on a scan is still reached.
func TestBatchScanMaxWithDeleted(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
e := NewInMem(roachpb.Attributes{}, 1<<20, stopper)
b := e.NewBatch()
defer b.Close()
// Write two values.
if err := e.Put(MVCCKey("a"), []byte("value1")); err != nil {
t.Fatal(err)
}
if err := e.Put(MVCCKey("b"), []byte("value2")); err != nil {
t.Fatal(err)
}
// Now, delete "a" in batch.
if err := b.Clear(MVCCKey("a")); err != nil {
t.Fatal(err)
}
// A scan with max=1 should scan "b".
kvs, err := Scan(b, MVCCKey(roachpb.RKeyMin), MVCCKey(roachpb.RKeyMax), 1)
if err != nil {
t.Fatal(err)
}
if len(kvs) != 1 || !bytes.Equal(kvs[0].Key, []byte("b")) {
t.Errorf("expected scan of \"b\"; got %v", kvs)
}
}
// TestBatchConcurrency verifies operation of batch when the
// underlying engine has concurrent modifications to overlapping
// keys. This should never happen with the way Cockroach uses
// batches, but worth verifying.
func TestBatchConcurrency(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
e := NewInMem(roachpb.Attributes{}, 1<<20, stopper)
b := e.NewBatch()
defer b.Close()
// Write a merge to the batch.
if err := b.Merge(MVCCKey("a"), appender("bar")); err != nil {
t.Fatal(err)
}
val, err := b.Get(MVCCKey("a"))
if err != nil {
t.Fatal(err)
}
if !compareMergedValues(t, val, appender("bar")) {
t.Error("mismatch of \"a\"")
}
// Write an engine value.
if err := e.Put(MVCCKey("a"), appender("foo")); err != nil {
t.Fatal(err)
}
// Now, read again and verify that the merge happens on top of the mod.
val, err = b.Get(MVCCKey("a"))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(val, appender("foobar")) {
t.Error("mismatch of \"a\"")
}
}
func TestStopper(t *testing.T) {
defer leaktest.AfterTest(t)
s := stop.NewStopper()
running := make(chan struct{})
waiting := make(chan struct{})
cleanup := make(chan struct{})
s.RunWorker(func() {
<-running
})
go func() {
<-s.ShouldStop()
select {
case <-waiting:
t.Fatal("expected stopper to have blocked")
case <-time.After(1 * time.Millisecond):
// Expected.
}
close(running)
select {
case <-waiting:
// Success.
case <-time.After(100 * time.Millisecond):
t.Fatal("stopper should have finished waiting")
}
close(cleanup)
}()
s.Stop()
close(waiting)
<-cleanup
}
func TestStopperIsStopped(t *testing.T) {
defer leaktest.AfterTest(t)
s := stop.NewStopper()
bc := newBlockingCloser()
s.AddCloser(bc)
go s.Stop()
select {
case <-s.ShouldStop():
case <-time.After(100 * time.Millisecond):
t.Fatal("stopper should have finished waiting")
}
select {
case <-s.IsStopped():
t.Fatal("expected blocked closer to prevent stop")
case <-time.After(1 * time.Millisecond):
// Expected.
}
bc.Unblock()
select {
case <-s.IsStopped():
// Expected
case <-time.After(100 * time.Millisecond):
t.Fatal("stopper should have finished stopping")
}
}
func benchmarkIterOnBatch(b *testing.B, writes int) {
stopper := stop.NewStopper()
defer stopper.Stop()
engine := createTestEngine(stopper)
for i := 0; i < writes; i++ {
if err := engine.Put(makeKey(i), []byte(strconv.Itoa(i))); err != nil {
b.Fatal(err)
}
}
batch := engine.NewBatch()
defer batch.Close()
for i := 0; i < writes; i++ {
if err := batch.Clear(makeKey(i)); err != nil {
b.Fatal(err)
}
}
r := rand.New(rand.NewSource(5))
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := makeKey(r.Intn(writes))
iter := batch.NewIterator(true)
iter.Seek(key)
iter.Close()
}
}
// openRocksDBWithVersion attempts to open a rocks db instance, optionally with
// the supplied Version struct.
func openRocksDBWithVersion(t *testing.T, hasVersionFile bool, ver Version) error {
stopper := stop.NewStopper()
defer stopper.Stop()
dir, err := ioutil.TempDir("", "testing")
if err != nil {
t.Fatal(err)
}
defer func() {
if err := os.RemoveAll(dir); err != nil {
t.Fatal(err)
}
}()
if hasVersionFile {
b, err := json.Marshal(ver)
if err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(getVersionFilename(dir), b, 0644); err != nil {
t.Fatal(err)
}
}
rocksdb := NewRocksDB(roachpb.Attributes{}, dir, 0, minMemtableBudget, 0, stopper)
return rocksdb.Open()
}
// TestHeartbeatResponseFanout check 2 raft groups on the same node distribution,
// but each group has different Term, heartbeat response from each group should
// not disturb other group's Term or Leadership
func TestHeartbeatResponseFanout(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
cluster := newTestCluster(nil, 3, stopper, t)
groupID1 := roachpb.RangeID(1)
cluster.createGroup(groupID1, 0, 3 /* replicas */)
groupID2 := roachpb.RangeID(2)
cluster.createGroup(groupID2, 0, 3 /* replicas */)
leaderIndex := 0
cluster.elect(leaderIndex, groupID1)
// GroupID2 will have 3 round of election, so it will have different
// term with groupID1, but both leader on the same node.
for i := 2; i >= 0; i-- {
leaderIndex = i
cluster.elect(leaderIndex, groupID2)
}
// Send a coalesced heartbeat.
// Heartbeat response from groupID2 will have a big term than which from groupID1.
cluster.nodes[0].coalescedHeartbeat()
// Start submit a command to see if groupID1's leader changed?
cluster.nodes[0].SubmitCommand(groupID1, makeCommandID(), []byte("command"))
select {
case _ = <-cluster.events[0].CommandCommitted:
log.Infof("SubmitCommand succeed after Heartbeat Response fanout")
case <-time.After(500 * time.Millisecond):
t.Fatalf("No leader after Heartbeat Response fanout")
}
}
// TestTxnCoordSenderSingleRoundtripTxn checks that a batch which completely
// holds the writing portion of a Txn (including EndTransaction) does not
// launch a heartbeat goroutine at all.
func TestTxnCoordSenderSingleRoundtripTxn(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(20)
ts := NewTxnCoordSender(senderFn(func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
br := ba.CreateReply()
br.Txn = ba.Txn.Clone()
br.Txn.Writing = true
return br, nil
}), clock, false, nil, stopper)
// Stop the stopper manually, prior to trying the transaction. This has the
// effect of returning a NodeUnavailableError for any attempts at launching
// a heartbeat goroutine.
stopper.Stop()
var ba roachpb.BatchRequest
key := roachpb.Key("test")
ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.EndTransactionRequest{})
ba.Txn = &roachpb.Transaction{Name: "test"}
_, pErr := ts.Send(context.Background(), ba)
if pErr != nil {
t.Fatal(pErr)
}
}
// TestInfoStoreMostDistant verifies selection of most distant node &
// associated hops.
func TestInfoStoreMostDistant(t *testing.T) {
defer leaktest.AfterTest(t)()
nodes := []roachpb.NodeID{
roachpb.NodeID(1),
roachpb.NodeID(2),
roachpb.NodeID(3),
}
stopper := stop.NewStopper()
defer stopper.Stop()
is := newInfoStore(context.TODO(), 1, emptyAddr, stopper)
// Add info from each address, with hop count equal to index+1.
for i := 0; i < len(nodes); i++ {
inf := is.newInfo(nil, time.Second)
inf.Hops = uint32(i + 1)
inf.NodeID = nodes[i]
if err := is.addInfo(fmt.Sprintf("b.%d", i), inf); err != nil {
t.Fatal(err)
}
nodeID, hops := is.mostDistant()
if nodeID != inf.NodeID {
t.Errorf("%d: expected node %d; got %d", i, inf.NodeID, nodeID)
}
if hops != inf.Hops {
t.Errorf("%d: expected node %d; got %d", i, inf.Hops, hops)
}
}
}
// Verify that adding two infos with different hops but same keys
// always chooses the minimum hops.
func TestAddInfoSameKeyDifferentHops(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
is := newInfoStore(context.TODO(), 1, emptyAddr, stopper)
info1 := is.newInfo(nil, time.Second)
info1.Hops = 1
info2 := is.newInfo(nil, time.Second)
info2.Value.Timestamp.WallTime = info1.Value.Timestamp.WallTime
info2.Hops = 2
if err := is.addInfo("a", info1); err != nil {
t.Errorf("failed insert: %s", err)
}
if err := is.addInfo("a", info2); err == nil {
t.Errorf("shouldn't have inserted info 2: %s", err)
}
i := is.getInfo("a")
if i.Hops != info1.Hops || !proto.Equal(i, info1) {
t.Error("failed to properly combine hops and value", i)
}
// Try yet another info, with lower hops yet (0).
info3 := is.newInfo(nil, time.Second)
if err := is.addInfo("a", info3); err != nil {
t.Error(err)
}
i = is.getInfo("a")
if i.Hops != info3.Hops || !proto.Equal(i, info3) {
t.Error("failed to properly combine hops and value", i)
}
}
// TestClientDisallowMultipleConns verifies that the server disallows
// multiple connections from the same client node ID.
func TestClientDisallowMultipleConns(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t)
remote := startGossip(2, stopper, t)
local.mu.Lock()
remote.mu.Lock()
rAddr := remote.is.NodeAddr
// Start two clients from local to remote. RPC client cache is
// disabled via the context, so we'll start two different outgoing
// connections.
local.startClient(&rAddr, stopper)
local.startClient(&rAddr, stopper)
local.mu.Unlock()
remote.mu.Unlock()
local.manage()
remote.manage()
util.SucceedsSoon(t, func() error {
// Verify that the remote server has only a single incoming
// connection and the local server has only a single outgoing
// connection.
local.mu.Lock()
remote.mu.Lock()
outgoing := local.outgoing.len()
incoming := remote.incoming.len()
local.mu.Unlock()
remote.mu.Unlock()
if outgoing == 1 && incoming == 1 && verifyServerMaps(local, 0) && verifyServerMaps(remote, 1) {
return nil
}
return util.Errorf("incorrect number of incoming (%d) or outgoing (%d) connections", incoming, outgoing)
})
}
func gossipForTest(t *testing.T) (*gossip.Gossip, *stop.Stopper) {
stopper := stop.NewStopper()
// Setup fake zone config handler.
config.TestingSetupZoneConfigHook(stopper)
rpcContext := rpc.NewContext(nil, nil, stopper)
g := gossip.New(rpcContext, nil, stopper)
// Have to call g.SetNodeID before call g.AddInfo
g.SetNodeID(roachpb.NodeID(1))
// Put an empty system config into gossip.
if err := g.AddInfoProto(gossip.KeySystemConfig,
&config.SystemConfig{}, 0); err != nil {
t.Fatal(err)
}
// Wait for SystemConfig.
util.SucceedsSoon(t, func() error {
if _, ok := g.GetSystemConfig(); !ok {
return util.Errorf("expected system config to be set")
}
return nil
})
return g, stopper
}
// TestTxnCoordSenderErrorWithIntent validates that if a transactional request
// returns an error but also indicates a Writing transaction, the coordinator
// tracks it just like a successful request.
func TestTxnCoordSenderErrorWithIntent(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(20)
ts := NewTxnCoordSender(senderFn(func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
txn := ba.Txn.Clone()
txn.Writing = true
pErr := roachpb.NewError(roachpb.NewTransactionRetryError())
pErr.SetTxn(txn)
return nil, pErr
}), clock, false, nil, stopper)
defer stopper.Stop()
var ba roachpb.BatchRequest
key := roachpb.Key("test")
ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.EndTransactionRequest{})
ba.Txn = &roachpb.Transaction{Name: "test"}
if _, pErr := ts.Send(context.Background(), ba); !testutils.IsPError(pErr, "retry txn") {
t.Fatalf("unexpected error: %v", pErr)
}
defer teardownHeartbeats(ts)
ts.Lock()
defer ts.Unlock()
if len(ts.txns) != 1 {
t.Fatalf("expected transaction to be tracked")
}
}
// TestClientRetryBootstrap verifies that an initial failure to connect
// to a bootstrap host doesn't stall the bootstrapping process in the
// absence of any additional activity. This can happen during acceptance
// tests if the DNS can't lookup hostnames when gossip is started.
func TestClientRetryBootstrap(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t)
remote := startGossip(2, stopper, t)
remote.mu.Lock()
rAddr := remote.is.NodeAddr
remote.mu.Unlock()
if err := local.AddInfo("local-key", []byte("hello"), 0*time.Second); err != nil {
t.Fatal(err)
}
local.SetBootstrapInterval(10 * time.Millisecond)
local.SetResolvers([]resolver.Resolver{
&testResolver{addr: rAddr.String(), numFails: 3, numSuccesses: 1},
})
local.bootstrap()
local.manage()
util.SucceedsSoon(t, func() error {
if _, err := remote.GetInfo("local-key"); err != nil {
return err
}
return nil
})
}
// createTestNode creates an rpc server using the specified address,
// gossip instance, KV database and a node using the specified slice
// of engines. The server, clock and node are returned. If gossipBS is
// not nil, the gossip bootstrap address is set to gossipBS.
func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) (
*rpc.Server, *hlc.Clock, *Node, *stop.Stopper) {
ctx := storage.StoreContext{}
stopper := stop.NewStopper()
ctx.Clock = hlc.NewClock(hlc.UnixNano)
nodeRPCContext := rpc.NewContext(nodeTestBaseContext, ctx.Clock, stopper)
ctx.ScanInterval = 10 * time.Hour
rpcServer := rpc.NewServer(addr, nodeRPCContext)
if err := rpcServer.Start(); err != nil {
t.Fatal(err)
}
g := gossip.New(nodeRPCContext, testContext.GossipInterval, testContext.GossipBootstrapResolvers)
if gossipBS != nil {
// Handle possibility of a :0 port specification.
if gossipBS == addr {
gossipBS = rpcServer.Addr()
}
g.SetResolvers([]resolver.Resolver{resolver.NewResolverFromAddress(gossipBS)})
g.Start(rpcServer, stopper)
}
ctx.Gossip = g
sender := kv.NewDistSender(&kv.DistSenderContext{Clock: ctx.Clock}, g)
ctx.DB = client.NewDB(sender)
// TODO(bdarnell): arrange to have the transport closed.
// (or attach LocalRPCTransport.Close to the stopper)
ctx.Transport = multiraft.NewLocalRPCTransport(stopper)
ctx.EventFeed = util.NewFeed(stopper)
node := NewNode(ctx)
return rpcServer, ctx.Clock, node, stopper
}
// Helper method creates an infostore with 10 infos.
func createTestInfoStore(t *testing.T) *infoStore {
stopper := stop.NewStopper()
defer stopper.Stop()
is := newInfoStore(context.TODO(), 1, emptyAddr, stopper)
for i := 0; i < 10; i++ {
infoA := is.newInfo(nil, time.Second)
infoA.NodeID = 1
infoA.Hops = 1
if err := is.addInfo(fmt.Sprintf("a.%d", i), infoA); err != nil {
t.Fatal(err)
}
infoB := is.newInfo(nil, time.Second)
infoB.NodeID = 2
infoB.Hops = 2
if err := is.addInfo(fmt.Sprintf("b.%d", i), infoB); err != nil {
t.Fatal(err)
}
infoC := is.newInfo(nil, time.Second)
infoC.NodeID = 3
infoC.Hops = 3
if err := is.addInfo(fmt.Sprintf("c.%d", i), infoC); err != nil {
t.Fatal(err)
}
}
return is
}
// TestClientForwardUnresolved verifies that a client does not resolve a forward
// address prematurely.
func TestClientForwardUnresolved(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
const nodeID = 1
local := startGossip(nodeID, stopper, t)
local.mu.Lock()
addr := local.is.NodeAddr
local.mu.Unlock()
client := newClient(&addr) // never started
newAddr := util.UnresolvedAddr{
NetworkField: "tcp",
AddressField: "localhost:2345",
}
reply := &Response{
NodeID: nodeID,
Addr: addr,
AlternateNodeID: nodeID + 1,
AlternateAddr: &newAddr,
}
if err := client.handleResponse(local, reply); !testutils.IsError(err, "received forward") {
t.Fatal(err)
}
if !proto.Equal(client.forwardAddr, &newAddr) {
t.Fatalf("unexpected forward address %v, expected %v", client.forwardAddr, &newAddr)
}
}
// TestRegisterCallback verifies that a callback is invoked when
// registered if there are items which match its regexp in the
// infostore.
func TestRegisterCallback(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
is := newInfoStore(context.TODO(), 1, emptyAddr, stopper)
wg := &sync.WaitGroup{}
cb := callbackRecord{wg: wg}
i1 := is.newInfo(nil, time.Second)
i2 := is.newInfo(nil, time.Second)
if err := is.addInfo("key1", i1); err != nil {
t.Fatal(err)
}
if err := is.addInfo("key2", i2); err != nil {
t.Fatal(err)
}
wg.Add(2)
is.registerCallback("key.*", cb.Add)
wg.Wait()
actKeys := cb.Keys()
sort.Strings(actKeys)
if expKeys := []string{"key1", "key2"}; !reflect.DeepEqual(actKeys, expKeys) {
t.Errorf("expected %v, got %v", expKeys, cb.Keys())
}
}
// startFakeServerGossips creates local gossip instances and remote
// faked gossip instance. The remote gossip instance launches its
// faked gossip service just for check the client message.
func startFakeServerGossips(t *testing.T) (local *Gossip, remote *fakeGossipServer, stopper *stop.Stopper) {
stopper = stop.NewStopper()
lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, nil, stopper)
lserver := rpc.NewServer(lRPCContext)
lln, err := util.ListenAndServeGRPC(stopper, lserver, util.TestAddr)
if err != nil {
t.Fatal(err)
}
local = New(lRPCContext, nil, stopper)
local.start(lserver, lln.Addr())
rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, nil, stopper)
rserver := rpc.NewServer(rRPCContext)
rln, err := util.ListenAndServeGRPC(stopper, rserver, util.TestAddr)
if err != nil {
t.Fatal(err)
}
remote = newFakeGossipServer(rserver, stopper)
addr := rln.Addr()
remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String())
return
}