// StartWithStopper is the same as Start, but allows passing a stopper // explicitly. func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error { if ts.Ctx == nil { ts.Ctx = NewTestContext() } if stopper == nil { stopper = stop.NewStopper() } // Change the replication requirements so we don't get log spam // about ranges not being replicated enough. // TODO(marc): set this in the zones table when we have an entry // for the default cluster-wide zone config and remove these // shenanigans about mutating the global default. oldDefaultZC := proto.Clone(config.DefaultZoneConfig).(*config.ZoneConfig) config.DefaultZoneConfig.ReplicaAttrs = []roachpb.Attributes{{}} stopper.AddCloser(stop.CloserFn(func() { config.DefaultZoneConfig = oldDefaultZC })) var err error ts.Server, err = NewServer(ts.Ctx, stopper) if err != nil { return err } // Ensure we have the correct number of engines. Add in in-memory ones where // needed. There must be at least one store/engine. if ts.StoresPerNode < 1 { ts.StoresPerNode = 1 } for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ { ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, ts.Server.stopper)) } if !ts.SkipBootstrap { stopper := stop.NewStopper() _, err := BootstrapCluster("cluster-1", ts.Ctx.Engines, stopper) if err != nil { return util.Errorf("could not bootstrap cluster: %s", err) } stopper.Stop() } if err := ts.Server.Start(true); err != nil { return err } // If enabled, wait for initial splits to complete before returning control. // If initial splits do not complete, the server is stopped before // returning. if config.TestingTableSplitsDisabled() { return nil } if err := ts.WaitForInitialSplits(); err != nil { ts.Stop() return err } return nil }
// TestCorruptedClusterID verifies that a node fails to start when a // store's cluster ID is empty. func TestCorruptedClusterID(t *testing.T) { defer leaktest.AfterTest(t) engineStopper := stop.NewStopper() e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper) defer engineStopper.Stop() eagerStopper := stop.NewStopper() _, err := BootstrapCluster("cluster-1", []engine.Engine{e}, eagerStopper) if err != nil { t.Fatal(err) } eagerStopper.Stop() // Set the cluster ID to an empty string. sIdent := roachpb.StoreIdent{ ClusterID: "", NodeID: 1, StoreID: 1, } if err = engine.MVCCPutProto(e, nil, keys.StoreIdentKey(), roachpb.ZeroTimestamp, nil, &sIdent); err != nil { t.Fatal(err) } engines := []engine.Engine{e} server, serverAddr, _, node, stopper := createTestNode(util.CreateTestAddr("tcp"), engines, nil, t) if err := node.start(server, serverAddr, engines, roachpb.Attributes{}, stopper); err == nil { t.Errorf("unexpected success") } stopper.Stop() }
func TestGossipOrphanedStallDetection(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t, metric.NewRegistry()) local.SetStallInterval(5 * time.Millisecond) // Make sure we have the sentinel to ensure that its absence is not the // cause of stall detection. if err := local.AddInfo(KeySentinel, nil, time.Hour); err != nil { t.Fatal(err) } peerStopper := stop.NewStopper() peer := startGossip(2, peerStopper, t, metric.NewRegistry()) peerNodeID := peer.GetNodeID() peerAddr := peer.GetNodeAddr() local.startClient(peerAddr, peerNodeID) util.SucceedsSoon(t, func() error { for _, peerID := range local.Outgoing() { if peerID == peerNodeID { return nil } } return errors.Errorf("%d not yet connected", peerNodeID) }) local.bootstrap() local.manage() peerStopper.Stop() util.SucceedsSoon(t, func() error { for _, peerID := range local.Outgoing() { if peerID == peerNodeID { return errors.Errorf("%d still connected", peerNodeID) } } return nil }) peerStopper = stop.NewStopper() defer peerStopper.Stop() peer = startGossipAtAddr(peerNodeID, peerAddr, peerStopper, t, metric.NewRegistry()) util.SucceedsSoon(t, func() error { for _, peerID := range local.Outgoing() { if peerID == peerNodeID { return nil } } return errors.Errorf("%d not yet connected", peerNodeID) }) }
func (m *multiTestContext) Start(t *testing.T, numStores int) { m.t = t if m.manualClock == nil { m.manualClock = hlc.NewManualClock(0) } if m.clock == nil { m.clock = hlc.NewClock(m.manualClock.UnixNano) } if m.gossip == nil { rpcContext := rpc.NewContext(&base.Context{}, m.clock, nil) m.gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) } if m.clientStopper == nil { m.clientStopper = stop.NewStopper() } if m.transport == nil { m.transport = multiraft.NewLocalRPCTransport(m.clientStopper) } if m.storePool == nil { if m.timeUntilStoreDead == 0 { m.timeUntilStoreDead = storage.TestTimeUntilStoreDeadOff } m.storePool = storage.NewStorePool(m.gossip, m.timeUntilStoreDead, m.clientStopper) } // Always create the first sender. m.senders = append(m.senders, kv.NewLocalSender()) rpcSend := func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) { call := proto.Call{ Args: getArgs(nil /* net.Addr */).(proto.Request), Reply: getReply().(proto.Response), } m.senders[0].Send(context.Background(), call) return []gogoproto.Message{call.Reply}, call.Reply.Header().GoError() } if m.db == nil { distSender := kv.NewDistSender(&kv.DistSenderContext{ Clock: m.clock, RangeDescriptorDB: m.senders[0], RPCSend: rpcSend, }, m.gossip) sender := kv.NewTxnCoordSender(distSender, m.clock, false, nil, m.clientStopper) m.db = client.NewDB(sender) } for i := 0; i < numStores; i++ { m.addStore() } if m.transportStopper == nil { m.transportStopper = stop.NewStopper() } m.transportStopper.AddCloser(m.transport) }
// TestRetryableError verifies that Send returns a retryable error // when it hits an RPC error. func TestRetryableError(t *testing.T) { defer leaktest.AfterTest(t)() clientStopper := stop.NewStopper() defer clientStopper.Stop() clientContext := newNodeTestContext(nil, clientStopper) serverStopper := stop.NewStopper() serverContext := newNodeTestContext(nil, serverStopper) s, ln := newTestServer(t, serverContext) roachpb.RegisterInternalServer(s, Node(0)) conn, err := clientContext.GRPCDial(ln.Addr().String()) if err != nil { t.Fatal(err) } ctx := context.Background() waitForConnState := func(desiredState grpc.ConnectivityState) { clientState, err := conn.State() for clientState != desiredState { if err != nil { t.Fatal(err) } if clientState == grpc.Shutdown { t.Fatalf("%v has unexpectedly shut down", conn) } clientState, err = conn.WaitForStateChange(ctx, clientState) } } // Wait until the client becomes healthy and shut down the server. waitForConnState(grpc.Ready) serverStopper.Stop() // Wait until the client becomes unhealthy. waitForConnState(grpc.TransientFailure) sp := tracing.NewTracer().StartSpan("node test") defer sp.Finish() opts := SendOptions{ Ordering: orderStable, SendNextTimeout: 100 * time.Millisecond, Timeout: 100 * time.Millisecond, Trace: sp, } if _, err := sendBatch(opts, []net.Addr{ln.Addr()}, clientContext); err != nil { retryErr, ok := err.(retry.Retryable) if !ok { t.Fatalf("Unexpected error type: %v", err) } if !retryErr.CanRetry() { t.Errorf("Expected retryable error: %v", retryErr) } } else { t.Fatalf("Unexpected success") } }
// StartTestCluster starts up a TestCluster made up of `nodes` in-memory testing // servers. // The cluster should be stopped using cluster.Stop(). func StartTestCluster(t testing.TB, nodes int, args base.TestClusterArgs) *TestCluster { if nodes < 1 { t.Fatal("invalid cluster size: ", nodes) } if args.ServerArgs.JoinAddr != "" { t.Fatal("can't specify a join addr when starting a cluster") } if args.ServerArgs.Stopper != nil { t.Fatal("can't set individual server stoppers when starting a cluster") } storeKnobs := args.ServerArgs.Knobs.Store if storeKnobs != nil && (storeKnobs.(*storage.StoreTestingKnobs).DisableSplitQueue || storeKnobs.(*storage.StoreTestingKnobs).DisableReplicateQueue) { t.Fatal("can't disable an individual server's queues when starting a cluster; " + "the cluster controls replication") } switch args.ReplicationMode { case base.ReplicationAuto: case base.ReplicationManual: if args.ServerArgs.Knobs.Store == nil { args.ServerArgs.Knobs.Store = &storage.StoreTestingKnobs{} } storeKnobs := args.ServerArgs.Knobs.Store.(*storage.StoreTestingKnobs) storeKnobs.DisableSplitQueue = true storeKnobs.DisableReplicateQueue = true default: t.Fatal("unexpected replication mode") } tc := &TestCluster{} tc.stopper = stop.NewStopper() args.ServerArgs.PartOfCluster = true for i := 0; i < nodes; i++ { serverArgs := args.ServerArgs serverArgs.Stopper = stop.NewStopper() if i > 0 { serverArgs.JoinAddr = tc.Servers[0].ServingAddr() } s, conn, _ := serverutils.StartServer(t, serverArgs) tc.Servers = append(tc.Servers, s.(*server.TestServer)) tc.Conns = append(tc.Conns, conn) tc.mu.Lock() tc.mu.serverStoppers = append(tc.mu.serverStoppers, serverArgs.Stopper) tc.mu.Unlock() } // Create a closer that will stop the individual server stoppers when the // cluster stopper is stopped. tc.stopper.AddCloser(stop.CloserFn(tc.stopServers)) tc.waitForStores(t) return tc }
// TestStoreRecoverWithErrors verifies that even commands that fail are marked as // applied so they are not retried after recovery. func TestStoreRecoverWithErrors(t *testing.T) { defer leaktest.AfterTest(t) defer func() { storage.TestingCommandFilter = nil }() manual := hlc.NewManualClock(0) clock := hlc.NewClock(manual.UnixNano) engineStopper := stop.NewStopper() defer engineStopper.Stop() eng := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper) numIncrements := 0 storage.TestingCommandFilter = func(_ roachpb.StoreID, args roachpb.Request, _ roachpb.Header) error { if _, ok := args.(*roachpb.IncrementRequest); ok && args.Header().Key.Equal(roachpb.Key("a")) { numIncrements++ } return nil } func() { stopper := stop.NewStopper() defer stopper.Stop() store := createTestStoreWithEngine(t, eng, clock, true, nil, stopper) // Write a bytes value so the increment will fail. putArgs := putArgs(roachpb.Key("a"), []byte("asdf")) if _, err := client.SendWrapped(rg1(store), nil, &putArgs); err != nil { t.Fatal(err) } // Try and fail to increment the key. It is important for this test that the // failure be the last thing in the raft log when the store is stopped. incArgs := incrementArgs(roachpb.Key("a"), 42) if _, err := client.SendWrapped(rg1(store), nil, &incArgs); err == nil { t.Fatal("did not get expected error") } }() if numIncrements != 1 { t.Fatalf("expected 1 increments; was %d", numIncrements) } // Recover from the engine. store := createTestStoreWithEngine(t, eng, clock, false, nil, engineStopper) // Issue a no-op write to lazily initialize raft on the range. incArgs := incrementArgs(roachpb.Key("b"), 0) if _, err := client.SendWrapped(rg1(store), nil, &incArgs); err != nil { t.Fatal(err) } // No additional increments were performed on key A during recovery. if numIncrements != 1 { t.Fatalf("expected 1 increments; was %d", numIncrements) } }
// StartWithStopper is the same as Start, but allows passing a stopper // explicitly. func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error { if ts.Ctx == nil { ts.Ctx = NewTestContext() } // TODO(marc): set this in the zones table when we have an entry // for the default cluster-wide zone config. config.DefaultZoneConfig.ReplicaAttrs = []roachpb.Attributes{{}} if stopper == nil { stopper = stop.NewStopper() } var err error ts.Server, err = NewServer(ts.Ctx, stopper) if err != nil { return err } // Ensure we have the correct number of engines. Add in in-memory ones where // needed. There must be at least one store/engine. if ts.StoresPerNode < 1 { ts.StoresPerNode = 1 } for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ { ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, ts.Server.stopper)) } if !ts.SkipBootstrap { stopper := stop.NewStopper() _, err := BootstrapCluster("cluster-1", ts.Ctx.Engines, stopper) if err != nil { return util.Errorf("could not bootstrap cluster: %s", err) } stopper.Stop() } if err := ts.Server.Start(true); err != nil { return err } // If enabled, wait for initial splits to complete before returning control. // If initial splits do not complete, the server is stopped before // returning. if config.TestingTableSplitsDisabled() { return nil } if err := ts.WaitForInitialSplits(); err != nil { ts.Stop() return err } return nil }
// TestNodeJoin verifies a new node is able to join a bootstrapped // cluster consisting of one node. func TestNodeJoin(t *testing.T) { defer leaktest.AfterTest(t) engineStopper := stop.NewStopper() defer engineStopper.Stop() e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper) stopper := stop.NewStopper() _, err := BootstrapCluster("cluster-1", []engine.Engine{e}, stopper) if err != nil { t.Fatal(err) } stopper.Stop() // Set an aggressive gossip interval to make sure information is exchanged tout de suite. testContext.GossipInterval = gossip.TestInterval // Start the bootstrap node. engines1 := []engine.Engine{e} addr1 := util.CreateTestAddr("tcp") server1, node1, stopper1 := createAndStartTestNode(addr1, engines1, addr1, t) defer stopper1.Stop() // Create a new node. engines2 := []engine.Engine{engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)} server2, node2, stopper2 := createAndStartTestNode(util.CreateTestAddr("tcp"), engines2, server1.Addr(), t) defer stopper2.Stop() // Verify new node is able to bootstrap its store. if err := util.IsTrueWithin(func() bool { return node2.lSender.GetStoreCount() == 1 }, 50*time.Millisecond); err != nil { t.Fatal(err) } // Verify node1 sees node2 via gossip and vice versa. node1Key := gossip.MakeNodeIDKey(node1.Descriptor.NodeID) node2Key := gossip.MakeNodeIDKey(node2.Descriptor.NodeID) if err := util.IsTrueWithin(func() bool { nodeDesc1 := &roachpb.NodeDescriptor{} if err := node1.ctx.Gossip.GetInfoProto(node2Key, nodeDesc1); err != nil { return false } if addr2 := nodeDesc1.Address.AddressField; addr2 != server2.Addr().String() { t.Errorf("addr2 gossip %s doesn't match addr2 address %s", addr2, server2.Addr().String()) } nodeDesc2 := &roachpb.NodeDescriptor{} if err := node2.ctx.Gossip.GetInfoProto(node1Key, nodeDesc2); err != nil { return false } if addr1 := nodeDesc2.Address.AddressField; addr1 != server1.Addr().String() { t.Errorf("addr1 gossip %s doesn't match addr1 address %s", addr1, server1.Addr().String()) } return true }, 50*time.Millisecond); err != nil { t.Error(err) } }
// TestRetryableError verifies that Send returns a retryable error // when it hits an RPC error. func TestRetryableError(t *testing.T) { defer leaktest.AfterTest(t)() clientStopper := stop.NewStopper() defer clientStopper.Stop() clientContext := newNodeTestContext(nil, clientStopper) clientContext.HeartbeatTimeout = 10 * clientContext.HeartbeatInterval serverStopper := stop.NewStopper() serverContext := newNodeTestContext(nil, serverStopper) s, ln := newTestServer(t, serverContext) registerBatch(t, s, 0) c := rpc.NewClient(ln.Addr(), clientContext) // Wait until the client becomes healthy and shut down the server. <-c.Healthy() serverStopper.Stop() // Wait until the client becomes unhealthy. func() { for r := retry.Start(retry.Options{}); r.Next(); { select { case <-c.Healthy(): case <-time.After(1 * time.Nanosecond): return } } }() sp := tracing.NewTracer().StartSpan("node test") defer sp.Finish() opts := SendOptions{ Ordering: orderStable, SendNextTimeout: 100 * time.Millisecond, Timeout: 100 * time.Millisecond, Trace: sp, } if _, err := sendBatch(opts, []net.Addr{ln.Addr()}, clientContext); err != nil { retryErr, ok := err.(retry.Retryable) if !ok { t.Fatalf("Unexpected error type: %v", err) } if !retryErr.CanRetry() { t.Errorf("Expected retryable error: %v", retryErr) } } else { t.Fatalf("Unexpected success") } }
func TestBatchDefer(t *testing.T) { defer leaktest.AfterTest(t) stopper := stop.NewStopper() defer stopper.Stop() e := NewInMem(roachpb.Attributes{}, 1<<20, stopper) b := e.NewBatch() defer b.Close() list := []string{} b.Defer(func() { list = append(list, "one") }) b.Defer(func() { list = append(list, "two") }) if err := b.Commit(); err != nil { t.Fatal(err) } // Order was reversed when the defers were run. if !reflect.DeepEqual(list, []string{"two", "one"}) { t.Errorf("expected [two, one]; got %v", list) } }
// TestBatchScanMaxWithDeleted verifies that if a deletion // in the updates map shadows an entry from the engine, the // max on a scan is still reached. func TestBatchScanMaxWithDeleted(t *testing.T) { defer leaktest.AfterTest(t) stopper := stop.NewStopper() defer stopper.Stop() e := NewInMem(roachpb.Attributes{}, 1<<20, stopper) b := e.NewBatch() defer b.Close() // Write two values. if err := e.Put(MVCCKey("a"), []byte("value1")); err != nil { t.Fatal(err) } if err := e.Put(MVCCKey("b"), []byte("value2")); err != nil { t.Fatal(err) } // Now, delete "a" in batch. if err := b.Clear(MVCCKey("a")); err != nil { t.Fatal(err) } // A scan with max=1 should scan "b". kvs, err := Scan(b, MVCCKey(roachpb.RKeyMin), MVCCKey(roachpb.RKeyMax), 1) if err != nil { t.Fatal(err) } if len(kvs) != 1 || !bytes.Equal(kvs[0].Key, []byte("b")) { t.Errorf("expected scan of \"b\"; got %v", kvs) } }
// TestBatchConcurrency verifies operation of batch when the // underlying engine has concurrent modifications to overlapping // keys. This should never happen with the way Cockroach uses // batches, but worth verifying. func TestBatchConcurrency(t *testing.T) { defer leaktest.AfterTest(t) stopper := stop.NewStopper() defer stopper.Stop() e := NewInMem(roachpb.Attributes{}, 1<<20, stopper) b := e.NewBatch() defer b.Close() // Write a merge to the batch. if err := b.Merge(MVCCKey("a"), appender("bar")); err != nil { t.Fatal(err) } val, err := b.Get(MVCCKey("a")) if err != nil { t.Fatal(err) } if !compareMergedValues(t, val, appender("bar")) { t.Error("mismatch of \"a\"") } // Write an engine value. if err := e.Put(MVCCKey("a"), appender("foo")); err != nil { t.Fatal(err) } // Now, read again and verify that the merge happens on top of the mod. val, err = b.Get(MVCCKey("a")) if err != nil { t.Fatal(err) } if !bytes.Equal(val, appender("foobar")) { t.Error("mismatch of \"a\"") } }
func TestStopper(t *testing.T) { defer leaktest.AfterTest(t) s := stop.NewStopper() running := make(chan struct{}) waiting := make(chan struct{}) cleanup := make(chan struct{}) s.RunWorker(func() { <-running }) go func() { <-s.ShouldStop() select { case <-waiting: t.Fatal("expected stopper to have blocked") case <-time.After(1 * time.Millisecond): // Expected. } close(running) select { case <-waiting: // Success. case <-time.After(100 * time.Millisecond): t.Fatal("stopper should have finished waiting") } close(cleanup) }() s.Stop() close(waiting) <-cleanup }
func TestStopperIsStopped(t *testing.T) { defer leaktest.AfterTest(t) s := stop.NewStopper() bc := newBlockingCloser() s.AddCloser(bc) go s.Stop() select { case <-s.ShouldStop(): case <-time.After(100 * time.Millisecond): t.Fatal("stopper should have finished waiting") } select { case <-s.IsStopped(): t.Fatal("expected blocked closer to prevent stop") case <-time.After(1 * time.Millisecond): // Expected. } bc.Unblock() select { case <-s.IsStopped(): // Expected case <-time.After(100 * time.Millisecond): t.Fatal("stopper should have finished stopping") } }
func benchmarkIterOnBatch(b *testing.B, writes int) { stopper := stop.NewStopper() defer stopper.Stop() engine := createTestEngine(stopper) for i := 0; i < writes; i++ { if err := engine.Put(makeKey(i), []byte(strconv.Itoa(i))); err != nil { b.Fatal(err) } } batch := engine.NewBatch() defer batch.Close() for i := 0; i < writes; i++ { if err := batch.Clear(makeKey(i)); err != nil { b.Fatal(err) } } r := rand.New(rand.NewSource(5)) b.ResetTimer() for i := 0; i < b.N; i++ { key := makeKey(r.Intn(writes)) iter := batch.NewIterator(true) iter.Seek(key) iter.Close() } }
// openRocksDBWithVersion attempts to open a rocks db instance, optionally with // the supplied Version struct. func openRocksDBWithVersion(t *testing.T, hasVersionFile bool, ver Version) error { stopper := stop.NewStopper() defer stopper.Stop() dir, err := ioutil.TempDir("", "testing") if err != nil { t.Fatal(err) } defer func() { if err := os.RemoveAll(dir); err != nil { t.Fatal(err) } }() if hasVersionFile { b, err := json.Marshal(ver) if err != nil { t.Fatal(err) } if err := ioutil.WriteFile(getVersionFilename(dir), b, 0644); err != nil { t.Fatal(err) } } rocksdb := NewRocksDB(roachpb.Attributes{}, dir, 0, minMemtableBudget, 0, stopper) return rocksdb.Open() }
// TestHeartbeatResponseFanout check 2 raft groups on the same node distribution, // but each group has different Term, heartbeat response from each group should // not disturb other group's Term or Leadership func TestHeartbeatResponseFanout(t *testing.T) { defer leaktest.AfterTest(t) stopper := stop.NewStopper() defer stopper.Stop() cluster := newTestCluster(nil, 3, stopper, t) groupID1 := roachpb.RangeID(1) cluster.createGroup(groupID1, 0, 3 /* replicas */) groupID2 := roachpb.RangeID(2) cluster.createGroup(groupID2, 0, 3 /* replicas */) leaderIndex := 0 cluster.elect(leaderIndex, groupID1) // GroupID2 will have 3 round of election, so it will have different // term with groupID1, but both leader on the same node. for i := 2; i >= 0; i-- { leaderIndex = i cluster.elect(leaderIndex, groupID2) } // Send a coalesced heartbeat. // Heartbeat response from groupID2 will have a big term than which from groupID1. cluster.nodes[0].coalescedHeartbeat() // Start submit a command to see if groupID1's leader changed? cluster.nodes[0].SubmitCommand(groupID1, makeCommandID(), []byte("command")) select { case _ = <-cluster.events[0].CommandCommitted: log.Infof("SubmitCommand succeed after Heartbeat Response fanout") case <-time.After(500 * time.Millisecond): t.Fatalf("No leader after Heartbeat Response fanout") } }
// TestTxnCoordSenderSingleRoundtripTxn checks that a batch which completely // holds the writing portion of a Txn (including EndTransaction) does not // launch a heartbeat goroutine at all. func TestTxnCoordSenderSingleRoundtripTxn(t *testing.T) { defer leaktest.AfterTest(t) stopper := stop.NewStopper() manual := hlc.NewManualClock(0) clock := hlc.NewClock(manual.UnixNano) clock.SetMaxOffset(20) ts := NewTxnCoordSender(senderFn(func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { br := ba.CreateReply() br.Txn = ba.Txn.Clone() br.Txn.Writing = true return br, nil }), clock, false, nil, stopper) // Stop the stopper manually, prior to trying the transaction. This has the // effect of returning a NodeUnavailableError for any attempts at launching // a heartbeat goroutine. stopper.Stop() var ba roachpb.BatchRequest key := roachpb.Key("test") ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}}) ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}}) ba.Add(&roachpb.EndTransactionRequest{}) ba.Txn = &roachpb.Transaction{Name: "test"} _, pErr := ts.Send(context.Background(), ba) if pErr != nil { t.Fatal(pErr) } }
// TestInfoStoreMostDistant verifies selection of most distant node & // associated hops. func TestInfoStoreMostDistant(t *testing.T) { defer leaktest.AfterTest(t)() nodes := []roachpb.NodeID{ roachpb.NodeID(1), roachpb.NodeID(2), roachpb.NodeID(3), } stopper := stop.NewStopper() defer stopper.Stop() is := newInfoStore(context.TODO(), 1, emptyAddr, stopper) // Add info from each address, with hop count equal to index+1. for i := 0; i < len(nodes); i++ { inf := is.newInfo(nil, time.Second) inf.Hops = uint32(i + 1) inf.NodeID = nodes[i] if err := is.addInfo(fmt.Sprintf("b.%d", i), inf); err != nil { t.Fatal(err) } nodeID, hops := is.mostDistant() if nodeID != inf.NodeID { t.Errorf("%d: expected node %d; got %d", i, inf.NodeID, nodeID) } if hops != inf.Hops { t.Errorf("%d: expected node %d; got %d", i, inf.Hops, hops) } } }
// Verify that adding two infos with different hops but same keys // always chooses the minimum hops. func TestAddInfoSameKeyDifferentHops(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() is := newInfoStore(context.TODO(), 1, emptyAddr, stopper) info1 := is.newInfo(nil, time.Second) info1.Hops = 1 info2 := is.newInfo(nil, time.Second) info2.Value.Timestamp.WallTime = info1.Value.Timestamp.WallTime info2.Hops = 2 if err := is.addInfo("a", info1); err != nil { t.Errorf("failed insert: %s", err) } if err := is.addInfo("a", info2); err == nil { t.Errorf("shouldn't have inserted info 2: %s", err) } i := is.getInfo("a") if i.Hops != info1.Hops || !proto.Equal(i, info1) { t.Error("failed to properly combine hops and value", i) } // Try yet another info, with lower hops yet (0). info3 := is.newInfo(nil, time.Second) if err := is.addInfo("a", info3); err != nil { t.Error(err) } i = is.getInfo("a") if i.Hops != info3.Hops || !proto.Equal(i, info3) { t.Error("failed to properly combine hops and value", i) } }
// TestClientDisallowMultipleConns verifies that the server disallows // multiple connections from the same client node ID. func TestClientDisallowMultipleConns(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t) remote := startGossip(2, stopper, t) local.mu.Lock() remote.mu.Lock() rAddr := remote.is.NodeAddr // Start two clients from local to remote. RPC client cache is // disabled via the context, so we'll start two different outgoing // connections. local.startClient(&rAddr, stopper) local.startClient(&rAddr, stopper) local.mu.Unlock() remote.mu.Unlock() local.manage() remote.manage() util.SucceedsSoon(t, func() error { // Verify that the remote server has only a single incoming // connection and the local server has only a single outgoing // connection. local.mu.Lock() remote.mu.Lock() outgoing := local.outgoing.len() incoming := remote.incoming.len() local.mu.Unlock() remote.mu.Unlock() if outgoing == 1 && incoming == 1 && verifyServerMaps(local, 0) && verifyServerMaps(remote, 1) { return nil } return util.Errorf("incorrect number of incoming (%d) or outgoing (%d) connections", incoming, outgoing) }) }
func gossipForTest(t *testing.T) (*gossip.Gossip, *stop.Stopper) { stopper := stop.NewStopper() // Setup fake zone config handler. config.TestingSetupZoneConfigHook(stopper) rpcContext := rpc.NewContext(nil, nil, stopper) g := gossip.New(rpcContext, nil, stopper) // Have to call g.SetNodeID before call g.AddInfo g.SetNodeID(roachpb.NodeID(1)) // Put an empty system config into gossip. if err := g.AddInfoProto(gossip.KeySystemConfig, &config.SystemConfig{}, 0); err != nil { t.Fatal(err) } // Wait for SystemConfig. util.SucceedsSoon(t, func() error { if _, ok := g.GetSystemConfig(); !ok { return util.Errorf("expected system config to be set") } return nil }) return g, stopper }
// TestTxnCoordSenderErrorWithIntent validates that if a transactional request // returns an error but also indicates a Writing transaction, the coordinator // tracks it just like a successful request. func TestTxnCoordSenderErrorWithIntent(t *testing.T) { defer leaktest.AfterTest(t) stopper := stop.NewStopper() manual := hlc.NewManualClock(0) clock := hlc.NewClock(manual.UnixNano) clock.SetMaxOffset(20) ts := NewTxnCoordSender(senderFn(func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { txn := ba.Txn.Clone() txn.Writing = true pErr := roachpb.NewError(roachpb.NewTransactionRetryError()) pErr.SetTxn(txn) return nil, pErr }), clock, false, nil, stopper) defer stopper.Stop() var ba roachpb.BatchRequest key := roachpb.Key("test") ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}}) ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}}) ba.Add(&roachpb.EndTransactionRequest{}) ba.Txn = &roachpb.Transaction{Name: "test"} if _, pErr := ts.Send(context.Background(), ba); !testutils.IsPError(pErr, "retry txn") { t.Fatalf("unexpected error: %v", pErr) } defer teardownHeartbeats(ts) ts.Lock() defer ts.Unlock() if len(ts.txns) != 1 { t.Fatalf("expected transaction to be tracked") } }
// TestClientRetryBootstrap verifies that an initial failure to connect // to a bootstrap host doesn't stall the bootstrapping process in the // absence of any additional activity. This can happen during acceptance // tests if the DNS can't lookup hostnames when gossip is started. func TestClientRetryBootstrap(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t) remote := startGossip(2, stopper, t) remote.mu.Lock() rAddr := remote.is.NodeAddr remote.mu.Unlock() if err := local.AddInfo("local-key", []byte("hello"), 0*time.Second); err != nil { t.Fatal(err) } local.SetBootstrapInterval(10 * time.Millisecond) local.SetResolvers([]resolver.Resolver{ &testResolver{addr: rAddr.String(), numFails: 3, numSuccesses: 1}, }) local.bootstrap() local.manage() util.SucceedsSoon(t, func() error { if _, err := remote.GetInfo("local-key"); err != nil { return err } return nil }) }
// createTestNode creates an rpc server using the specified address, // gossip instance, KV database and a node using the specified slice // of engines. The server, clock and node are returned. If gossipBS is // not nil, the gossip bootstrap address is set to gossipBS. func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) ( *rpc.Server, *hlc.Clock, *Node, *stop.Stopper) { ctx := storage.StoreContext{} stopper := stop.NewStopper() ctx.Clock = hlc.NewClock(hlc.UnixNano) nodeRPCContext := rpc.NewContext(nodeTestBaseContext, ctx.Clock, stopper) ctx.ScanInterval = 10 * time.Hour rpcServer := rpc.NewServer(addr, nodeRPCContext) if err := rpcServer.Start(); err != nil { t.Fatal(err) } g := gossip.New(nodeRPCContext, testContext.GossipInterval, testContext.GossipBootstrapResolvers) if gossipBS != nil { // Handle possibility of a :0 port specification. if gossipBS == addr { gossipBS = rpcServer.Addr() } g.SetResolvers([]resolver.Resolver{resolver.NewResolverFromAddress(gossipBS)}) g.Start(rpcServer, stopper) } ctx.Gossip = g sender := kv.NewDistSender(&kv.DistSenderContext{Clock: ctx.Clock}, g) ctx.DB = client.NewDB(sender) // TODO(bdarnell): arrange to have the transport closed. // (or attach LocalRPCTransport.Close to the stopper) ctx.Transport = multiraft.NewLocalRPCTransport(stopper) ctx.EventFeed = util.NewFeed(stopper) node := NewNode(ctx) return rpcServer, ctx.Clock, node, stopper }
// Helper method creates an infostore with 10 infos. func createTestInfoStore(t *testing.T) *infoStore { stopper := stop.NewStopper() defer stopper.Stop() is := newInfoStore(context.TODO(), 1, emptyAddr, stopper) for i := 0; i < 10; i++ { infoA := is.newInfo(nil, time.Second) infoA.NodeID = 1 infoA.Hops = 1 if err := is.addInfo(fmt.Sprintf("a.%d", i), infoA); err != nil { t.Fatal(err) } infoB := is.newInfo(nil, time.Second) infoB.NodeID = 2 infoB.Hops = 2 if err := is.addInfo(fmt.Sprintf("b.%d", i), infoB); err != nil { t.Fatal(err) } infoC := is.newInfo(nil, time.Second) infoC.NodeID = 3 infoC.Hops = 3 if err := is.addInfo(fmt.Sprintf("c.%d", i), infoC); err != nil { t.Fatal(err) } } return is }
// TestClientForwardUnresolved verifies that a client does not resolve a forward // address prematurely. func TestClientForwardUnresolved(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() const nodeID = 1 local := startGossip(nodeID, stopper, t) local.mu.Lock() addr := local.is.NodeAddr local.mu.Unlock() client := newClient(&addr) // never started newAddr := util.UnresolvedAddr{ NetworkField: "tcp", AddressField: "localhost:2345", } reply := &Response{ NodeID: nodeID, Addr: addr, AlternateNodeID: nodeID + 1, AlternateAddr: &newAddr, } if err := client.handleResponse(local, reply); !testutils.IsError(err, "received forward") { t.Fatal(err) } if !proto.Equal(client.forwardAddr, &newAddr) { t.Fatalf("unexpected forward address %v, expected %v", client.forwardAddr, &newAddr) } }
// TestRegisterCallback verifies that a callback is invoked when // registered if there are items which match its regexp in the // infostore. func TestRegisterCallback(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() is := newInfoStore(context.TODO(), 1, emptyAddr, stopper) wg := &sync.WaitGroup{} cb := callbackRecord{wg: wg} i1 := is.newInfo(nil, time.Second) i2 := is.newInfo(nil, time.Second) if err := is.addInfo("key1", i1); err != nil { t.Fatal(err) } if err := is.addInfo("key2", i2); err != nil { t.Fatal(err) } wg.Add(2) is.registerCallback("key.*", cb.Add) wg.Wait() actKeys := cb.Keys() sort.Strings(actKeys) if expKeys := []string{"key1", "key2"}; !reflect.DeepEqual(actKeys, expKeys) { t.Errorf("expected %v, got %v", expKeys, cb.Keys()) } }
// startFakeServerGossips creates local gossip instances and remote // faked gossip instance. The remote gossip instance launches its // faked gossip service just for check the client message. func startFakeServerGossips(t *testing.T) (local *Gossip, remote *fakeGossipServer, stopper *stop.Stopper) { stopper = stop.NewStopper() lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, nil, stopper) lserver := rpc.NewServer(lRPCContext) lln, err := util.ListenAndServeGRPC(stopper, lserver, util.TestAddr) if err != nil { t.Fatal(err) } local = New(lRPCContext, nil, stopper) local.start(lserver, lln.Addr()) rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, nil, stopper) rserver := rpc.NewServer(rRPCContext) rln, err := util.ListenAndServeGRPC(stopper, rserver, util.TestAddr) if err != nil { t.Fatal(err) } remote = newFakeGossipServer(rserver, stopper) addr := rln.Addr() remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String()) return }