// TestClientDisconnectRedundant verifies that the gossip server // will drop an outgoing client connection that is already an // inbound client connection of another node. func TestClientDisconnectRedundant(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t, metric.NewRegistry()) remote := startGossip(2, stopper, t, metric.NewRegistry()) // startClient requires locks are held, so acquire here. local.mu.Lock() remote.mu.Lock() rAddr := remote.mu.is.NodeAddr lAddr := local.mu.is.NodeAddr local.startClient(&rAddr, remote.NodeID.Get()) remote.startClient(&lAddr, local.NodeID.Get()) local.mu.Unlock() remote.mu.Unlock() local.manage() remote.manage() util.SucceedsSoon(t, func() error { // Check which of the clients is connected to the other. ok1 := local.findClient(func(c *client) bool { return c.addr.String() == rAddr.String() }) != nil ok2 := remote.findClient(func(c *client) bool { return c.addr.String() == lAddr.String() }) != nil // We expect node 2 to disconnect; if both are still connected, // it's possible that node 1 gossiped before node 2 connected, in // which case we have to gossip from node 1 to trigger the // disconnect redundant client code. if ok1 && ok2 { if err := local.AddInfo("local-key", nil, time.Second); err != nil { t.Fatal(err) } } else if ok1 && !ok2 && verifyServerMaps(local, 0) && verifyServerMaps(remote, 1) { return nil } return errors.New("local client to remote not yet closed as redundant") }) }
func TestGossipRaceLogStatus(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t, metric.NewRegistry()) local.mu.Lock() peer := startGossip(2, stopper, t, metric.NewRegistry()) local.startClient(&peer.mu.is.NodeAddr) local.mu.Unlock() // Race gossiping against LogStatus. gun := make(chan struct{}) for i := uint8(0); i < 10; i++ { go func() { <-gun local.LogStatus() gun <- struct{}{} }() gun <- struct{}{} if err := local.AddInfo( strconv.FormatUint(uint64(i), 10), []byte{i}, time.Hour, ); err != nil { t.Fatal(err) } <-gun } close(gun) }
// TestClientGossip verifies a client can gossip a delta to the server. func TestClientGossip(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() local := startGossip(1, stopper, t, metric.NewRegistry()) remote := startGossip(2, stopper, t, metric.NewRegistry()) disconnected := make(chan *client, 1) c := newClient(log.AmbientContext{}, remote.GetNodeAddr(), makeMetrics()) defer func() { stopper.Stop() if c != <-disconnected { t.Errorf("expected client disconnect after remote close") } }() if err := local.AddInfo("local-key", nil, time.Hour); err != nil { t.Fatal(err) } if err := remote.AddInfo("remote-key", nil, time.Hour); err != nil { t.Fatal(err) } gossipSucceedsSoon(t, stopper, disconnected, map[*client]*Gossip{ c: local, }, func() error { if _, err := remote.GetInfo("local-key"); err != nil { return err } if _, err := local.GetInfo("remote-key"); err != nil { return err } return nil }) }
// createTestNode creates an rpc server using the specified address, // gossip instance, KV database and a node using the specified slice // of engines. The server, clock and node are returned. If gossipBS is // not nil, the gossip bootstrap address is set to gossipBS. func createTestNode( addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T, ) (*grpc.Server, net.Addr, *hlc.Clock, *Node, *stop.Stopper) { cfg := storage.StoreConfig{} stopper := stop.NewStopper() cfg.Clock = hlc.NewClock(hlc.UnixNano) nodeRPCContext := rpc.NewContext(log.AmbientContext{}, nodeTestBaseContext, cfg.Clock, stopper) cfg.ScanInterval = 10 * time.Hour cfg.ConsistencyCheckInterval = 10 * time.Hour grpcServer := rpc.NewServer(nodeRPCContext) serverCfg := makeTestConfig() cfg.Gossip = gossip.NewTest( 0, nodeRPCContext, grpcServer, serverCfg.GossipBootstrapResolvers, stopper, metric.NewRegistry(), ) ln, err := netutil.ListenAndServeGRPC(stopper, grpcServer, addr) if err != nil { t.Fatal(err) } if gossipBS != nil { // Handle possibility of a :0 port specification. if gossipBS.Network() == addr.Network() && gossipBS.String() == addr.String() { gossipBS = ln.Addr() } r, err := resolver.NewResolverFromAddress(gossipBS) if err != nil { t.Fatalf("bad gossip address %s: %s", gossipBS, err) } cfg.Gossip.SetResolvers([]resolver.Resolver{r}) cfg.Gossip.Start(ln.Addr()) } retryOpts := base.DefaultRetryOptions() retryOpts.Closer = stopper.ShouldQuiesce() distSender := kv.NewDistSender(kv.DistSenderConfig{ Clock: cfg.Clock, RPCContext: nodeRPCContext, RPCRetryOptions: &retryOpts, }, cfg.Gossip) cfg.AmbientCtx.Tracer = tracing.NewTracer() sender := kv.NewTxnCoordSender( cfg.AmbientCtx, distSender, cfg.Clock, false, stopper, kv.MakeTxnMetrics(metric.TestSampleInterval), ) cfg.DB = client.NewDB(sender) cfg.Transport = storage.NewDummyRaftTransport() cfg.MetricsSampleInterval = metric.TestSampleInterval node := NewNode(cfg, status.NewMetricsRecorder(cfg.Clock), metric.NewRegistry(), stopper, kv.MakeTxnMetrics(metric.TestSampleInterval), sql.MakeEventLogger(nil)) roachpb.RegisterInternalServer(grpcServer, node) return grpcServer, ln.Addr(), cfg.Clock, node, stopper }
// TestClientDisallowMultipleConns verifies that the server disallows // multiple connections from the same client node ID. func TestClientDisallowMultipleConns(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t, metric.NewRegistry()) remote := startGossip(2, stopper, t, metric.NewRegistry()) local.mu.Lock() remote.mu.Lock() rAddr := remote.mu.is.NodeAddr // Start two clients from local to remote. RPC client cache is // disabled via the context, so we'll start two different outgoing // connections. local.startClient(&rAddr, remote.NodeID.Get()) local.startClient(&rAddr, remote.NodeID.Get()) local.mu.Unlock() remote.mu.Unlock() local.manage() remote.manage() util.SucceedsSoon(t, func() error { // Verify that the remote server has only a single incoming // connection and the local server has only a single outgoing // connection. local.mu.Lock() remote.mu.Lock() outgoing := local.outgoing.len() incoming := remote.mu.incoming.len() local.mu.Unlock() remote.mu.Unlock() if outgoing == 1 && incoming == 1 && verifyServerMaps(local, 0) && verifyServerMaps(remote, 1) { return nil } return errors.Errorf("incorrect number of incoming (%d) or outgoing (%d) connections", incoming, outgoing) }) }
// TestClientGossipMetrics verifies a that gossip stats are generated. func TestClientGossipMetrics(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t, metric.NewRegistry()) remote := startGossip(2, stopper, t, metric.NewRegistry()) if err := local.AddInfo("local-key", nil, time.Hour); err != nil { t.Fatal(err) } if err := remote.AddInfo("remote-key", nil, time.Hour); err != nil { t.Fatal(err) } gossipSucceedsSoon( t, stopper, make(chan *client, 2), map[*client]*Gossip{ newClient(log.AmbientContext{}, local.GetNodeAddr(), remote.nodeMetrics): remote, }, func() error { // Infos/Bytes Sent/Received should not be zero. for i, s := range []*server{local.server, remote.server} { for _, counter := range []*metric.Counter{ s.nodeMetrics.InfosSent, s.nodeMetrics.InfosReceived, s.nodeMetrics.BytesSent, s.nodeMetrics.BytesReceived, } { if count := counter.Count(); count <= 0 { return errors.Errorf("%d: expected metrics counter %q > 0; = %d", i, counter.GetName(), count) } } } // Since there are two gossip nodes, there should be exactly one incoming // or outgoing connection due to gossip's connection de-duplication. for i, g := range []*Gossip{local, remote} { g.mu.Lock() defer g.mu.Unlock() count := int64(0) for _, gauge := range []*metric.Gauge{g.mu.incoming.gauge, g.outgoing.gauge} { if gauge == nil { return errors.Errorf("%d: missing gauge", i) } count += gauge.Value() } const expected = 1 if count != expected { return errors.Errorf("%d: expected metrics incoming + outgoing connection count == %d; = %d", i, expected, count) } } return nil }) }
// startFakeServerGossips creates local gossip instances and remote // faked gossip instance. The remote gossip instance launches its // faked gossip service just for check the client message. func startFakeServerGossips( t *testing.T, localNodeID roachpb.NodeID, ) (*Gossip, *fakeGossipServer, *stop.Stopper) { stopper := stop.NewStopper() lRPCContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper) lserver := rpc.NewServer(lRPCContext) local := NewTest(localNodeID, lRPCContext, lserver, nil, stopper, metric.NewRegistry()) lln, err := netutil.ListenAndServeGRPC(stopper, lserver, util.IsolatedTestAddr) if err != nil { t.Fatal(err) } local.start(lln.Addr()) rRPCContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper) rserver := rpc.NewServer(rRPCContext) rln, err := netutil.ListenAndServeGRPC(stopper, rserver, util.IsolatedTestAddr) if err != nil { t.Fatal(err) } remote := newFakeGossipServer(rserver, stopper) addr := rln.Addr() remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String()) return local, remote, stopper }
// TestClientForwardUnresolved verifies that a client does not resolve a forward // address prematurely. func TestClientForwardUnresolved(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() const nodeID = 1 local := startGossip(nodeID, stopper, t, metric.NewRegistry()) addr := local.GetNodeAddr() client := newClient(log.AmbientContext{}, addr, makeMetrics()) // never started newAddr := util.UnresolvedAddr{ NetworkField: "tcp", AddressField: "localhost:2345", } reply := &Response{ NodeID: nodeID, Addr: *addr, AlternateNodeID: nodeID + 1, AlternateAddr: &newAddr, } if err := client.handleResponse( context.TODO(), local, reply, ); !testutils.IsError(err, "received forward") { t.Fatal(err) } if !proto.Equal(client.forwardAddr, &newAddr) { t.Fatalf("unexpected forward address %v, expected %v", client.forwardAddr, &newAddr) } }
// Start starts the test cluster by bootstrapping an in-memory store // (defaults to maximum of 50M). The server is started, launching the // node RPC server and all HTTP endpoints. Use the value of // TestServer.Addr after Start() for client connections. Use Stop() // to shutdown the server after the test completes. func (ltc *LocalTestCluster) Start(t util.Tester, baseCtx *base.Config, initSender InitSenderFn) { ambient := log.AmbientContext{Tracer: tracing.NewTracer()} nc := &base.NodeIDContainer{} ambient.AddLogTag("n", nc) nodeID := roachpb.NodeID(1) nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID} ltc.tester = t ltc.Manual = hlc.NewManualClock(0) ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano) ltc.Stopper = stop.NewStopper() rpcContext := rpc.NewContext(ambient, baseCtx, ltc.Clock, ltc.Stopper) server := rpc.NewServer(rpcContext) // never started ltc.Gossip = gossip.New(ambient, nc, rpcContext, server, nil, ltc.Stopper, metric.NewRegistry()) ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20) ltc.Stopper.AddCloser(ltc.Eng) ltc.Stores = storage.NewStores(ambient, ltc.Clock) ltc.Sender = initSender(nodeDesc, ambient.Tracer, ltc.Clock, ltc.Latency, ltc.Stores, ltc.Stopper, ltc.Gossip) if ltc.DBContext == nil { dbCtx := client.DefaultDBContext() ltc.DBContext = &dbCtx } ltc.DB = client.NewDBWithContext(ltc.Sender, *ltc.DBContext) transport := storage.NewDummyRaftTransport() cfg := storage.TestStoreConfig() if ltc.RangeRetryOptions != nil { cfg.RangeRetryOptions = *ltc.RangeRetryOptions } cfg.AmbientCtx = ambient cfg.Clock = ltc.Clock cfg.DB = ltc.DB cfg.Gossip = ltc.Gossip cfg.Transport = transport cfg.MetricsSampleInterval = metric.TestSampleInterval ltc.Store = storage.NewStore(cfg, ltc.Eng, nodeDesc) if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } ltc.Stores.AddStore(ltc.Store) if err := ltc.Store.BootstrapRange(nil); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } if err := ltc.Store.Start(context.Background(), ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } nc.Set(context.TODO(), nodeDesc.NodeID) if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil { t.Fatalf("unable to set node descriptor: %s", err) } }
// TestGossipCullNetwork verifies that a client will be culled from // the network periodically (at cullInterval duration intervals). func TestGossipCullNetwork(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t, metric.NewRegistry()) local.SetCullInterval(5 * time.Millisecond) local.mu.Lock() for i := 0; i < minPeers; i++ { peer := startGossip(roachpb.NodeID(i+2), stopper, t, metric.NewRegistry()) local.startClient(peer.GetNodeAddr()) } local.mu.Unlock() const slowGossipDuration = time.Minute if err := util.RetryForDuration(slowGossipDuration, func() error { if peers := len(local.Outgoing()); peers != minPeers { return errors.Errorf("%d of %d peers connected", peers, minPeers) } return nil }); err != nil { t.Fatalf("condition failed to evaluate within %s: %s", slowGossipDuration, err) } local.manage() if err := util.RetryForDuration(slowGossipDuration, func() error { // Verify that a client is closed within the cull interval. if peers := len(local.Outgoing()); peers != minPeers-1 { return errors.Errorf("%d of %d peers connected", peers, minPeers-1) } return nil }); err != nil { t.Fatalf("condition failed to evaluate within %s: %s", slowGossipDuration, err) } }
// TestClientRetryBootstrap verifies that an initial failure to connect // to a bootstrap host doesn't stall the bootstrapping process in the // absence of any additional activity. This can happen during acceptance // tests if the DNS can't lookup hostnames when gossip is started. func TestClientRetryBootstrap(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t, metric.NewRegistry()) remote := startGossip(2, stopper, t, metric.NewRegistry()) if err := local.AddInfo("local-key", []byte("hello"), 0*time.Second); err != nil { t.Fatal(err) } local.SetBootstrapInterval(10 * time.Millisecond) local.SetResolvers([]resolver.Resolver{ &testResolver{addr: remote.GetNodeAddr().String(), numFails: 3, numSuccesses: 1}, }) local.bootstrap() local.manage() util.SucceedsSoon(t, func() error { _, err := remote.GetInfo("local-key") return err }) }
func newRaftTransportTestContext(t testing.TB) *raftTransportTestContext { rttc := &raftTransportTestContext{ t: t, stopper: stop.NewStopper(), transports: map[roachpb.NodeID]*storage.RaftTransport{}, } rttc.nodeRPCContext = rpc.NewContext( log.AmbientContext{}, testutils.NewNodeTestBaseContext(), nil, rttc.stopper, ) server := rpc.NewServer(rttc.nodeRPCContext) // never started rttc.gossip = gossip.NewTest( 1, rttc.nodeRPCContext, server, nil, rttc.stopper, metric.NewRegistry(), ) return rttc }
// TestClientRegisterInitNodeID verifies two client's gossip request with NodeID 0. func TestClientRegisterWithInitNodeID(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() // Create three gossip nodes, and connect to the first with NodeID 0. var g []*Gossip var gossipAddr string for i := 0; i < 3; i++ { RPCContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper) server := rpc.NewServer(RPCContext) ln, err := netutil.ListenAndServeGRPC(stopper, server, util.IsolatedTestAddr) if err != nil { t.Fatal(err) } // Connect to the first gossip node. if gossipAddr == "" { gossipAddr = ln.Addr().String() } var resolvers []resolver.Resolver resolver, err := resolver.NewResolver(gossipAddr) if err != nil { t.Fatal(err) } resolvers = append(resolvers, resolver) // node ID must be non-zero gnode := NewTest( roachpb.NodeID(i+1), RPCContext, server, resolvers, stopper, metric.NewRegistry(), ) g = append(g, gnode) gnode.Start(ln.Addr()) } util.SucceedsSoon(t, func() error { // The first gossip node should have two gossip client address // in nodeMap if these three gossip nodes registered success. g[0].mu.Lock() defer g[0].mu.Unlock() if a, e := len(g[0].mu.nodeMap), 2; a != e { return errors.Errorf("expected %s to contain %d nodes, got %d", g[0].mu.nodeMap, e, a) } return nil }) }
// MustGetSQLNetworkCounter implements TestServerInterface. func (ts *TestServer) MustGetSQLNetworkCounter(name string) int64 { var c int64 var found bool reg := metric.NewRegistry() reg.AddMetricStruct(ts.pgServer.Metrics()) reg.Each(func(n string, v interface{}) { if name == n { c = v.(*metric.Counter).Count() found = true } }) if !found { panic(fmt.Sprintf("couldn't find metric %s", name)) } return c }
// TestGossipInfoStore verifies operation of gossip instance infostore. func TestGossipInfoStore(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() rpcContext := newInsecureRPCContext(stopper) g := NewTest(1, rpcContext, rpc.NewServer(rpcContext), nil, stopper, metric.NewRegistry()) slice := []byte("b") if err := g.AddInfo("s", slice, time.Hour); err != nil { t.Fatal(err) } if val, err := g.GetInfo("s"); !bytes.Equal(val, slice) || err != nil { t.Errorf("error fetching string: %v", err) } if _, err := g.GetInfo("s2"); err == nil { t.Errorf("expected error fetching nonexistent key \"s2\"") } }
// TestGossipOverwriteNode verifies that if a new node is added with the same // address as an old node, that old node is removed from the cluster. func TestGossipOverwriteNode(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() rpcContext := newInsecureRPCContext(stopper) g := NewTest(1, rpcContext, rpc.NewServer(rpcContext), nil, stopper, metric.NewRegistry()) node1 := &roachpb.NodeDescriptor{NodeID: 1, Address: util.MakeUnresolvedAddr("tcp", "1.1.1.1:1")} node2 := &roachpb.NodeDescriptor{NodeID: 2, Address: util.MakeUnresolvedAddr("tcp", "2.2.2.2:2")} if err := g.SetNodeDescriptor(node1); err != nil { t.Fatal(err) } if err := g.SetNodeDescriptor(node2); err != nil { t.Fatal(err) } if val, err := g.GetNodeDescriptor(node1.NodeID); err != nil { t.Error(err) } else if val.NodeID != node1.NodeID { t.Errorf("expected node %d, got %+v", node1.NodeID, val) } if val, err := g.GetNodeDescriptor(node2.NodeID); err != nil { t.Error(err) } else if val.NodeID != node2.NodeID { t.Errorf("expected node %d, got %+v", node2.NodeID, val) } // Give node3 the same address as node1, which should cause node1 to be // removed from the cluster. node3 := &roachpb.NodeDescriptor{NodeID: 3, Address: node1.Address} if err := g.SetNodeDescriptor(node3); err != nil { t.Fatal(err) } if val, err := g.GetNodeDescriptor(node3.NodeID); err != nil { t.Error(err) } else if val.NodeID != node3.NodeID { t.Errorf("expected node %d, got %+v", node3.NodeID, val) } // Quiesce the stopper now to ensure that the update has propagated before // checking whether node 1 has been removed from the infoStore. stopper.Quiesce() expectedErr := "unable to look up descriptor for node" if val, err := g.GetNodeDescriptor(node1.NodeID); !testutils.IsError(err, expectedErr) { t.Errorf("expected error %q fetching node %d; got error %v and node %+v", expectedErr, node1.NodeID, err, val) } }
// CreateNode creates a simulation node and starts an RPC server for it. func (n *Network) CreateNode() (*Node, error) { server := rpc.NewServer(n.rpcContext) ln, err := net.Listen(util.TestAddr.Network(), util.TestAddr.String()) if err != nil { return nil, err } node := &Node{Server: server, Listener: ln, Registry: metric.NewRegistry()} node.Gossip = gossip.NewTest(0, n.rpcContext, server, nil, n.Stopper, node.Registry) n.Stopper.RunWorker(func() { <-n.Stopper.ShouldQuiesce() netutil.FatalIfUnexpected(ln.Close()) <-n.Stopper.ShouldStop() server.Stop() node.Gossip.EnableSimulationCycler(false) }) n.Nodes = append(n.Nodes, node) return node, nil }
// createTestStorePool creates a stopper, gossip and storePool for use in // tests. Stopper must be stopped by the caller. func createTestStorePool( timeUntilStoreDead time.Duration, ) (*stop.Stopper, *gossip.Gossip, *hlc.ManualClock, *StorePool) { stopper := stop.NewStopper() mc := hlc.NewManualClock(0) clock := hlc.NewClock(mc.UnixNano) rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, clock, stopper) server := rpc.NewServer(rpcContext) // never started g := gossip.NewTest(1, rpcContext, server, nil, stopper, metric.NewRegistry()) storePool := NewStorePool( log.AmbientContext{}, g, clock, rpcContext, timeUntilStoreDead, stopper, ) return stopper, g, mc, storePool }
// TestClientDisconnectLoopback verifies that the gossip server // will drop an outgoing client connection that is already an // inbound client connection of another node. func TestClientDisconnectLoopback(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t, metric.NewRegistry()) // startClient requires locks are held, so acquire here. local.mu.Lock() lAddr := local.mu.is.NodeAddr local.startClient(&lAddr, local.NodeID.Get()) local.mu.Unlock() local.manage() util.SucceedsSoon(t, func() error { ok := local.findClient(func(c *client) bool { return c.addr.String() == lAddr.String() }) != nil if !ok && verifyServerMaps(local, 0) { return nil } return errors.New("local client still connected to itself") }) }
// createTestStorePool creates a stopper, gossip and storePool for use in // tests. Stopper must be stopped by the caller. func createTestStorePool( timeUntilStoreDead time.Duration, deterministic bool, defaultNodeLiveness bool, ) (*stop.Stopper, *gossip.Gossip, *hlc.ManualClock, *StorePool, *mockNodeLiveness) { stopper := stop.NewStopper() mc := hlc.NewManualClock(123) clock := hlc.NewClock(mc.UnixNano, time.Nanosecond) rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, clock, stopper) server := rpc.NewServer(rpcContext) // never started g := gossip.NewTest(1, rpcContext, server, nil, stopper, metric.NewRegistry()) mnl := newMockNodeLiveness(defaultNodeLiveness) storePool := NewStorePool( log.AmbientContext{}, g, clock, mnl.nodeLivenessFunc, timeUntilStoreDead, deterministic, ) return stopper, g, mc, storePool, mnl }
func TestGossipGetNextBootstrapAddress(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() resolverSpecs := []string{ "127.0.0.1:9000", "127.0.0.1:9001", "localhost:9004", } resolvers := []resolver.Resolver{} for _, rs := range resolverSpecs { resolver, err := resolver.NewResolver(rs) if err == nil { resolvers = append(resolvers, resolver) } } if len(resolvers) != 3 { t.Errorf("expected 3 resolvers; got %d", len(resolvers)) } server := rpc.NewServer( rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper), ) g := NewTest(0, nil, server, resolvers, stop.NewStopper(), metric.NewRegistry()) // Using specified resolvers, fetch bootstrap addresses 3 times // and verify the results match expected addresses. expAddresses := []string{ "127.0.0.1:9000", "127.0.0.1:9001", "localhost:9004", } for i := 0; i < len(expAddresses); i++ { if addr := g.getNextBootstrapAddress(); addr == nil { t.Errorf("%d: unexpected nil addr when expecting %s", i, expAddresses[i]) } else if addrStr := addr.String(); addrStr != expAddresses[i] { t.Errorf("%d: expected addr %s; got %s", i, expAddresses[i], addrStr) } } }
// createCluster generates a new cluster using the provided stopper and the // number of nodes supplied. Each node will have one store to start. func createCluster( stopper *stop.Stopper, nodeCount int, epochWriter, actionWriter io.Writer, script Script, rand *rand.Rand, ) *Cluster { clock := hlc.NewClock(hlc.UnixNano, time.Nanosecond) rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, clock, stopper) server := rpc.NewServer(rpcContext) // We set the node ID to MaxInt32 for the cluster Gossip instance to prevent // conflicts with real node IDs. g := gossip.NewTest(math.MaxInt32, rpcContext, server, nil, stopper, metric.NewRegistry()) // Set the store pool to deterministic so that a run with the exact same // input will always produce the same output. storePool := storage.NewStorePool( log.AmbientContext{}, g, clock, rpcContext, storage.TestTimeUntilStoreDeadOff, stopper, /* deterministic */ true, ) c := &Cluster{ stopper: stopper, clock: clock, rpc: rpcContext, gossip: g, storePool: storePool, allocator: storage.MakeAllocator(storePool, storage.AllocatorOptions{ AllowRebalance: true, }), storeGossiper: gossiputil.NewStoreGossiper(g), nodes: make(map[roachpb.NodeID]*Node), stores: make(map[roachpb.StoreID]*Store), ranges: make(map[roachpb.RangeID]*Range), rangeIDsByStore: make(map[roachpb.StoreID]roachpb.RangeIDSlice), rand: rand, epochWriter: tabwriter.NewWriter(epochWriter, 8, 1, 2, ' ', 0), actionWriter: tabwriter.NewWriter(actionWriter, 8, 1, 2, ' ', 0), script: script, epoch: -1, } // Add the nodes. for i := 0; i < nodeCount; i++ { c.addNewNodeWithStore() } // Add a single range and add to this first node's first store. firstRange := c.addRange() firstRange.addReplica(c.stores[0]) c.calculateRangeIDsByStore() // Output the first epoch header. c.epoch = 0 c.OutputEpochHeader() c.OutputEpoch() c.flush() return c }
// NewServer creates a Server from a server.Context. func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { if _, err := net.ResolveTCPAddr("tcp", cfg.AdvertiseAddr); err != nil { return nil, errors.Errorf("unable to resolve RPC address %q: %v", cfg.AdvertiseAddr, err) } if cfg.AmbientCtx.Tracer == nil { cfg.AmbientCtx.Tracer = tracing.NewTracer() } // Try loading the TLS configs before anything else. if _, err := cfg.GetServerTLSConfig(); err != nil { return nil, err } if _, err := cfg.GetClientTLSConfig(); err != nil { return nil, err } s := &Server{ mux: http.NewServeMux(), clock: hlc.NewClock(hlc.UnixNano, cfg.MaxOffset), stopper: stopper, cfg: cfg, } // Add a dynamic log tag value for the node ID. // // We need to pass an ambient context to the various server components, but we // won't know the node ID until we Start(). At that point it's too late to // change the ambient contexts in the components (various background processes // will have already started using them). // // NodeIDContainer allows us to add the log tag to the context now and update // the value asynchronously. It's not significantly more expensive than a // regular tag since it's just doing an (atomic) load when a log/trace message // is constructed. The node ID is set by the Store if this host was // bootstrapped; otherwise a new one is allocated in Node. s.cfg.AmbientCtx.AddLogTag("n", &s.nodeIDContainer) ctx := s.AnnotateCtx(context.Background()) if s.cfg.Insecure { log.Warning(ctx, "running in insecure mode, this is strongly discouraged. See --insecure.") } s.rpcContext = rpc.NewContext(s.cfg.AmbientCtx, s.cfg.Config, s.clock, s.stopper) s.rpcContext.HeartbeatCB = func() { if err := s.rpcContext.RemoteClocks.VerifyClockOffset(); err != nil { log.Fatal(ctx, err) } } s.grpc = rpc.NewServer(s.rpcContext) s.registry = metric.NewRegistry() s.gossip = gossip.New( s.cfg.AmbientCtx, &s.nodeIDContainer, s.rpcContext, s.grpc, s.cfg.GossipBootstrapResolvers, s.stopper, s.registry, ) s.storePool = storage.NewStorePool( s.cfg.AmbientCtx, s.gossip, s.clock, s.rpcContext, s.cfg.TimeUntilStoreDead, s.stopper, /* deterministic */ false, ) // A custom RetryOptions is created which uses stopper.ShouldQuiesce() as // the Closer. This prevents infinite retry loops from occurring during // graceful server shutdown // // Such a loop loop occurs with the DistSender attempts a connection to the // local server during shutdown, and receives an internal server error (HTTP // Code 5xx). This is the correct error for a server to return when it is // shutting down, and is normally retryable in a cluster environment. // However, on a single-node setup (such as a test), retries will never // succeed because the only server has been shut down; thus, thus the // DistSender needs to know that it should not retry in this situation. retryOpts := base.DefaultRetryOptions() retryOpts.Closer = s.stopper.ShouldQuiesce() distSenderCfg := kv.DistSenderConfig{ AmbientCtx: s.cfg.AmbientCtx, Clock: s.clock, RPCContext: s.rpcContext, RPCRetryOptions: &retryOpts, } s.distSender = kv.NewDistSender(distSenderCfg, s.gossip) txnMetrics := kv.MakeTxnMetrics(s.cfg.MetricsSampleInterval) s.registry.AddMetricStruct(txnMetrics) s.txnCoordSender = kv.NewTxnCoordSender( s.cfg.AmbientCtx, s.distSender, s.clock, s.cfg.Linearizable, s.stopper, txnMetrics, ) s.db = client.NewDB(s.txnCoordSender) // Use the range lease expiration and renewal durations as the node // liveness expiration and heartbeat interval. active, renewal := storage.RangeLeaseDurations( storage.RaftElectionTimeout(s.cfg.RaftTickInterval, s.cfg.RaftElectionTimeoutTicks)) s.nodeLiveness = storage.NewNodeLiveness( s.cfg.AmbientCtx, s.clock, s.db, s.gossip, active, renewal, ) s.registry.AddMetricStruct(s.nodeLiveness.Metrics()) s.raftTransport = storage.NewRaftTransport( s.cfg.AmbientCtx, storage.GossipAddressResolver(s.gossip), s.grpc, s.rpcContext, ) s.kvDB = kv.NewDBServer(s.cfg.Config, s.txnCoordSender, s.stopper) roachpb.RegisterExternalServer(s.grpc, s.kvDB) // Set up internal memory metrics for use by internal SQL executors. s.internalMemMetrics = sql.MakeMemMetrics("internal") s.registry.AddMetricStruct(s.internalMemMetrics) // Set up Lease Manager var lmKnobs sql.LeaseManagerTestingKnobs if cfg.TestingKnobs.SQLLeaseManager != nil { lmKnobs = *s.cfg.TestingKnobs.SQLLeaseManager.(*sql.LeaseManagerTestingKnobs) } s.leaseMgr = sql.NewLeaseManager(&s.nodeIDContainer, *s.db, s.clock, lmKnobs, s.stopper, &s.internalMemMetrics) s.leaseMgr.RefreshLeases(s.stopper, s.db, s.gossip) // Set up the DistSQL server distSQLCfg := distsql.ServerConfig{ AmbientContext: s.cfg.AmbientCtx, DB: s.db, RPCContext: s.rpcContext, Stopper: s.stopper, } s.distSQLServer = distsql.NewServer(distSQLCfg) distsql.RegisterDistSQLServer(s.grpc, s.distSQLServer) // Set up admin memory metrics for use by admin SQL executors. s.adminMemMetrics = sql.MakeMemMetrics("admin") s.registry.AddMetricStruct(s.adminMemMetrics) // Set up Executor execCfg := sql.ExecutorConfig{ AmbientCtx: s.cfg.AmbientCtx, NodeID: &s.nodeIDContainer, DB: s.db, Gossip: s.gossip, LeaseManager: s.leaseMgr, Clock: s.clock, DistSQLSrv: s.distSQLServer, MetricsSampleInterval: s.cfg.MetricsSampleInterval, } if s.cfg.TestingKnobs.SQLExecutor != nil { execCfg.TestingKnobs = s.cfg.TestingKnobs.SQLExecutor.(*sql.ExecutorTestingKnobs) } else { execCfg.TestingKnobs = &sql.ExecutorTestingKnobs{} } if s.cfg.TestingKnobs.SQLSchemaChanger != nil { execCfg.SchemaChangerTestingKnobs = s.cfg.TestingKnobs.SQLSchemaChanger.(*sql.SchemaChangerTestingKnobs) } else { execCfg.SchemaChangerTestingKnobs = &sql.SchemaChangerTestingKnobs{} } s.sqlExecutor = sql.NewExecutor(execCfg, s.stopper, &s.adminMemMetrics) s.registry.AddMetricStruct(s.sqlExecutor) s.pgServer = pgwire.MakeServer( s.cfg.AmbientCtx, s.cfg.Config, s.sqlExecutor, &s.internalMemMetrics, s.cfg.SQLMemoryPoolSize, ) s.registry.AddMetricStruct(s.pgServer.Metrics()) s.tsDB = ts.NewDB(s.db) s.tsServer = ts.MakeServer(s.cfg.AmbientCtx, s.tsDB, s.cfg.TimeSeriesServerConfig, s.stopper) // TODO(bdarnell): make StoreConfig configurable. storeCfg := storage.StoreConfig{ AmbientCtx: s.cfg.AmbientCtx, Clock: s.clock, DB: s.db, Gossip: s.gossip, NodeLiveness: s.nodeLiveness, Transport: s.raftTransport, RaftTickInterval: s.cfg.RaftTickInterval, ScanInterval: s.cfg.ScanInterval, ScanMaxIdleTime: s.cfg.ScanMaxIdleTime, ConsistencyCheckInterval: s.cfg.ConsistencyCheckInterval, ConsistencyCheckPanicOnFailure: s.cfg.ConsistencyCheckPanicOnFailure, MetricsSampleInterval: s.cfg.MetricsSampleInterval, StorePool: s.storePool, SQLExecutor: sql.InternalExecutor{ LeaseManager: s.leaseMgr, }, LogRangeEvents: s.cfg.EventLogEnabled, AllocatorOptions: storage.AllocatorOptions{ AllowRebalance: true, }, RangeLeaseActiveDuration: active, RangeLeaseRenewalDuration: renewal, TimeSeriesDataStore: s.tsDB, } if s.cfg.TestingKnobs.Store != nil { storeCfg.TestingKnobs = *s.cfg.TestingKnobs.Store.(*storage.StoreTestingKnobs) } s.recorder = status.NewMetricsRecorder(s.clock) s.registry.AddMetricStruct(s.rpcContext.RemoteClocks.Metrics()) s.runtime = status.MakeRuntimeStatSampler(s.clock) s.registry.AddMetricStruct(s.runtime) s.node = NewNode(storeCfg, s.recorder, s.registry, s.stopper, txnMetrics, sql.MakeEventLogger(s.leaseMgr)) roachpb.RegisterInternalServer(s.grpc, s.node) storage.RegisterConsistencyServer(s.grpc, s.node.storesServer) storage.RegisterFreezeServer(s.grpc, s.node.storesServer) s.admin = newAdminServer(s) s.status = newStatusServer( s.cfg.AmbientCtx, s.db, s.gossip, s.recorder, s.rpcContext, s.node.stores, ) for _, gw := range []grpcGatewayServer{s.admin, s.status, &s.tsServer} { gw.RegisterService(s.grpc) } return s, nil }
func TestGossipOrphanedStallDetection(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t, metric.NewRegistry()) local.SetStallInterval(5 * time.Millisecond) // Make sure we have the sentinel to ensure that its absence is not the // cause of stall detection. if err := local.AddInfo(KeySentinel, nil, time.Hour); err != nil { t.Fatal(err) } peerStopper := stop.NewStopper() peer := startGossip(2, peerStopper, t, metric.NewRegistry()) peerNodeID := peer.NodeID.Get() peerAddr := peer.GetNodeAddr() peerAddrStr := peerAddr.String() local.startClient(peerAddr) util.SucceedsSoon(t, func() error { for _, peerID := range local.Outgoing() { if peerID == peerNodeID { return nil } } return errors.Errorf("node %d not yet connected", peerNodeID) }) util.SucceedsSoon(t, func() error { for _, resolver := range local.GetResolvers() { if resolver.Addr() == peerAddrStr { return nil } } return errors.Errorf("node %d descriptor not yet available", peerNodeID) }) local.bootstrap() local.manage() peerStopper.Stop() util.SucceedsSoon(t, func() error { for _, peerID := range local.Outgoing() { if peerID == peerNodeID { return errors.Errorf("node %d still connected", peerNodeID) } } return nil }) peerStopper = stop.NewStopper() defer peerStopper.Stop() startGossipAtAddr(peerNodeID, peerAddr, peerStopper, t, metric.NewRegistry()) util.SucceedsSoon(t, func() error { for _, peerID := range local.Outgoing() { if peerID == peerNodeID { return nil } } return errors.Errorf("node %d not yet connected", peerNodeID) }) }
func newStoreMetrics(sampleInterval time.Duration) *StoreMetrics { storeRegistry := metric.NewRegistry() sm := &StoreMetrics{ registry: storeRegistry, // Replica metrics. ReplicaCount: metric.NewCounter(metaReplicaCount), ReservedReplicaCount: metric.NewCounter(metaReservedReplicaCount), RaftLeaderCount: metric.NewGauge(metaRaftLeaderCount), RaftLeaderNotLeaseHolderCount: metric.NewGauge(metaRaftLeaderNotLeaseHolderCount), LeaseHolderCount: metric.NewGauge(metaLeaseHolderCount), QuiescentCount: metric.NewGauge(metaQuiescentCount), // Replica CommandQueue metrics. MaxCommandQueueSize: metric.NewGauge(metaMaxCommandQueueSize), MaxCommandQueueWriteCount: metric.NewGauge(metaMaxCommandQueueWriteCount), MaxCommandQueueReadCount: metric.NewGauge(metaMaxCommandQueueReadCount), MaxCommandQueueTreeSize: metric.NewGauge(metaMaxCommandQueueTreeSize), MaxCommandQueueOverlaps: metric.NewGauge(metaMaxCommandQueueOverlaps), CombinedCommandQueueSize: metric.NewGauge(metaCombinedCommandQueueSize), CombinedCommandWriteCount: metric.NewGauge(metaCombinedCommandWriteCount), CombinedCommandReadCount: metric.NewGauge(metaCombinedCommandReadCount), // Range metrics. RangeCount: metric.NewGauge(metaRangeCount), UnavailableRangeCount: metric.NewGauge(metaUnavailableRangeCount), UnderReplicatedRangeCount: metric.NewGauge(metaUnderReplicatedRangeCount), // Lease request metrics. LeaseRequestSuccessCount: metric.NewCounter(metaLeaseRequestSuccessCount), LeaseRequestErrorCount: metric.NewCounter(metaLeaseRequestErrorCount), // Storage metrics. LiveBytes: metric.NewGauge(metaLiveBytes), KeyBytes: metric.NewGauge(metaKeyBytes), ValBytes: metric.NewGauge(metaValBytes), IntentBytes: metric.NewGauge(metaIntentBytes), LiveCount: metric.NewGauge(metaLiveCount), KeyCount: metric.NewGauge(metaKeyCount), ValCount: metric.NewGauge(metaValCount), IntentCount: metric.NewGauge(metaIntentCount), IntentAge: metric.NewGauge(metaIntentAge), GcBytesAge: metric.NewGauge(metaGcBytesAge), LastUpdateNanos: metric.NewGauge(metaLastUpdateNanos), Capacity: metric.NewGauge(metaCapacity), Available: metric.NewGauge(metaAvailable), Reserved: metric.NewCounter(metaReserved), SysBytes: metric.NewGauge(metaSysBytes), SysCount: metric.NewGauge(metaSysCount), // RocksDB metrics. RdbBlockCacheHits: metric.NewGauge(metaRdbBlockCacheHits), RdbBlockCacheMisses: metric.NewGauge(metaRdbBlockCacheMisses), RdbBlockCacheUsage: metric.NewGauge(metaRdbBlockCacheUsage), RdbBlockCachePinnedUsage: metric.NewGauge(metaRdbBlockCachePinnedUsage), RdbBloomFilterPrefixChecked: metric.NewGauge(metaRdbBloomFilterPrefixChecked), RdbBloomFilterPrefixUseful: metric.NewGauge(metaRdbBloomFilterPrefixUseful), RdbMemtableHits: metric.NewGauge(metaRdbMemtableHits), RdbMemtableMisses: metric.NewGauge(metaRdbMemtableMisses), RdbMemtableTotalSize: metric.NewGauge(metaRdbMemtableTotalSize), RdbFlushes: metric.NewGauge(metaRdbFlushes), RdbCompactions: metric.NewGauge(metaRdbCompactions), RdbTableReadersMemEstimate: metric.NewGauge(metaRdbTableReadersMemEstimate), RdbReadAmplification: metric.NewGauge(metaRdbReadAmplification), RdbNumSSTables: metric.NewGauge(metaRdbNumSSTables), // Range event metrics. RangeSplits: metric.NewCounter(metaRangeSplits), RangeAdds: metric.NewCounter(metaRangeAdds), RangeRemoves: metric.NewCounter(metaRangeRemoves), RangeSnapshotsGenerated: metric.NewCounter(metaRangeSnapshotsGenerated), RangeSnapshotsNormalApplied: metric.NewCounter(metaRangeSnapshotsNormalApplied), RangeSnapshotsPreemptiveApplied: metric.NewCounter(metaRangeSnapshotsPreemptiveApplied), // Raft processing metrics. RaftTicks: metric.NewCounter(metaRaftTicks), RaftWorkingDurationNanos: metric.NewCounter(metaRaftWorkingDurationNanos), RaftTickingDurationNanos: metric.NewCounter(metaRaftTickingDurationNanos), // Raft message metrics. RaftRcvdMsgProp: metric.NewCounter(metaRaftRcvdProp), RaftRcvdMsgApp: metric.NewCounter(metaRaftRcvdApp), RaftRcvdMsgAppResp: metric.NewCounter(metaRaftRcvdAppResp), RaftRcvdMsgVote: metric.NewCounter(metaRaftRcvdVote), RaftRcvdMsgVoteResp: metric.NewCounter(metaRaftRcvdVoteResp), RaftRcvdMsgPreVote: metric.NewCounter(metaRaftRcvdPreVote), RaftRcvdMsgPreVoteResp: metric.NewCounter(metaRaftRcvdPreVoteResp), RaftRcvdMsgSnap: metric.NewCounter(metaRaftRcvdSnap), RaftRcvdMsgHeartbeat: metric.NewCounter(metaRaftRcvdHeartbeat), RaftRcvdMsgHeartbeatResp: metric.NewCounter(metaRaftRcvdHeartbeatResp), RaftRcvdMsgTransferLeader: metric.NewCounter(metaRaftRcvdTransferLeader), RaftRcvdMsgTimeoutNow: metric.NewCounter(metaRaftRcvdTimeoutNow), RaftRcvdMsgDropped: metric.NewCounter(metaRaftRcvdDropped), raftRcvdMessages: make(map[raftpb.MessageType]*metric.Counter, len(raftpb.MessageType_name)), RaftEnqueuedPending: metric.NewGauge(metaRaftEnqueuedPending), // This Gauge measures the number of heartbeats queued up just before // the queue is cleared, to avoid flapping wildly. RaftCoalescedHeartbeatsPending: metric.NewGauge(metaRaftCoalescedHeartbeatsPending), // Replica queue metrics. GCQueueSuccesses: metric.NewCounter(metaGCQueueSuccesses), GCQueueFailures: metric.NewCounter(metaGCQueueFailures), GCQueuePending: metric.NewGauge(metaGCQueuePending), GCQueueProcessingNanos: metric.NewCounter(metaGCQueueProcessingNanos), RaftLogQueueSuccesses: metric.NewCounter(metaRaftLogQueueSuccesses), RaftLogQueueFailures: metric.NewCounter(metaRaftLogQueueFailures), RaftLogQueuePending: metric.NewGauge(metaRaftLogQueuePending), RaftLogQueueProcessingNanos: metric.NewCounter(metaRaftLogQueueProcessingNanos), ConsistencyQueueSuccesses: metric.NewCounter(metaConsistencyQueueSuccesses), ConsistencyQueueFailures: metric.NewCounter(metaConsistencyQueueFailures), ConsistencyQueuePending: metric.NewGauge(metaConsistencyQueuePending), ConsistencyQueueProcessingNanos: metric.NewCounter(metaConsistencyQueueProcessingNanos), ReplicaGCQueueSuccesses: metric.NewCounter(metaReplicaGCQueueSuccesses), ReplicaGCQueueFailures: metric.NewCounter(metaReplicaGCQueueFailures), ReplicaGCQueuePending: metric.NewGauge(metaReplicaGCQueuePending), ReplicaGCQueueProcessingNanos: metric.NewCounter(metaReplicaGCQueueProcessingNanos), ReplicateQueueSuccesses: metric.NewCounter(metaReplicateQueueSuccesses), ReplicateQueueFailures: metric.NewCounter(metaReplicateQueueFailures), ReplicateQueuePending: metric.NewGauge(metaReplicateQueuePending), ReplicateQueueProcessingNanos: metric.NewCounter(metaReplicateQueueProcessingNanos), ReplicateQueuePurgatory: metric.NewGauge(metaReplicateQueuePurgatory), SplitQueueSuccesses: metric.NewCounter(metaSplitQueueSuccesses), SplitQueueFailures: metric.NewCounter(metaSplitQueueFailures), SplitQueuePending: metric.NewGauge(metaSplitQueuePending), SplitQueueProcessingNanos: metric.NewCounter(metaSplitQueueProcessingNanos), TimeSeriesMaintenanceQueueSuccesses: metric.NewCounter(metaTimeSeriesMaintenanceQueueFailures), TimeSeriesMaintenanceQueueFailures: metric.NewCounter(metaTimeSeriesMaintenanceQueueSuccesses), TimeSeriesMaintenanceQueuePending: metric.NewGauge(metaTimeSeriesMaintenanceQueuePending), TimeSeriesMaintenanceQueueProcessingNanos: metric.NewCounter(metaTimeSeriesMaintenanceQueueProcessingNanos), // GCInfo cumulative totals. GCNumKeysAffected: metric.NewCounter(metaGCNumKeysAffected), GCIntentsConsidered: metric.NewCounter(metaGCIntentsConsidered), GCIntentTxns: metric.NewCounter(metaGCIntentTxns), GCTransactionSpanScanned: metric.NewCounter(metaGCTransactionSpanScanned), GCTransactionSpanGCAborted: metric.NewCounter(metaGCTransactionSpanGCAborted), GCTransactionSpanGCCommitted: metric.NewCounter(metaGCTransactionSpanGCCommitted), GCTransactionSpanGCPending: metric.NewCounter(metaGCTransactionSpanGCPending), GCAbortSpanScanned: metric.NewCounter(metaGCAbortSpanScanned), GCAbortSpanConsidered: metric.NewCounter(metaGCAbortSpanConsidered), GCAbortSpanGCNum: metric.NewCounter(metaGCAbortSpanGCNum), GCPushTxn: metric.NewCounter(metaGCPushTxn), GCResolveTotal: metric.NewCounter(metaGCResolveTotal), GCResolveSuccess: metric.NewCounter(metaGCResolveSuccess), // Mutex timing. // // TODO(tschottdorf): Histograms don't work very well as they were // inherently built in a windowed (i.e. events-discarding) way, which // is not at all the correct way. Discard at one-minute interval which // gives sane (though mathematically nonsensical) results when exposed // at the moment. MuReplicaNanos: metric.NewHistogram( metaMuReplicaNanos, sampleInterval, time.Second.Nanoseconds(), 1, ), MuCommandQueueNanos: metric.NewHistogram( metaMuCommandQueueNanos, sampleInterval, time.Second.Nanoseconds(), 1, ), MuRaftNanos: metric.NewHistogram( metaMuRaftNanos, sampleInterval, time.Second.Nanoseconds(), 1, ), MuStoreNanos: metric.NewHistogram( metaMuStoreNanos, sampleInterval, time.Second.Nanoseconds(), 1, ), MuSchedulerNanos: metric.NewHistogram( metaMuSchedulerNanos, time.Minute, time.Second.Nanoseconds(), 1, ), } sm.raftRcvdMessages[raftpb.MsgProp] = sm.RaftRcvdMsgProp sm.raftRcvdMessages[raftpb.MsgApp] = sm.RaftRcvdMsgApp sm.raftRcvdMessages[raftpb.MsgAppResp] = sm.RaftRcvdMsgAppResp sm.raftRcvdMessages[raftpb.MsgVote] = sm.RaftRcvdMsgVote sm.raftRcvdMessages[raftpb.MsgVoteResp] = sm.RaftRcvdMsgVoteResp sm.raftRcvdMessages[raftpb.MsgPreVote] = sm.RaftRcvdMsgPreVote sm.raftRcvdMessages[raftpb.MsgPreVoteResp] = sm.RaftRcvdMsgPreVoteResp sm.raftRcvdMessages[raftpb.MsgSnap] = sm.RaftRcvdMsgSnap sm.raftRcvdMessages[raftpb.MsgHeartbeat] = sm.RaftRcvdMsgHeartbeat sm.raftRcvdMessages[raftpb.MsgHeartbeatResp] = sm.RaftRcvdMsgHeartbeatResp sm.raftRcvdMessages[raftpb.MsgTransferLeader] = sm.RaftRcvdMsgTransferLeader sm.raftRcvdMessages[raftpb.MsgTimeoutNow] = sm.RaftRcvdMsgTimeoutNow storeRegistry.AddMetricStruct(sm) return sm }
// TestGossipNoForwardSelf verifies that when a Gossip instance is full, it // redirects clients elsewhere (in particular not to itself). // // NB: Stress testing this test really stresses the OS networking stack // more than anything else. For example, on Linux it may quickly deplete // the ephemeral port range (due to the TIME_WAIT state). // On a box which only runs tests, this can be circumvented by running // // sudo bash -c "echo 1 > /proc/sys/net/ipv4/tcp_tw_recycle" // // See https://vincent.bernat.im/en/blog/2014-tcp-time-wait-state-linux.html // for details. // // On OSX, things similarly fall apart. See #7524 and #5218 for some discussion // of this. func TestGossipNoForwardSelf(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() local := startGossip(1, stopper, t, metric.NewRegistry()) ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Start one loopback client plus enough additional clients to fill the // incoming clients. peers := []*Gossip{local} local.server.mu.Lock() maxSize := local.server.mu.incoming.maxSize local.server.mu.Unlock() for i := 0; i < maxSize; i++ { peers = append(peers, startGossip(roachpb.NodeID(i+2), stopper, t, metric.NewRegistry())) } for _, peer := range peers { c := newClient(log.AmbientContext{}, local.GetNodeAddr(), makeMetrics()) util.SucceedsSoon(t, func() error { conn, err := peer.rpcContext.GRPCDial(c.addr.String(), grpc.WithBlock()) if err != nil { return err } stream, err := NewGossipClient(conn).Gossip(ctx) if err != nil { return err } if err := c.requestGossip(peer, stream); err != nil { return err } // Wait until the server responds, so we know we're connected. _, err = stream.Recv() return err }) } numClients := len(peers) * 2 disconnectedCh := make(chan *client) // Start a few overflow peers and assert that they don't get forwarded to us // again. for i := 0; i < numClients; i++ { local.server.mu.Lock() maxSize := local.server.mu.incoming.maxSize local.server.mu.Unlock() peer := startGossip(roachpb.NodeID(i+maxSize+2), stopper, t, metric.NewRegistry()) for { localAddr := local.GetNodeAddr() c := newClient(log.AmbientContext{}, localAddr, makeMetrics()) c.start(peer, disconnectedCh, peer.rpcContext, stopper, peer.rpcContext.NewBreaker()) disconnectedClient := <-disconnectedCh if disconnectedClient != c { t.Fatalf("expected %p to be disconnected, got %p", c, disconnectedClient) } else if c.forwardAddr == nil { // Under high load, clients sometimes fail to connect for reasons // unrelated to the test, so we need to permit some. t.Logf("node #%d: got nil forwarding address", peer.NodeID.Get()) continue } else if *c.forwardAddr == *localAddr { t.Errorf("node #%d: got local's forwarding address", peer.NodeID.Get()) } break } } }
// TestMetricsRecorder verifies that the metrics recorder properly formats the // statistics from various registries, both for Time Series and for Status // Summaries. func TestMetricsRecorder(t *testing.T) { defer leaktest.AfterTest(t)() // ======================================== // Construct a series of fake descriptors for use in test. // ======================================== nodeDesc := roachpb.NodeDescriptor{ NodeID: roachpb.NodeID(1), } storeDesc1 := roachpb.StoreDescriptor{ StoreID: roachpb.StoreID(1), Capacity: roachpb.StoreCapacity{ Capacity: 100, Available: 50, }, } storeDesc2 := roachpb.StoreDescriptor{ StoreID: roachpb.StoreID(2), Capacity: roachpb.StoreCapacity{ Capacity: 200, Available: 75, }, } // ======================================== // Create registries and add them to the recorder (two node-level, two // store-level). // ======================================== reg1 := metric.NewRegistry() store1 := fakeStore{ storeID: roachpb.StoreID(1), desc: storeDesc1, registry: metric.NewRegistry(), } store2 := fakeStore{ storeID: roachpb.StoreID(2), desc: storeDesc2, registry: metric.NewRegistry(), } manual := hlc.NewManualClock(100) recorder := NewMetricsRecorder(hlc.NewClock(manual.UnixNano, time.Nanosecond)) recorder.AddStore(store1) recorder.AddStore(store2) recorder.AddNode(reg1, nodeDesc, 50) // Ensure the metric system's view of time does not advance during this test // as the test expects time to not advance too far which would age the actual // data (e.g. in histogram's) unexpectedly. defer metric.TestingSetNow(func() time.Time { return time.Unix(0, manual.UnixNano()).UTC() })() // ======================================== // Generate Metrics Data & Expected Results // ======================================== // Flatten the four registries into an array for ease of use. regList := []struct { reg *metric.Registry prefix string source int64 isNode bool }{ { reg: reg1, prefix: "one.", source: 1, isNode: true, }, { reg: reg1, prefix: "two.", source: 1, isNode: true, }, { reg: store1.registry, prefix: "", source: int64(store1.storeID), isNode: false, }, { reg: store2.registry, prefix: "", source: int64(store2.storeID), isNode: false, }, } // Every registry will have a copy of the following metrics. metricNames := []struct { name string typ string val int64 }{ {"testGauge", "gauge", 20}, {"testGaugeFloat64", "floatgauge", 20}, {"testCounter", "counter", 5}, {"testCounterWithRates", "counterwithrates", 2}, {"testHistogram", "histogram", 10}, {"testLatency", "latency", 10}, // Stats needed for store summaries. {"ranges", "counter", 1}, {"replicas.leaders", "gauge", 1}, {"replicas.leaseholders", "gauge", 1}, {"ranges", "gauge", 1}, {"ranges.available", "gauge", 1}, } // Add the metrics to each registry and set their values. At the same time, // generate expected time series results and status summary metric values. var expected []tspb.TimeSeriesData expectedNodeSummaryMetrics := make(map[string]float64) expectedStoreSummaryMetrics := make(map[string]float64) // addExpected generates expected data for a single metric data point. addExpected := func(prefix, name string, source, time, val int64, isNode bool) { // Generate time series data. tsPrefix := "cr.node." if !isNode { tsPrefix = "cr.store." } expect := tspb.TimeSeriesData{ Name: tsPrefix + prefix + name, Source: strconv.FormatInt(source, 10), Datapoints: []tspb.TimeSeriesDatapoint{ { TimestampNanos: time, Value: float64(val), }, }, } expected = append(expected, expect) // Generate status summary data. if isNode { expectedNodeSummaryMetrics[prefix+name] = float64(val) } else { // This can overwrite the previous value, but this is expected as // all stores in our tests have identical values; when comparing // status summaries, the same map is used as expected data for all // stores. expectedStoreSummaryMetrics[prefix+name] = float64(val) } } for _, reg := range regList { for _, data := range metricNames { switch data.typ { case "gauge": g := metric.NewGauge(metric.Metadata{Name: reg.prefix + data.name}) reg.reg.AddMetric(g) g.Update(data.val) addExpected(reg.prefix, data.name, reg.source, 100, data.val, reg.isNode) case "floatgauge": g := metric.NewGaugeFloat64(metric.Metadata{Name: reg.prefix + data.name}) reg.reg.AddMetric(g) g.Update(float64(data.val)) addExpected(reg.prefix, data.name, reg.source, 100, data.val, reg.isNode) case "counter": c := metric.NewCounter(metric.Metadata{Name: reg.prefix + data.name}) reg.reg.AddMetric(c) c.Inc((data.val)) addExpected(reg.prefix, data.name, reg.source, 100, data.val, reg.isNode) case "counterwithrates": r := metric.NewCounterWithRates(metric.Metadata{Name: reg.prefix + data.name}) reg.reg.AddMetric(r) r.Inc(data.val) addExpected(reg.prefix, data.name, reg.source, 100, data.val, reg.isNode) case "histogram": h := metric.NewHistogram(metric.Metadata{Name: reg.prefix + data.name}, time.Second, 1000, 2) reg.reg.AddMetric(h) h.RecordValue(data.val) for _, q := range recordHistogramQuantiles { addExpected(reg.prefix, data.name+q.suffix, reg.source, 100, data.val, reg.isNode) } case "latency": l := metric.NewLatency(metric.Metadata{Name: reg.prefix + data.name}, time.Hour) reg.reg.AddMetric(l) l.RecordValue(data.val) // Latency is simply three histograms (at different resolution // time scales). for _, q := range recordHistogramQuantiles { addExpected(reg.prefix, data.name+q.suffix, reg.source, 100, data.val, reg.isNode) } default: t.Fatalf("unexpected: %+v", data) } } } // ======================================== // Verify time series data // ======================================== actual := recorder.GetTimeSeriesData() // Actual comparison is simple: sort the resulting arrays by time and name, // and use reflect.DeepEqual. sort.Sort(byTimeAndName(actual)) sort.Sort(byTimeAndName(expected)) if a, e := actual, expected; !reflect.DeepEqual(a, e) { t.Errorf("recorder did not yield expected time series collection; diff:\n %v", pretty.Diff(e, a)) } // ======================================== // Verify node summary generation // ======================================== expectedNodeSummary := &NodeStatus{ Desc: nodeDesc, BuildInfo: build.GetInfo(), StartedAt: 50, UpdatedAt: 100, Metrics: expectedNodeSummaryMetrics, StoreStatuses: []StoreStatus{ { Desc: storeDesc1, Metrics: expectedStoreSummaryMetrics, }, { Desc: storeDesc2, Metrics: expectedStoreSummaryMetrics, }, }, } nodeSummary := recorder.GetStatusSummary() if nodeSummary == nil { t.Fatalf("recorder did not return nodeSummary") } sort.Sort(byStoreDescID(nodeSummary.StoreStatuses)) if a, e := nodeSummary, expectedNodeSummary; !reflect.DeepEqual(a, e) { t.Errorf("recorder did not produce expected NodeSummary; diff:\n %v", pretty.Diff(e, a)) } }