// TestClientDisconnectRedundant verifies that the gossip server
// will drop an outgoing client connection that is already an
// inbound client connection of another node.
func TestClientDisconnectRedundant(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t, metric.NewRegistry())
remote := startGossip(2, stopper, t, metric.NewRegistry())
// startClient requires locks are held, so acquire here.
local.mu.Lock()
remote.mu.Lock()
rAddr := remote.mu.is.NodeAddr
lAddr := local.mu.is.NodeAddr
local.startClient(&rAddr, remote.NodeID.Get())
remote.startClient(&lAddr, local.NodeID.Get())
local.mu.Unlock()
remote.mu.Unlock()
local.manage()
remote.manage()
util.SucceedsSoon(t, func() error {
// Check which of the clients is connected to the other.
ok1 := local.findClient(func(c *client) bool { return c.addr.String() == rAddr.String() }) != nil
ok2 := remote.findClient(func(c *client) bool { return c.addr.String() == lAddr.String() }) != nil
// We expect node 2 to disconnect; if both are still connected,
// it's possible that node 1 gossiped before node 2 connected, in
// which case we have to gossip from node 1 to trigger the
// disconnect redundant client code.
if ok1 && ok2 {
if err := local.AddInfo("local-key", nil, time.Second); err != nil {
t.Fatal(err)
}
} else if ok1 && !ok2 && verifyServerMaps(local, 0) && verifyServerMaps(remote, 1) {
return nil
}
return errors.New("local client to remote not yet closed as redundant")
})
}
func TestGossipRaceLogStatus(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t, metric.NewRegistry())
local.mu.Lock()
peer := startGossip(2, stopper, t, metric.NewRegistry())
local.startClient(&peer.mu.is.NodeAddr)
local.mu.Unlock()
// Race gossiping against LogStatus.
gun := make(chan struct{})
for i := uint8(0); i < 10; i++ {
go func() {
<-gun
local.LogStatus()
gun <- struct{}{}
}()
gun <- struct{}{}
if err := local.AddInfo(
strconv.FormatUint(uint64(i), 10),
[]byte{i},
time.Hour,
); err != nil {
t.Fatal(err)
}
<-gun
}
close(gun)
}
// TestClientGossip verifies a client can gossip a delta to the server.
func TestClientGossip(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
local := startGossip(1, stopper, t, metric.NewRegistry())
remote := startGossip(2, stopper, t, metric.NewRegistry())
disconnected := make(chan *client, 1)
c := newClient(log.AmbientContext{}, remote.GetNodeAddr(), makeMetrics())
defer func() {
stopper.Stop()
if c != <-disconnected {
t.Errorf("expected client disconnect after remote close")
}
}()
if err := local.AddInfo("local-key", nil, time.Hour); err != nil {
t.Fatal(err)
}
if err := remote.AddInfo("remote-key", nil, time.Hour); err != nil {
t.Fatal(err)
}
gossipSucceedsSoon(t, stopper, disconnected, map[*client]*Gossip{
c: local,
}, func() error {
if _, err := remote.GetInfo("local-key"); err != nil {
return err
}
if _, err := local.GetInfo("remote-key"); err != nil {
return err
}
return nil
})
}
// createTestNode creates an rpc server using the specified address,
// gossip instance, KV database and a node using the specified slice
// of engines. The server, clock and node are returned. If gossipBS is
// not nil, the gossip bootstrap address is set to gossipBS.
func createTestNode(
addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T,
) (*grpc.Server, net.Addr, *hlc.Clock, *Node, *stop.Stopper) {
cfg := storage.StoreConfig{}
stopper := stop.NewStopper()
cfg.Clock = hlc.NewClock(hlc.UnixNano)
nodeRPCContext := rpc.NewContext(log.AmbientContext{}, nodeTestBaseContext, cfg.Clock, stopper)
cfg.ScanInterval = 10 * time.Hour
cfg.ConsistencyCheckInterval = 10 * time.Hour
grpcServer := rpc.NewServer(nodeRPCContext)
serverCfg := makeTestConfig()
cfg.Gossip = gossip.NewTest(
0,
nodeRPCContext,
grpcServer,
serverCfg.GossipBootstrapResolvers,
stopper,
metric.NewRegistry(),
)
ln, err := netutil.ListenAndServeGRPC(stopper, grpcServer, addr)
if err != nil {
t.Fatal(err)
}
if gossipBS != nil {
// Handle possibility of a :0 port specification.
if gossipBS.Network() == addr.Network() && gossipBS.String() == addr.String() {
gossipBS = ln.Addr()
}
r, err := resolver.NewResolverFromAddress(gossipBS)
if err != nil {
t.Fatalf("bad gossip address %s: %s", gossipBS, err)
}
cfg.Gossip.SetResolvers([]resolver.Resolver{r})
cfg.Gossip.Start(ln.Addr())
}
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = stopper.ShouldQuiesce()
distSender := kv.NewDistSender(kv.DistSenderConfig{
Clock: cfg.Clock,
RPCContext: nodeRPCContext,
RPCRetryOptions: &retryOpts,
}, cfg.Gossip)
cfg.AmbientCtx.Tracer = tracing.NewTracer()
sender := kv.NewTxnCoordSender(
cfg.AmbientCtx,
distSender,
cfg.Clock,
false,
stopper,
kv.MakeTxnMetrics(metric.TestSampleInterval),
)
cfg.DB = client.NewDB(sender)
cfg.Transport = storage.NewDummyRaftTransport()
cfg.MetricsSampleInterval = metric.TestSampleInterval
node := NewNode(cfg, status.NewMetricsRecorder(cfg.Clock), metric.NewRegistry(), stopper,
kv.MakeTxnMetrics(metric.TestSampleInterval), sql.MakeEventLogger(nil))
roachpb.RegisterInternalServer(grpcServer, node)
return grpcServer, ln.Addr(), cfg.Clock, node, stopper
}
// TestClientDisallowMultipleConns verifies that the server disallows
// multiple connections from the same client node ID.
func TestClientDisallowMultipleConns(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t, metric.NewRegistry())
remote := startGossip(2, stopper, t, metric.NewRegistry())
local.mu.Lock()
remote.mu.Lock()
rAddr := remote.mu.is.NodeAddr
// Start two clients from local to remote. RPC client cache is
// disabled via the context, so we'll start two different outgoing
// connections.
local.startClient(&rAddr, remote.NodeID.Get())
local.startClient(&rAddr, remote.NodeID.Get())
local.mu.Unlock()
remote.mu.Unlock()
local.manage()
remote.manage()
util.SucceedsSoon(t, func() error {
// Verify that the remote server has only a single incoming
// connection and the local server has only a single outgoing
// connection.
local.mu.Lock()
remote.mu.Lock()
outgoing := local.outgoing.len()
incoming := remote.mu.incoming.len()
local.mu.Unlock()
remote.mu.Unlock()
if outgoing == 1 && incoming == 1 && verifyServerMaps(local, 0) && verifyServerMaps(remote, 1) {
return nil
}
return errors.Errorf("incorrect number of incoming (%d) or outgoing (%d) connections", incoming, outgoing)
})
}
// TestClientGossipMetrics verifies a that gossip stats are generated.
func TestClientGossipMetrics(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t, metric.NewRegistry())
remote := startGossip(2, stopper, t, metric.NewRegistry())
if err := local.AddInfo("local-key", nil, time.Hour); err != nil {
t.Fatal(err)
}
if err := remote.AddInfo("remote-key", nil, time.Hour); err != nil {
t.Fatal(err)
}
gossipSucceedsSoon(
t, stopper, make(chan *client, 2),
map[*client]*Gossip{
newClient(log.AmbientContext{}, local.GetNodeAddr(), remote.nodeMetrics): remote,
},
func() error {
// Infos/Bytes Sent/Received should not be zero.
for i, s := range []*server{local.server, remote.server} {
for _, counter := range []*metric.Counter{
s.nodeMetrics.InfosSent,
s.nodeMetrics.InfosReceived,
s.nodeMetrics.BytesSent,
s.nodeMetrics.BytesReceived,
} {
if count := counter.Count(); count <= 0 {
return errors.Errorf("%d: expected metrics counter %q > 0; = %d", i, counter.GetName(), count)
}
}
}
// Since there are two gossip nodes, there should be exactly one incoming
// or outgoing connection due to gossip's connection de-duplication.
for i, g := range []*Gossip{local, remote} {
g.mu.Lock()
defer g.mu.Unlock()
count := int64(0)
for _, gauge := range []*metric.Gauge{g.mu.incoming.gauge, g.outgoing.gauge} {
if gauge == nil {
return errors.Errorf("%d: missing gauge", i)
}
count += gauge.Value()
}
const expected = 1
if count != expected {
return errors.Errorf("%d: expected metrics incoming + outgoing connection count == %d; = %d", i, expected, count)
}
}
return nil
})
}
// startFakeServerGossips creates local gossip instances and remote
// faked gossip instance. The remote gossip instance launches its
// faked gossip service just for check the client message.
func startFakeServerGossips(
t *testing.T, localNodeID roachpb.NodeID,
) (*Gossip, *fakeGossipServer, *stop.Stopper) {
stopper := stop.NewStopper()
lRPCContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper)
lserver := rpc.NewServer(lRPCContext)
local := NewTest(localNodeID, lRPCContext, lserver, nil, stopper, metric.NewRegistry())
lln, err := netutil.ListenAndServeGRPC(stopper, lserver, util.IsolatedTestAddr)
if err != nil {
t.Fatal(err)
}
local.start(lln.Addr())
rRPCContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper)
rserver := rpc.NewServer(rRPCContext)
rln, err := netutil.ListenAndServeGRPC(stopper, rserver, util.IsolatedTestAddr)
if err != nil {
t.Fatal(err)
}
remote := newFakeGossipServer(rserver, stopper)
addr := rln.Addr()
remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String())
return local, remote, stopper
}
// TestClientForwardUnresolved verifies that a client does not resolve a forward
// address prematurely.
func TestClientForwardUnresolved(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
const nodeID = 1
local := startGossip(nodeID, stopper, t, metric.NewRegistry())
addr := local.GetNodeAddr()
client := newClient(log.AmbientContext{}, addr, makeMetrics()) // never started
newAddr := util.UnresolvedAddr{
NetworkField: "tcp",
AddressField: "localhost:2345",
}
reply := &Response{
NodeID: nodeID,
Addr: *addr,
AlternateNodeID: nodeID + 1,
AlternateAddr: &newAddr,
}
if err := client.handleResponse(
context.TODO(), local, reply,
); !testutils.IsError(err, "received forward") {
t.Fatal(err)
}
if !proto.Equal(client.forwardAddr, &newAddr) {
t.Fatalf("unexpected forward address %v, expected %v", client.forwardAddr, &newAddr)
}
}
// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester, baseCtx *base.Config, initSender InitSenderFn) {
ambient := log.AmbientContext{Tracer: tracing.NewTracer()}
nc := &base.NodeIDContainer{}
ambient.AddLogTag("n", nc)
nodeID := roachpb.NodeID(1)
nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID}
ltc.tester = t
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = stop.NewStopper()
rpcContext := rpc.NewContext(ambient, baseCtx, ltc.Clock, ltc.Stopper)
server := rpc.NewServer(rpcContext) // never started
ltc.Gossip = gossip.New(ambient, nc, rpcContext, server, nil, ltc.Stopper, metric.NewRegistry())
ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20)
ltc.Stopper.AddCloser(ltc.Eng)
ltc.Stores = storage.NewStores(ambient, ltc.Clock)
ltc.Sender = initSender(nodeDesc, ambient.Tracer, ltc.Clock, ltc.Latency, ltc.Stores, ltc.Stopper,
ltc.Gossip)
if ltc.DBContext == nil {
dbCtx := client.DefaultDBContext()
ltc.DBContext = &dbCtx
}
ltc.DB = client.NewDBWithContext(ltc.Sender, *ltc.DBContext)
transport := storage.NewDummyRaftTransport()
cfg := storage.TestStoreConfig()
if ltc.RangeRetryOptions != nil {
cfg.RangeRetryOptions = *ltc.RangeRetryOptions
}
cfg.AmbientCtx = ambient
cfg.Clock = ltc.Clock
cfg.DB = ltc.DB
cfg.Gossip = ltc.Gossip
cfg.Transport = transport
cfg.MetricsSampleInterval = metric.TestSampleInterval
ltc.Store = storage.NewStore(cfg, ltc.Eng, nodeDesc)
if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.Stores.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(nil); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(context.Background(), ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
nc.Set(context.TODO(), nodeDesc.NodeID)
if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil {
t.Fatalf("unable to set node descriptor: %s", err)
}
}
// TestGossipCullNetwork verifies that a client will be culled from
// the network periodically (at cullInterval duration intervals).
func TestGossipCullNetwork(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t, metric.NewRegistry())
local.SetCullInterval(5 * time.Millisecond)
local.mu.Lock()
for i := 0; i < minPeers; i++ {
peer := startGossip(roachpb.NodeID(i+2), stopper, t, metric.NewRegistry())
local.startClient(peer.GetNodeAddr())
}
local.mu.Unlock()
const slowGossipDuration = time.Minute
if err := util.RetryForDuration(slowGossipDuration, func() error {
if peers := len(local.Outgoing()); peers != minPeers {
return errors.Errorf("%d of %d peers connected", peers, minPeers)
}
return nil
}); err != nil {
t.Fatalf("condition failed to evaluate within %s: %s", slowGossipDuration, err)
}
local.manage()
if err := util.RetryForDuration(slowGossipDuration, func() error {
// Verify that a client is closed within the cull interval.
if peers := len(local.Outgoing()); peers != minPeers-1 {
return errors.Errorf("%d of %d peers connected", peers, minPeers-1)
}
return nil
}); err != nil {
t.Fatalf("condition failed to evaluate within %s: %s", slowGossipDuration, err)
}
}
// TestClientRetryBootstrap verifies that an initial failure to connect
// to a bootstrap host doesn't stall the bootstrapping process in the
// absence of any additional activity. This can happen during acceptance
// tests if the DNS can't lookup hostnames when gossip is started.
func TestClientRetryBootstrap(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t, metric.NewRegistry())
remote := startGossip(2, stopper, t, metric.NewRegistry())
if err := local.AddInfo("local-key", []byte("hello"), 0*time.Second); err != nil {
t.Fatal(err)
}
local.SetBootstrapInterval(10 * time.Millisecond)
local.SetResolvers([]resolver.Resolver{
&testResolver{addr: remote.GetNodeAddr().String(), numFails: 3, numSuccesses: 1},
})
local.bootstrap()
local.manage()
util.SucceedsSoon(t, func() error {
_, err := remote.GetInfo("local-key")
return err
})
}
func newRaftTransportTestContext(t testing.TB) *raftTransportTestContext {
rttc := &raftTransportTestContext{
t: t,
stopper: stop.NewStopper(),
transports: map[roachpb.NodeID]*storage.RaftTransport{},
}
rttc.nodeRPCContext = rpc.NewContext(
log.AmbientContext{}, testutils.NewNodeTestBaseContext(), nil, rttc.stopper,
)
server := rpc.NewServer(rttc.nodeRPCContext) // never started
rttc.gossip = gossip.NewTest(
1, rttc.nodeRPCContext, server, nil, rttc.stopper, metric.NewRegistry(),
)
return rttc
}
// TestClientRegisterInitNodeID verifies two client's gossip request with NodeID 0.
func TestClientRegisterWithInitNodeID(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
// Create three gossip nodes, and connect to the first with NodeID 0.
var g []*Gossip
var gossipAddr string
for i := 0; i < 3; i++ {
RPCContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper)
server := rpc.NewServer(RPCContext)
ln, err := netutil.ListenAndServeGRPC(stopper, server, util.IsolatedTestAddr)
if err != nil {
t.Fatal(err)
}
// Connect to the first gossip node.
if gossipAddr == "" {
gossipAddr = ln.Addr().String()
}
var resolvers []resolver.Resolver
resolver, err := resolver.NewResolver(gossipAddr)
if err != nil {
t.Fatal(err)
}
resolvers = append(resolvers, resolver)
// node ID must be non-zero
gnode := NewTest(
roachpb.NodeID(i+1), RPCContext, server, resolvers, stopper, metric.NewRegistry(),
)
g = append(g, gnode)
gnode.Start(ln.Addr())
}
util.SucceedsSoon(t, func() error {
// The first gossip node should have two gossip client address
// in nodeMap if these three gossip nodes registered success.
g[0].mu.Lock()
defer g[0].mu.Unlock()
if a, e := len(g[0].mu.nodeMap), 2; a != e {
return errors.Errorf("expected %s to contain %d nodes, got %d", g[0].mu.nodeMap, e, a)
}
return nil
})
}
// MustGetSQLNetworkCounter implements TestServerInterface.
func (ts *TestServer) MustGetSQLNetworkCounter(name string) int64 {
var c int64
var found bool
reg := metric.NewRegistry()
reg.AddMetricStruct(ts.pgServer.Metrics())
reg.Each(func(n string, v interface{}) {
if name == n {
c = v.(*metric.Counter).Count()
found = true
}
})
if !found {
panic(fmt.Sprintf("couldn't find metric %s", name))
}
return c
}
// TestGossipInfoStore verifies operation of gossip instance infostore.
func TestGossipInfoStore(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
rpcContext := newInsecureRPCContext(stopper)
g := NewTest(1, rpcContext, rpc.NewServer(rpcContext), nil, stopper, metric.NewRegistry())
slice := []byte("b")
if err := g.AddInfo("s", slice, time.Hour); err != nil {
t.Fatal(err)
}
if val, err := g.GetInfo("s"); !bytes.Equal(val, slice) || err != nil {
t.Errorf("error fetching string: %v", err)
}
if _, err := g.GetInfo("s2"); err == nil {
t.Errorf("expected error fetching nonexistent key \"s2\"")
}
}
// TestGossipOverwriteNode verifies that if a new node is added with the same
// address as an old node, that old node is removed from the cluster.
func TestGossipOverwriteNode(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
rpcContext := newInsecureRPCContext(stopper)
g := NewTest(1, rpcContext, rpc.NewServer(rpcContext), nil, stopper, metric.NewRegistry())
node1 := &roachpb.NodeDescriptor{NodeID: 1, Address: util.MakeUnresolvedAddr("tcp", "1.1.1.1:1")}
node2 := &roachpb.NodeDescriptor{NodeID: 2, Address: util.MakeUnresolvedAddr("tcp", "2.2.2.2:2")}
if err := g.SetNodeDescriptor(node1); err != nil {
t.Fatal(err)
}
if err := g.SetNodeDescriptor(node2); err != nil {
t.Fatal(err)
}
if val, err := g.GetNodeDescriptor(node1.NodeID); err != nil {
t.Error(err)
} else if val.NodeID != node1.NodeID {
t.Errorf("expected node %d, got %+v", node1.NodeID, val)
}
if val, err := g.GetNodeDescriptor(node2.NodeID); err != nil {
t.Error(err)
} else if val.NodeID != node2.NodeID {
t.Errorf("expected node %d, got %+v", node2.NodeID, val)
}
// Give node3 the same address as node1, which should cause node1 to be
// removed from the cluster.
node3 := &roachpb.NodeDescriptor{NodeID: 3, Address: node1.Address}
if err := g.SetNodeDescriptor(node3); err != nil {
t.Fatal(err)
}
if val, err := g.GetNodeDescriptor(node3.NodeID); err != nil {
t.Error(err)
} else if val.NodeID != node3.NodeID {
t.Errorf("expected node %d, got %+v", node3.NodeID, val)
}
// Quiesce the stopper now to ensure that the update has propagated before
// checking whether node 1 has been removed from the infoStore.
stopper.Quiesce()
expectedErr := "unable to look up descriptor for node"
if val, err := g.GetNodeDescriptor(node1.NodeID); !testutils.IsError(err, expectedErr) {
t.Errorf("expected error %q fetching node %d; got error %v and node %+v",
expectedErr, node1.NodeID, err, val)
}
}
// CreateNode creates a simulation node and starts an RPC server for it.
func (n *Network) CreateNode() (*Node, error) {
server := rpc.NewServer(n.rpcContext)
ln, err := net.Listen(util.TestAddr.Network(), util.TestAddr.String())
if err != nil {
return nil, err
}
node := &Node{Server: server, Listener: ln, Registry: metric.NewRegistry()}
node.Gossip = gossip.NewTest(0, n.rpcContext, server, nil, n.Stopper, node.Registry)
n.Stopper.RunWorker(func() {
<-n.Stopper.ShouldQuiesce()
netutil.FatalIfUnexpected(ln.Close())
<-n.Stopper.ShouldStop()
server.Stop()
node.Gossip.EnableSimulationCycler(false)
})
n.Nodes = append(n.Nodes, node)
return node, nil
}
// createTestStorePool creates a stopper, gossip and storePool for use in
// tests. Stopper must be stopped by the caller.
func createTestStorePool(
timeUntilStoreDead time.Duration,
) (*stop.Stopper, *gossip.Gossip, *hlc.ManualClock, *StorePool) {
stopper := stop.NewStopper()
mc := hlc.NewManualClock(0)
clock := hlc.NewClock(mc.UnixNano)
rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, clock, stopper)
server := rpc.NewServer(rpcContext) // never started
g := gossip.NewTest(1, rpcContext, server, nil, stopper, metric.NewRegistry())
storePool := NewStorePool(
log.AmbientContext{},
g,
clock,
rpcContext,
timeUntilStoreDead,
stopper,
)
return stopper, g, mc, storePool
}
// TestClientDisconnectLoopback verifies that the gossip server
// will drop an outgoing client connection that is already an
// inbound client connection of another node.
func TestClientDisconnectLoopback(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t, metric.NewRegistry())
// startClient requires locks are held, so acquire here.
local.mu.Lock()
lAddr := local.mu.is.NodeAddr
local.startClient(&lAddr, local.NodeID.Get())
local.mu.Unlock()
local.manage()
util.SucceedsSoon(t, func() error {
ok := local.findClient(func(c *client) bool { return c.addr.String() == lAddr.String() }) != nil
if !ok && verifyServerMaps(local, 0) {
return nil
}
return errors.New("local client still connected to itself")
})
}
// createTestStorePool creates a stopper, gossip and storePool for use in
// tests. Stopper must be stopped by the caller.
func createTestStorePool(
timeUntilStoreDead time.Duration, deterministic bool, defaultNodeLiveness bool,
) (*stop.Stopper, *gossip.Gossip, *hlc.ManualClock, *StorePool, *mockNodeLiveness) {
stopper := stop.NewStopper()
mc := hlc.NewManualClock(123)
clock := hlc.NewClock(mc.UnixNano, time.Nanosecond)
rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, clock, stopper)
server := rpc.NewServer(rpcContext) // never started
g := gossip.NewTest(1, rpcContext, server, nil, stopper, metric.NewRegistry())
mnl := newMockNodeLiveness(defaultNodeLiveness)
storePool := NewStorePool(
log.AmbientContext{},
g,
clock,
mnl.nodeLivenessFunc,
timeUntilStoreDead,
deterministic,
)
return stopper, g, mc, storePool, mnl
}
func TestGossipGetNextBootstrapAddress(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
resolverSpecs := []string{
"127.0.0.1:9000",
"127.0.0.1:9001",
"localhost:9004",
}
resolvers := []resolver.Resolver{}
for _, rs := range resolverSpecs {
resolver, err := resolver.NewResolver(rs)
if err == nil {
resolvers = append(resolvers, resolver)
}
}
if len(resolvers) != 3 {
t.Errorf("expected 3 resolvers; got %d", len(resolvers))
}
server := rpc.NewServer(
rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper),
)
g := NewTest(0, nil, server, resolvers, stop.NewStopper(), metric.NewRegistry())
// Using specified resolvers, fetch bootstrap addresses 3 times
// and verify the results match expected addresses.
expAddresses := []string{
"127.0.0.1:9000",
"127.0.0.1:9001",
"localhost:9004",
}
for i := 0; i < len(expAddresses); i++ {
if addr := g.getNextBootstrapAddress(); addr == nil {
t.Errorf("%d: unexpected nil addr when expecting %s", i, expAddresses[i])
} else if addrStr := addr.String(); addrStr != expAddresses[i] {
t.Errorf("%d: expected addr %s; got %s", i, expAddresses[i], addrStr)
}
}
}
// createCluster generates a new cluster using the provided stopper and the
// number of nodes supplied. Each node will have one store to start.
func createCluster(
stopper *stop.Stopper,
nodeCount int,
epochWriter, actionWriter io.Writer,
script Script,
rand *rand.Rand,
) *Cluster {
clock := hlc.NewClock(hlc.UnixNano, time.Nanosecond)
rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, clock, stopper)
server := rpc.NewServer(rpcContext)
// We set the node ID to MaxInt32 for the cluster Gossip instance to prevent
// conflicts with real node IDs.
g := gossip.NewTest(math.MaxInt32, rpcContext, server, nil, stopper, metric.NewRegistry())
// Set the store pool to deterministic so that a run with the exact same
// input will always produce the same output.
storePool := storage.NewStorePool(
log.AmbientContext{},
g,
clock,
rpcContext,
storage.TestTimeUntilStoreDeadOff,
stopper,
/* deterministic */ true,
)
c := &Cluster{
stopper: stopper,
clock: clock,
rpc: rpcContext,
gossip: g,
storePool: storePool,
allocator: storage.MakeAllocator(storePool, storage.AllocatorOptions{
AllowRebalance: true,
}),
storeGossiper: gossiputil.NewStoreGossiper(g),
nodes: make(map[roachpb.NodeID]*Node),
stores: make(map[roachpb.StoreID]*Store),
ranges: make(map[roachpb.RangeID]*Range),
rangeIDsByStore: make(map[roachpb.StoreID]roachpb.RangeIDSlice),
rand: rand,
epochWriter: tabwriter.NewWriter(epochWriter, 8, 1, 2, ' ', 0),
actionWriter: tabwriter.NewWriter(actionWriter, 8, 1, 2, ' ', 0),
script: script,
epoch: -1,
}
// Add the nodes.
for i := 0; i < nodeCount; i++ {
c.addNewNodeWithStore()
}
// Add a single range and add to this first node's first store.
firstRange := c.addRange()
firstRange.addReplica(c.stores[0])
c.calculateRangeIDsByStore()
// Output the first epoch header.
c.epoch = 0
c.OutputEpochHeader()
c.OutputEpoch()
c.flush()
return c
}
// NewServer creates a Server from a server.Context.
func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) {
if _, err := net.ResolveTCPAddr("tcp", cfg.AdvertiseAddr); err != nil {
return nil, errors.Errorf("unable to resolve RPC address %q: %v", cfg.AdvertiseAddr, err)
}
if cfg.AmbientCtx.Tracer == nil {
cfg.AmbientCtx.Tracer = tracing.NewTracer()
}
// Try loading the TLS configs before anything else.
if _, err := cfg.GetServerTLSConfig(); err != nil {
return nil, err
}
if _, err := cfg.GetClientTLSConfig(); err != nil {
return nil, err
}
s := &Server{
mux: http.NewServeMux(),
clock: hlc.NewClock(hlc.UnixNano, cfg.MaxOffset),
stopper: stopper,
cfg: cfg,
}
// Add a dynamic log tag value for the node ID.
//
// We need to pass an ambient context to the various server components, but we
// won't know the node ID until we Start(). At that point it's too late to
// change the ambient contexts in the components (various background processes
// will have already started using them).
//
// NodeIDContainer allows us to add the log tag to the context now and update
// the value asynchronously. It's not significantly more expensive than a
// regular tag since it's just doing an (atomic) load when a log/trace message
// is constructed. The node ID is set by the Store if this host was
// bootstrapped; otherwise a new one is allocated in Node.
s.cfg.AmbientCtx.AddLogTag("n", &s.nodeIDContainer)
ctx := s.AnnotateCtx(context.Background())
if s.cfg.Insecure {
log.Warning(ctx, "running in insecure mode, this is strongly discouraged. See --insecure.")
}
s.rpcContext = rpc.NewContext(s.cfg.AmbientCtx, s.cfg.Config, s.clock, s.stopper)
s.rpcContext.HeartbeatCB = func() {
if err := s.rpcContext.RemoteClocks.VerifyClockOffset(); err != nil {
log.Fatal(ctx, err)
}
}
s.grpc = rpc.NewServer(s.rpcContext)
s.registry = metric.NewRegistry()
s.gossip = gossip.New(
s.cfg.AmbientCtx,
&s.nodeIDContainer,
s.rpcContext,
s.grpc,
s.cfg.GossipBootstrapResolvers,
s.stopper,
s.registry,
)
s.storePool = storage.NewStorePool(
s.cfg.AmbientCtx,
s.gossip,
s.clock,
s.rpcContext,
s.cfg.TimeUntilStoreDead,
s.stopper,
/* deterministic */ false,
)
// A custom RetryOptions is created which uses stopper.ShouldQuiesce() as
// the Closer. This prevents infinite retry loops from occurring during
// graceful server shutdown
//
// Such a loop loop occurs with the DistSender attempts a connection to the
// local server during shutdown, and receives an internal server error (HTTP
// Code 5xx). This is the correct error for a server to return when it is
// shutting down, and is normally retryable in a cluster environment.
// However, on a single-node setup (such as a test), retries will never
// succeed because the only server has been shut down; thus, thus the
// DistSender needs to know that it should not retry in this situation.
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = s.stopper.ShouldQuiesce()
distSenderCfg := kv.DistSenderConfig{
AmbientCtx: s.cfg.AmbientCtx,
Clock: s.clock,
RPCContext: s.rpcContext,
RPCRetryOptions: &retryOpts,
}
s.distSender = kv.NewDistSender(distSenderCfg, s.gossip)
txnMetrics := kv.MakeTxnMetrics(s.cfg.MetricsSampleInterval)
s.registry.AddMetricStruct(txnMetrics)
s.txnCoordSender = kv.NewTxnCoordSender(
s.cfg.AmbientCtx,
s.distSender,
s.clock,
s.cfg.Linearizable,
s.stopper,
txnMetrics,
)
s.db = client.NewDB(s.txnCoordSender)
// Use the range lease expiration and renewal durations as the node
// liveness expiration and heartbeat interval.
active, renewal := storage.RangeLeaseDurations(
storage.RaftElectionTimeout(s.cfg.RaftTickInterval, s.cfg.RaftElectionTimeoutTicks))
s.nodeLiveness = storage.NewNodeLiveness(
s.cfg.AmbientCtx, s.clock, s.db, s.gossip, active, renewal,
)
s.registry.AddMetricStruct(s.nodeLiveness.Metrics())
s.raftTransport = storage.NewRaftTransport(
s.cfg.AmbientCtx, storage.GossipAddressResolver(s.gossip), s.grpc, s.rpcContext,
)
s.kvDB = kv.NewDBServer(s.cfg.Config, s.txnCoordSender, s.stopper)
roachpb.RegisterExternalServer(s.grpc, s.kvDB)
// Set up internal memory metrics for use by internal SQL executors.
s.internalMemMetrics = sql.MakeMemMetrics("internal")
s.registry.AddMetricStruct(s.internalMemMetrics)
// Set up Lease Manager
var lmKnobs sql.LeaseManagerTestingKnobs
if cfg.TestingKnobs.SQLLeaseManager != nil {
lmKnobs = *s.cfg.TestingKnobs.SQLLeaseManager.(*sql.LeaseManagerTestingKnobs)
}
s.leaseMgr = sql.NewLeaseManager(&s.nodeIDContainer, *s.db, s.clock, lmKnobs,
s.stopper, &s.internalMemMetrics)
s.leaseMgr.RefreshLeases(s.stopper, s.db, s.gossip)
// Set up the DistSQL server
distSQLCfg := distsql.ServerConfig{
AmbientContext: s.cfg.AmbientCtx,
DB: s.db,
RPCContext: s.rpcContext,
Stopper: s.stopper,
}
s.distSQLServer = distsql.NewServer(distSQLCfg)
distsql.RegisterDistSQLServer(s.grpc, s.distSQLServer)
// Set up admin memory metrics for use by admin SQL executors.
s.adminMemMetrics = sql.MakeMemMetrics("admin")
s.registry.AddMetricStruct(s.adminMemMetrics)
// Set up Executor
execCfg := sql.ExecutorConfig{
AmbientCtx: s.cfg.AmbientCtx,
NodeID: &s.nodeIDContainer,
DB: s.db,
Gossip: s.gossip,
LeaseManager: s.leaseMgr,
Clock: s.clock,
DistSQLSrv: s.distSQLServer,
MetricsSampleInterval: s.cfg.MetricsSampleInterval,
}
if s.cfg.TestingKnobs.SQLExecutor != nil {
execCfg.TestingKnobs = s.cfg.TestingKnobs.SQLExecutor.(*sql.ExecutorTestingKnobs)
} else {
execCfg.TestingKnobs = &sql.ExecutorTestingKnobs{}
}
if s.cfg.TestingKnobs.SQLSchemaChanger != nil {
execCfg.SchemaChangerTestingKnobs =
s.cfg.TestingKnobs.SQLSchemaChanger.(*sql.SchemaChangerTestingKnobs)
} else {
execCfg.SchemaChangerTestingKnobs = &sql.SchemaChangerTestingKnobs{}
}
s.sqlExecutor = sql.NewExecutor(execCfg, s.stopper, &s.adminMemMetrics)
s.registry.AddMetricStruct(s.sqlExecutor)
s.pgServer = pgwire.MakeServer(
s.cfg.AmbientCtx, s.cfg.Config, s.sqlExecutor, &s.internalMemMetrics, s.cfg.SQLMemoryPoolSize,
)
s.registry.AddMetricStruct(s.pgServer.Metrics())
s.tsDB = ts.NewDB(s.db)
s.tsServer = ts.MakeServer(s.cfg.AmbientCtx, s.tsDB, s.cfg.TimeSeriesServerConfig, s.stopper)
// TODO(bdarnell): make StoreConfig configurable.
storeCfg := storage.StoreConfig{
AmbientCtx: s.cfg.AmbientCtx,
Clock: s.clock,
DB: s.db,
Gossip: s.gossip,
NodeLiveness: s.nodeLiveness,
Transport: s.raftTransport,
RaftTickInterval: s.cfg.RaftTickInterval,
ScanInterval: s.cfg.ScanInterval,
ScanMaxIdleTime: s.cfg.ScanMaxIdleTime,
ConsistencyCheckInterval: s.cfg.ConsistencyCheckInterval,
ConsistencyCheckPanicOnFailure: s.cfg.ConsistencyCheckPanicOnFailure,
MetricsSampleInterval: s.cfg.MetricsSampleInterval,
StorePool: s.storePool,
SQLExecutor: sql.InternalExecutor{
LeaseManager: s.leaseMgr,
},
LogRangeEvents: s.cfg.EventLogEnabled,
AllocatorOptions: storage.AllocatorOptions{
AllowRebalance: true,
},
RangeLeaseActiveDuration: active,
RangeLeaseRenewalDuration: renewal,
TimeSeriesDataStore: s.tsDB,
}
if s.cfg.TestingKnobs.Store != nil {
storeCfg.TestingKnobs = *s.cfg.TestingKnobs.Store.(*storage.StoreTestingKnobs)
}
s.recorder = status.NewMetricsRecorder(s.clock)
s.registry.AddMetricStruct(s.rpcContext.RemoteClocks.Metrics())
s.runtime = status.MakeRuntimeStatSampler(s.clock)
s.registry.AddMetricStruct(s.runtime)
s.node = NewNode(storeCfg, s.recorder, s.registry, s.stopper, txnMetrics, sql.MakeEventLogger(s.leaseMgr))
roachpb.RegisterInternalServer(s.grpc, s.node)
storage.RegisterConsistencyServer(s.grpc, s.node.storesServer)
storage.RegisterFreezeServer(s.grpc, s.node.storesServer)
s.admin = newAdminServer(s)
s.status = newStatusServer(
s.cfg.AmbientCtx, s.db, s.gossip, s.recorder, s.rpcContext, s.node.stores,
)
for _, gw := range []grpcGatewayServer{s.admin, s.status, &s.tsServer} {
gw.RegisterService(s.grpc)
}
return s, nil
}
func TestGossipOrphanedStallDetection(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t, metric.NewRegistry())
local.SetStallInterval(5 * time.Millisecond)
// Make sure we have the sentinel to ensure that its absence is not the
// cause of stall detection.
if err := local.AddInfo(KeySentinel, nil, time.Hour); err != nil {
t.Fatal(err)
}
peerStopper := stop.NewStopper()
peer := startGossip(2, peerStopper, t, metric.NewRegistry())
peerNodeID := peer.NodeID.Get()
peerAddr := peer.GetNodeAddr()
peerAddrStr := peerAddr.String()
local.startClient(peerAddr)
util.SucceedsSoon(t, func() error {
for _, peerID := range local.Outgoing() {
if peerID == peerNodeID {
return nil
}
}
return errors.Errorf("node %d not yet connected", peerNodeID)
})
util.SucceedsSoon(t, func() error {
for _, resolver := range local.GetResolvers() {
if resolver.Addr() == peerAddrStr {
return nil
}
}
return errors.Errorf("node %d descriptor not yet available", peerNodeID)
})
local.bootstrap()
local.manage()
peerStopper.Stop()
util.SucceedsSoon(t, func() error {
for _, peerID := range local.Outgoing() {
if peerID == peerNodeID {
return errors.Errorf("node %d still connected", peerNodeID)
}
}
return nil
})
peerStopper = stop.NewStopper()
defer peerStopper.Stop()
startGossipAtAddr(peerNodeID, peerAddr, peerStopper, t, metric.NewRegistry())
util.SucceedsSoon(t, func() error {
for _, peerID := range local.Outgoing() {
if peerID == peerNodeID {
return nil
}
}
return errors.Errorf("node %d not yet connected", peerNodeID)
})
}
func newStoreMetrics(sampleInterval time.Duration) *StoreMetrics {
storeRegistry := metric.NewRegistry()
sm := &StoreMetrics{
registry: storeRegistry,
// Replica metrics.
ReplicaCount: metric.NewCounter(metaReplicaCount),
ReservedReplicaCount: metric.NewCounter(metaReservedReplicaCount),
RaftLeaderCount: metric.NewGauge(metaRaftLeaderCount),
RaftLeaderNotLeaseHolderCount: metric.NewGauge(metaRaftLeaderNotLeaseHolderCount),
LeaseHolderCount: metric.NewGauge(metaLeaseHolderCount),
QuiescentCount: metric.NewGauge(metaQuiescentCount),
// Replica CommandQueue metrics.
MaxCommandQueueSize: metric.NewGauge(metaMaxCommandQueueSize),
MaxCommandQueueWriteCount: metric.NewGauge(metaMaxCommandQueueWriteCount),
MaxCommandQueueReadCount: metric.NewGauge(metaMaxCommandQueueReadCount),
MaxCommandQueueTreeSize: metric.NewGauge(metaMaxCommandQueueTreeSize),
MaxCommandQueueOverlaps: metric.NewGauge(metaMaxCommandQueueOverlaps),
CombinedCommandQueueSize: metric.NewGauge(metaCombinedCommandQueueSize),
CombinedCommandWriteCount: metric.NewGauge(metaCombinedCommandWriteCount),
CombinedCommandReadCount: metric.NewGauge(metaCombinedCommandReadCount),
// Range metrics.
RangeCount: metric.NewGauge(metaRangeCount),
UnavailableRangeCount: metric.NewGauge(metaUnavailableRangeCount),
UnderReplicatedRangeCount: metric.NewGauge(metaUnderReplicatedRangeCount),
// Lease request metrics.
LeaseRequestSuccessCount: metric.NewCounter(metaLeaseRequestSuccessCount),
LeaseRequestErrorCount: metric.NewCounter(metaLeaseRequestErrorCount),
// Storage metrics.
LiveBytes: metric.NewGauge(metaLiveBytes),
KeyBytes: metric.NewGauge(metaKeyBytes),
ValBytes: metric.NewGauge(metaValBytes),
IntentBytes: metric.NewGauge(metaIntentBytes),
LiveCount: metric.NewGauge(metaLiveCount),
KeyCount: metric.NewGauge(metaKeyCount),
ValCount: metric.NewGauge(metaValCount),
IntentCount: metric.NewGauge(metaIntentCount),
IntentAge: metric.NewGauge(metaIntentAge),
GcBytesAge: metric.NewGauge(metaGcBytesAge),
LastUpdateNanos: metric.NewGauge(metaLastUpdateNanos),
Capacity: metric.NewGauge(metaCapacity),
Available: metric.NewGauge(metaAvailable),
Reserved: metric.NewCounter(metaReserved),
SysBytes: metric.NewGauge(metaSysBytes),
SysCount: metric.NewGauge(metaSysCount),
// RocksDB metrics.
RdbBlockCacheHits: metric.NewGauge(metaRdbBlockCacheHits),
RdbBlockCacheMisses: metric.NewGauge(metaRdbBlockCacheMisses),
RdbBlockCacheUsage: metric.NewGauge(metaRdbBlockCacheUsage),
RdbBlockCachePinnedUsage: metric.NewGauge(metaRdbBlockCachePinnedUsage),
RdbBloomFilterPrefixChecked: metric.NewGauge(metaRdbBloomFilterPrefixChecked),
RdbBloomFilterPrefixUseful: metric.NewGauge(metaRdbBloomFilterPrefixUseful),
RdbMemtableHits: metric.NewGauge(metaRdbMemtableHits),
RdbMemtableMisses: metric.NewGauge(metaRdbMemtableMisses),
RdbMemtableTotalSize: metric.NewGauge(metaRdbMemtableTotalSize),
RdbFlushes: metric.NewGauge(metaRdbFlushes),
RdbCompactions: metric.NewGauge(metaRdbCompactions),
RdbTableReadersMemEstimate: metric.NewGauge(metaRdbTableReadersMemEstimate),
RdbReadAmplification: metric.NewGauge(metaRdbReadAmplification),
RdbNumSSTables: metric.NewGauge(metaRdbNumSSTables),
// Range event metrics.
RangeSplits: metric.NewCounter(metaRangeSplits),
RangeAdds: metric.NewCounter(metaRangeAdds),
RangeRemoves: metric.NewCounter(metaRangeRemoves),
RangeSnapshotsGenerated: metric.NewCounter(metaRangeSnapshotsGenerated),
RangeSnapshotsNormalApplied: metric.NewCounter(metaRangeSnapshotsNormalApplied),
RangeSnapshotsPreemptiveApplied: metric.NewCounter(metaRangeSnapshotsPreemptiveApplied),
// Raft processing metrics.
RaftTicks: metric.NewCounter(metaRaftTicks),
RaftWorkingDurationNanos: metric.NewCounter(metaRaftWorkingDurationNanos),
RaftTickingDurationNanos: metric.NewCounter(metaRaftTickingDurationNanos),
// Raft message metrics.
RaftRcvdMsgProp: metric.NewCounter(metaRaftRcvdProp),
RaftRcvdMsgApp: metric.NewCounter(metaRaftRcvdApp),
RaftRcvdMsgAppResp: metric.NewCounter(metaRaftRcvdAppResp),
RaftRcvdMsgVote: metric.NewCounter(metaRaftRcvdVote),
RaftRcvdMsgVoteResp: metric.NewCounter(metaRaftRcvdVoteResp),
RaftRcvdMsgPreVote: metric.NewCounter(metaRaftRcvdPreVote),
RaftRcvdMsgPreVoteResp: metric.NewCounter(metaRaftRcvdPreVoteResp),
RaftRcvdMsgSnap: metric.NewCounter(metaRaftRcvdSnap),
RaftRcvdMsgHeartbeat: metric.NewCounter(metaRaftRcvdHeartbeat),
RaftRcvdMsgHeartbeatResp: metric.NewCounter(metaRaftRcvdHeartbeatResp),
RaftRcvdMsgTransferLeader: metric.NewCounter(metaRaftRcvdTransferLeader),
RaftRcvdMsgTimeoutNow: metric.NewCounter(metaRaftRcvdTimeoutNow),
RaftRcvdMsgDropped: metric.NewCounter(metaRaftRcvdDropped),
raftRcvdMessages: make(map[raftpb.MessageType]*metric.Counter, len(raftpb.MessageType_name)),
RaftEnqueuedPending: metric.NewGauge(metaRaftEnqueuedPending),
// This Gauge measures the number of heartbeats queued up just before
// the queue is cleared, to avoid flapping wildly.
RaftCoalescedHeartbeatsPending: metric.NewGauge(metaRaftCoalescedHeartbeatsPending),
// Replica queue metrics.
GCQueueSuccesses: metric.NewCounter(metaGCQueueSuccesses),
GCQueueFailures: metric.NewCounter(metaGCQueueFailures),
GCQueuePending: metric.NewGauge(metaGCQueuePending),
GCQueueProcessingNanos: metric.NewCounter(metaGCQueueProcessingNanos),
RaftLogQueueSuccesses: metric.NewCounter(metaRaftLogQueueSuccesses),
RaftLogQueueFailures: metric.NewCounter(metaRaftLogQueueFailures),
RaftLogQueuePending: metric.NewGauge(metaRaftLogQueuePending),
RaftLogQueueProcessingNanos: metric.NewCounter(metaRaftLogQueueProcessingNanos),
ConsistencyQueueSuccesses: metric.NewCounter(metaConsistencyQueueSuccesses),
ConsistencyQueueFailures: metric.NewCounter(metaConsistencyQueueFailures),
ConsistencyQueuePending: metric.NewGauge(metaConsistencyQueuePending),
ConsistencyQueueProcessingNanos: metric.NewCounter(metaConsistencyQueueProcessingNanos),
ReplicaGCQueueSuccesses: metric.NewCounter(metaReplicaGCQueueSuccesses),
ReplicaGCQueueFailures: metric.NewCounter(metaReplicaGCQueueFailures),
ReplicaGCQueuePending: metric.NewGauge(metaReplicaGCQueuePending),
ReplicaGCQueueProcessingNanos: metric.NewCounter(metaReplicaGCQueueProcessingNanos),
ReplicateQueueSuccesses: metric.NewCounter(metaReplicateQueueSuccesses),
ReplicateQueueFailures: metric.NewCounter(metaReplicateQueueFailures),
ReplicateQueuePending: metric.NewGauge(metaReplicateQueuePending),
ReplicateQueueProcessingNanos: metric.NewCounter(metaReplicateQueueProcessingNanos),
ReplicateQueuePurgatory: metric.NewGauge(metaReplicateQueuePurgatory),
SplitQueueSuccesses: metric.NewCounter(metaSplitQueueSuccesses),
SplitQueueFailures: metric.NewCounter(metaSplitQueueFailures),
SplitQueuePending: metric.NewGauge(metaSplitQueuePending),
SplitQueueProcessingNanos: metric.NewCounter(metaSplitQueueProcessingNanos),
TimeSeriesMaintenanceQueueSuccesses: metric.NewCounter(metaTimeSeriesMaintenanceQueueFailures),
TimeSeriesMaintenanceQueueFailures: metric.NewCounter(metaTimeSeriesMaintenanceQueueSuccesses),
TimeSeriesMaintenanceQueuePending: metric.NewGauge(metaTimeSeriesMaintenanceQueuePending),
TimeSeriesMaintenanceQueueProcessingNanos: metric.NewCounter(metaTimeSeriesMaintenanceQueueProcessingNanos),
// GCInfo cumulative totals.
GCNumKeysAffected: metric.NewCounter(metaGCNumKeysAffected),
GCIntentsConsidered: metric.NewCounter(metaGCIntentsConsidered),
GCIntentTxns: metric.NewCounter(metaGCIntentTxns),
GCTransactionSpanScanned: metric.NewCounter(metaGCTransactionSpanScanned),
GCTransactionSpanGCAborted: metric.NewCounter(metaGCTransactionSpanGCAborted),
GCTransactionSpanGCCommitted: metric.NewCounter(metaGCTransactionSpanGCCommitted),
GCTransactionSpanGCPending: metric.NewCounter(metaGCTransactionSpanGCPending),
GCAbortSpanScanned: metric.NewCounter(metaGCAbortSpanScanned),
GCAbortSpanConsidered: metric.NewCounter(metaGCAbortSpanConsidered),
GCAbortSpanGCNum: metric.NewCounter(metaGCAbortSpanGCNum),
GCPushTxn: metric.NewCounter(metaGCPushTxn),
GCResolveTotal: metric.NewCounter(metaGCResolveTotal),
GCResolveSuccess: metric.NewCounter(metaGCResolveSuccess),
// Mutex timing.
//
// TODO(tschottdorf): Histograms don't work very well as they were
// inherently built in a windowed (i.e. events-discarding) way, which
// is not at all the correct way. Discard at one-minute interval which
// gives sane (though mathematically nonsensical) results when exposed
// at the moment.
MuReplicaNanos: metric.NewHistogram(
metaMuReplicaNanos, sampleInterval,
time.Second.Nanoseconds(), 1,
),
MuCommandQueueNanos: metric.NewHistogram(
metaMuCommandQueueNanos, sampleInterval,
time.Second.Nanoseconds(), 1,
),
MuRaftNanos: metric.NewHistogram(
metaMuRaftNanos, sampleInterval,
time.Second.Nanoseconds(), 1,
),
MuStoreNanos: metric.NewHistogram(
metaMuStoreNanos, sampleInterval,
time.Second.Nanoseconds(), 1,
),
MuSchedulerNanos: metric.NewHistogram(
metaMuSchedulerNanos, time.Minute,
time.Second.Nanoseconds(), 1,
),
}
sm.raftRcvdMessages[raftpb.MsgProp] = sm.RaftRcvdMsgProp
sm.raftRcvdMessages[raftpb.MsgApp] = sm.RaftRcvdMsgApp
sm.raftRcvdMessages[raftpb.MsgAppResp] = sm.RaftRcvdMsgAppResp
sm.raftRcvdMessages[raftpb.MsgVote] = sm.RaftRcvdMsgVote
sm.raftRcvdMessages[raftpb.MsgVoteResp] = sm.RaftRcvdMsgVoteResp
sm.raftRcvdMessages[raftpb.MsgPreVote] = sm.RaftRcvdMsgPreVote
sm.raftRcvdMessages[raftpb.MsgPreVoteResp] = sm.RaftRcvdMsgPreVoteResp
sm.raftRcvdMessages[raftpb.MsgSnap] = sm.RaftRcvdMsgSnap
sm.raftRcvdMessages[raftpb.MsgHeartbeat] = sm.RaftRcvdMsgHeartbeat
sm.raftRcvdMessages[raftpb.MsgHeartbeatResp] = sm.RaftRcvdMsgHeartbeatResp
sm.raftRcvdMessages[raftpb.MsgTransferLeader] = sm.RaftRcvdMsgTransferLeader
sm.raftRcvdMessages[raftpb.MsgTimeoutNow] = sm.RaftRcvdMsgTimeoutNow
storeRegistry.AddMetricStruct(sm)
return sm
}
// TestGossipNoForwardSelf verifies that when a Gossip instance is full, it
// redirects clients elsewhere (in particular not to itself).
//
// NB: Stress testing this test really stresses the OS networking stack
// more than anything else. For example, on Linux it may quickly deplete
// the ephemeral port range (due to the TIME_WAIT state).
// On a box which only runs tests, this can be circumvented by running
//
// sudo bash -c "echo 1 > /proc/sys/net/ipv4/tcp_tw_recycle"
//
// See https://vincent.bernat.im/en/blog/2014-tcp-time-wait-state-linux.html
// for details.
//
// On OSX, things similarly fall apart. See #7524 and #5218 for some discussion
// of this.
func TestGossipNoForwardSelf(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
local := startGossip(1, stopper, t, metric.NewRegistry())
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Start one loopback client plus enough additional clients to fill the
// incoming clients.
peers := []*Gossip{local}
local.server.mu.Lock()
maxSize := local.server.mu.incoming.maxSize
local.server.mu.Unlock()
for i := 0; i < maxSize; i++ {
peers = append(peers, startGossip(roachpb.NodeID(i+2), stopper, t, metric.NewRegistry()))
}
for _, peer := range peers {
c := newClient(log.AmbientContext{}, local.GetNodeAddr(), makeMetrics())
util.SucceedsSoon(t, func() error {
conn, err := peer.rpcContext.GRPCDial(c.addr.String(), grpc.WithBlock())
if err != nil {
return err
}
stream, err := NewGossipClient(conn).Gossip(ctx)
if err != nil {
return err
}
if err := c.requestGossip(peer, stream); err != nil {
return err
}
// Wait until the server responds, so we know we're connected.
_, err = stream.Recv()
return err
})
}
numClients := len(peers) * 2
disconnectedCh := make(chan *client)
// Start a few overflow peers and assert that they don't get forwarded to us
// again.
for i := 0; i < numClients; i++ {
local.server.mu.Lock()
maxSize := local.server.mu.incoming.maxSize
local.server.mu.Unlock()
peer := startGossip(roachpb.NodeID(i+maxSize+2), stopper, t, metric.NewRegistry())
for {
localAddr := local.GetNodeAddr()
c := newClient(log.AmbientContext{}, localAddr, makeMetrics())
c.start(peer, disconnectedCh, peer.rpcContext, stopper, peer.rpcContext.NewBreaker())
disconnectedClient := <-disconnectedCh
if disconnectedClient != c {
t.Fatalf("expected %p to be disconnected, got %p", c, disconnectedClient)
} else if c.forwardAddr == nil {
// Under high load, clients sometimes fail to connect for reasons
// unrelated to the test, so we need to permit some.
t.Logf("node #%d: got nil forwarding address", peer.NodeID.Get())
continue
} else if *c.forwardAddr == *localAddr {
t.Errorf("node #%d: got local's forwarding address", peer.NodeID.Get())
}
break
}
}
}
// TestMetricsRecorder verifies that the metrics recorder properly formats the
// statistics from various registries, both for Time Series and for Status
// Summaries.
func TestMetricsRecorder(t *testing.T) {
defer leaktest.AfterTest(t)()
// ========================================
// Construct a series of fake descriptors for use in test.
// ========================================
nodeDesc := roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(1),
}
storeDesc1 := roachpb.StoreDescriptor{
StoreID: roachpb.StoreID(1),
Capacity: roachpb.StoreCapacity{
Capacity: 100,
Available: 50,
},
}
storeDesc2 := roachpb.StoreDescriptor{
StoreID: roachpb.StoreID(2),
Capacity: roachpb.StoreCapacity{
Capacity: 200,
Available: 75,
},
}
// ========================================
// Create registries and add them to the recorder (two node-level, two
// store-level).
// ========================================
reg1 := metric.NewRegistry()
store1 := fakeStore{
storeID: roachpb.StoreID(1),
desc: storeDesc1,
registry: metric.NewRegistry(),
}
store2 := fakeStore{
storeID: roachpb.StoreID(2),
desc: storeDesc2,
registry: metric.NewRegistry(),
}
manual := hlc.NewManualClock(100)
recorder := NewMetricsRecorder(hlc.NewClock(manual.UnixNano, time.Nanosecond))
recorder.AddStore(store1)
recorder.AddStore(store2)
recorder.AddNode(reg1, nodeDesc, 50)
// Ensure the metric system's view of time does not advance during this test
// as the test expects time to not advance too far which would age the actual
// data (e.g. in histogram's) unexpectedly.
defer metric.TestingSetNow(func() time.Time {
return time.Unix(0, manual.UnixNano()).UTC()
})()
// ========================================
// Generate Metrics Data & Expected Results
// ========================================
// Flatten the four registries into an array for ease of use.
regList := []struct {
reg *metric.Registry
prefix string
source int64
isNode bool
}{
{
reg: reg1,
prefix: "one.",
source: 1,
isNode: true,
},
{
reg: reg1,
prefix: "two.",
source: 1,
isNode: true,
},
{
reg: store1.registry,
prefix: "",
source: int64(store1.storeID),
isNode: false,
},
{
reg: store2.registry,
prefix: "",
source: int64(store2.storeID),
isNode: false,
},
}
// Every registry will have a copy of the following metrics.
metricNames := []struct {
name string
typ string
val int64
}{
{"testGauge", "gauge", 20},
{"testGaugeFloat64", "floatgauge", 20},
{"testCounter", "counter", 5},
{"testCounterWithRates", "counterwithrates", 2},
{"testHistogram", "histogram", 10},
{"testLatency", "latency", 10},
// Stats needed for store summaries.
{"ranges", "counter", 1},
{"replicas.leaders", "gauge", 1},
{"replicas.leaseholders", "gauge", 1},
{"ranges", "gauge", 1},
{"ranges.available", "gauge", 1},
}
// Add the metrics to each registry and set their values. At the same time,
// generate expected time series results and status summary metric values.
var expected []tspb.TimeSeriesData
expectedNodeSummaryMetrics := make(map[string]float64)
expectedStoreSummaryMetrics := make(map[string]float64)
// addExpected generates expected data for a single metric data point.
addExpected := func(prefix, name string, source, time, val int64, isNode bool) {
// Generate time series data.
tsPrefix := "cr.node."
if !isNode {
tsPrefix = "cr.store."
}
expect := tspb.TimeSeriesData{
Name: tsPrefix + prefix + name,
Source: strconv.FormatInt(source, 10),
Datapoints: []tspb.TimeSeriesDatapoint{
{
TimestampNanos: time,
Value: float64(val),
},
},
}
expected = append(expected, expect)
// Generate status summary data.
if isNode {
expectedNodeSummaryMetrics[prefix+name] = float64(val)
} else {
// This can overwrite the previous value, but this is expected as
// all stores in our tests have identical values; when comparing
// status summaries, the same map is used as expected data for all
// stores.
expectedStoreSummaryMetrics[prefix+name] = float64(val)
}
}
for _, reg := range regList {
for _, data := range metricNames {
switch data.typ {
case "gauge":
g := metric.NewGauge(metric.Metadata{Name: reg.prefix + data.name})
reg.reg.AddMetric(g)
g.Update(data.val)
addExpected(reg.prefix, data.name, reg.source, 100, data.val, reg.isNode)
case "floatgauge":
g := metric.NewGaugeFloat64(metric.Metadata{Name: reg.prefix + data.name})
reg.reg.AddMetric(g)
g.Update(float64(data.val))
addExpected(reg.prefix, data.name, reg.source, 100, data.val, reg.isNode)
case "counter":
c := metric.NewCounter(metric.Metadata{Name: reg.prefix + data.name})
reg.reg.AddMetric(c)
c.Inc((data.val))
addExpected(reg.prefix, data.name, reg.source, 100, data.val, reg.isNode)
case "counterwithrates":
r := metric.NewCounterWithRates(metric.Metadata{Name: reg.prefix + data.name})
reg.reg.AddMetric(r)
r.Inc(data.val)
addExpected(reg.prefix, data.name, reg.source, 100, data.val, reg.isNode)
case "histogram":
h := metric.NewHistogram(metric.Metadata{Name: reg.prefix + data.name}, time.Second, 1000, 2)
reg.reg.AddMetric(h)
h.RecordValue(data.val)
for _, q := range recordHistogramQuantiles {
addExpected(reg.prefix, data.name+q.suffix, reg.source, 100, data.val, reg.isNode)
}
case "latency":
l := metric.NewLatency(metric.Metadata{Name: reg.prefix + data.name}, time.Hour)
reg.reg.AddMetric(l)
l.RecordValue(data.val)
// Latency is simply three histograms (at different resolution
// time scales).
for _, q := range recordHistogramQuantiles {
addExpected(reg.prefix, data.name+q.suffix, reg.source, 100, data.val, reg.isNode)
}
default:
t.Fatalf("unexpected: %+v", data)
}
}
}
// ========================================
// Verify time series data
// ========================================
actual := recorder.GetTimeSeriesData()
// Actual comparison is simple: sort the resulting arrays by time and name,
// and use reflect.DeepEqual.
sort.Sort(byTimeAndName(actual))
sort.Sort(byTimeAndName(expected))
if a, e := actual, expected; !reflect.DeepEqual(a, e) {
t.Errorf("recorder did not yield expected time series collection; diff:\n %v", pretty.Diff(e, a))
}
// ========================================
// Verify node summary generation
// ========================================
expectedNodeSummary := &NodeStatus{
Desc: nodeDesc,
BuildInfo: build.GetInfo(),
StartedAt: 50,
UpdatedAt: 100,
Metrics: expectedNodeSummaryMetrics,
StoreStatuses: []StoreStatus{
{
Desc: storeDesc1,
Metrics: expectedStoreSummaryMetrics,
},
{
Desc: storeDesc2,
Metrics: expectedStoreSummaryMetrics,
},
},
}
nodeSummary := recorder.GetStatusSummary()
if nodeSummary == nil {
t.Fatalf("recorder did not return nodeSummary")
}
sort.Sort(byStoreDescID(nodeSummary.StoreStatuses))
if a, e := nodeSummary, expectedNodeSummary; !reflect.DeepEqual(a, e) {
t.Errorf("recorder did not produce expected NodeSummary; diff:\n %v", pretty.Diff(e, a))
}
}