// startGossip creates local and remote gossip instances.
// Both remote and local instances launch the gossip service.
func startGossip(t *testing.T) (local, remote *Gossip, stopper *stop.Stopper) {
stopper = stop.NewStopper()
lclock := hlc.NewClock(hlc.UnixNano)
lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper)
laddr := util.CreateTestAddr("tcp")
lserver := rpc.NewServer(lRPCContext)
lTLSConfig, err := lRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
lln, err := util.ListenAndServe(stopper, lserver, laddr, lTLSConfig)
if err != nil {
t.Fatal(err)
}
local = New(lRPCContext, TestBootstrap)
local.SetNodeID(1)
if err := local.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: 1,
Address: util.MakeUnresolvedAddr(laddr.Network(), laddr.String()),
}); err != nil {
t.Fatal(err)
}
rclock := hlc.NewClock(hlc.UnixNano)
rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper)
raddr := util.CreateTestAddr("tcp")
rserver := rpc.NewServer(rRPCContext)
rTLSConfig, err := rRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
rln, err := util.ListenAndServe(stopper, rserver, raddr, rTLSConfig)
if err != nil {
t.Fatal(err)
}
remote = New(rRPCContext, TestBootstrap)
remote.SetNodeID(2)
if err := remote.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: 2,
Address: util.MakeUnresolvedAddr(raddr.Network(), raddr.String()),
}); err != nil {
t.Fatal(err)
}
local.start(lserver, lln.Addr(), stopper)
remote.start(rserver, rln.Addr(), stopper)
time.Sleep(time.Millisecond)
return
}
func newTestServer(t *testing.T, ctx *Context, manual bool) (*Server, net.Listener) {
var s *Server
if manual {
s = &Server{
insecure: ctx.Insecure,
activeConns: make(map[net.Conn]struct{}),
methods: map[string]method{},
}
} else {
s = NewServer(ctx)
}
tlsConfig, err := ctx.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
addr := util.CreateTestAddr("tcp")
ln, err := util.ListenAndServe(ctx.Stopper, s, addr, tlsConfig)
if err != nil {
t.Fatal(err)
}
return s, ln
}
// startGossip creates and starts a gossip instance.
func startGossip(nodeID roachpb.NodeID, stopper *stop.Stopper, t *testing.T) *Gossip {
clock := hlc.NewClock(hlc.UnixNano)
rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper)
addr := util.CreateTestAddr("tcp")
server := rpc.NewServer(rpcContext)
tlsConfig, err := rpcContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
ln, err := util.ListenAndServe(stopper, server, addr, tlsConfig)
if err != nil {
t.Fatal(err)
}
g := New(rpcContext, TestBootstrap, stopper)
g.SetNodeID(nodeID)
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: nodeID,
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}); err != nil {
t.Fatal(err)
}
g.start(server, ln.Addr())
time.Sleep(time.Millisecond)
return g
}
// Start runs the RPC and HTTP servers, starts the gossip instance (if
// selfBootstrap is true, uses the rpc server's address as the gossip
// bootstrap), and starts the node using the supplied engines slice.
func (s *Server) Start(selfBootstrap bool) error {
tlsConfig, err := s.ctx.GetServerTLSConfig()
if err != nil {
return err
}
unresolvedAddr := util.MakeUnresolvedAddr("tcp", s.ctx.Addr)
ln, err := util.ListenAndServe(s.stopper, s, unresolvedAddr, tlsConfig)
if err != nil {
return err
}
s.listener = ln
addr := ln.Addr()
addrStr := addr.String()
// Handle self-bootstrapping case for a single node.
if selfBootstrap {
selfResolver, err := resolver.NewResolver(&s.ctx.Context, addrStr)
if err != nil {
return err
}
s.gossip.SetResolvers([]resolver.Resolver{selfResolver})
}
s.gossip.Start(s.rpc, addr, s.stopper)
if err := s.node.start(s.rpc, addr, s.ctx.Engines, s.ctx.NodeAttributes); err != nil {
return err
}
// Begin recording runtime statistics.
runtime := status.NewRuntimeStatRecorder(s.node.Descriptor.NodeID, s.clock)
s.tsDB.PollSource(runtime, s.ctx.MetricsFrequency, ts.Resolution10s, s.stopper)
// Begin recording time series data collected by the status monitor.
s.recorder = status.NewNodeStatusRecorder(s.node.status, s.clock)
s.tsDB.PollSource(s.recorder, s.ctx.MetricsFrequency, ts.Resolution10s, s.stopper)
// Begin recording status summaries.
s.startWriteSummaries()
s.sqlServer.SetNodeID(s.node.Descriptor.NodeID)
s.status = newStatusServer(s.db, s.gossip, s.metaRegistry, s.ctx)
log.Infof("starting %s server at %s", s.ctx.HTTPRequestScheme(), addr)
s.initHTTP()
// TODO(tamird): pick a port here
host, _, err := net.SplitHostPort(addrStr)
if err != nil {
return err
}
return s.pgServer.Start(util.MakeUnresolvedAddr("tcp", net.JoinHostPort(host, "0")))
}
// Start runs the RPC and HTTP servers, starts the gossip instance (if
// selfBootstrap is true, uses the rpc server's address as the gossip
// bootstrap), and starts the node using the supplied engines slice.
func (s *Server) Start(selfBootstrap bool) error {
tlsConfig, err := s.ctx.GetServerTLSConfig()
if err != nil {
return err
}
unresolvedAddr := util.MakeUnresolvedAddr("tcp", s.ctx.Addr)
ln, err := util.ListenAndServe(s.stopper, s, unresolvedAddr, tlsConfig)
if err != nil {
return err
}
s.listener = ln
addr := ln.Addr()
addrStr := addr.String()
s.rpcContext.SetLocalServer(s.rpc, addrStr)
// Handle self-bootstrapping case for a single node.
if selfBootstrap {
selfResolver, err := resolver.NewResolver(&s.ctx.Context, addrStr)
if err != nil {
return err
}
s.gossip.SetResolvers([]resolver.Resolver{selfResolver})
}
s.gossip.Start(s.rpc, addr)
if err := s.node.start(s.rpc, addr, s.ctx.Engines, s.ctx.NodeAttributes); err != nil {
return err
}
// Begin recording runtime statistics.
runtime := status.NewRuntimeStatRecorder(s.node.Descriptor.NodeID, s.clock)
s.tsDB.PollSource(runtime, s.ctx.MetricsFrequency, ts.Resolution10s, s.stopper)
// Begin recording time series data collected by the status monitor.
s.recorder = status.NewNodeStatusRecorder(s.node.status, s.clock)
s.tsDB.PollSource(s.recorder, s.ctx.MetricsFrequency, ts.Resolution10s, s.stopper)
// Begin recording status summaries.
s.startWriteSummaries()
s.sqlServer.SetNodeID(s.node.Descriptor.NodeID)
// Create and start the schema change manager only after a NodeID
// has been assigned.
s.schemaChangeManager = sql.NewSchemaChangeManager(*s.db, s.gossip, s.leaseMgr)
s.schemaChangeManager.Start(s.stopper)
s.status = newStatusServer(s.db, s.gossip, s.metaRegistry, s.ctx)
log.Infof("starting %s server at %s", s.ctx.HTTPRequestScheme(), addr)
s.initHTTP()
return s.pgServer.Start(util.MakeUnresolvedAddr("tcp", s.ctx.PGAddr))
}
// NewNetwork creates nodeCount gossip nodes. The networkType should
// be set to either "tcp" or "unix".
func NewNetwork(nodeCount int, networkType string) *Network {
clock := hlc.NewClock(hlc.UnixNano)
log.Infof("simulating gossip network with %d nodes", nodeCount)
stopper := stop.NewStopper()
rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper)
tlsConfig, err := rpcContext.GetServerTLSConfig()
if err != nil {
log.Fatal(err)
}
nodes := make([]*Node, nodeCount)
for i := range nodes {
server := rpc.NewServer(rpcContext)
testAddr := util.CreateTestAddr(networkType)
ln, err := util.ListenAndServe(stopper, server, testAddr, tlsConfig)
if err != nil {
log.Fatal(err)
}
nodes[i] = &Node{Server: server, Addr: ln.Addr()}
}
for i, leftNode := range nodes {
// Build new resolvers for each instance or we'll get data races.
resolvers := []resolver.Resolver{resolver.NewResolverFromAddress(nodes[0].Addr)}
gossipNode := gossip.New(rpcContext, resolvers)
addr := leftNode.Addr
gossipNode.SetNodeID(roachpb.NodeID(i + 1))
if err := gossipNode.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(i + 1),
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}); err != nil {
log.Fatal(err)
}
if err := gossipNode.AddInfo(addr.String(), encoding.EncodeUint64(nil, 0), time.Hour); err != nil {
log.Fatal(err)
}
gossipNode.Start(leftNode.Server, addr, stopper)
gossipNode.EnableSimulationCycler(true)
leftNode.Gossip = gossipNode
}
return &Network{
Nodes: nodes,
NetworkType: networkType,
Stopper: stopper,
}
}
// CreateNode creates a simulation node and starts an RPC server for it.
func (n *Network) CreateNode() (*Node, error) {
server := rpc.NewServer(n.rpcContext)
testAddr := util.CreateTestAddr("tcp")
ln, err := util.ListenAndServe(n.Stopper, server, testAddr, n.tlsConfig)
if err != nil {
return nil, err
}
node := &Node{Server: server, Addr: ln.Addr()}
node.Gossip = gossip.New(n.rpcContext, nil)
n.Nodes = append(n.Nodes, node)
return node, nil
}
// startFakeServerGossips creates local gossip instances and remote
// faked gossip instance. The remote gossip instance launches its
// faked gossip service just for check the client message.
func startFakeServerGossips(t *testing.T) (local *Gossip, remote *fakeGossipServer, stopper *stop.Stopper) {
stopper = stop.NewStopper()
lclock := hlc.NewClock(hlc.UnixNano)
lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper)
laddr := util.CreateTestAddr("tcp")
lserver := rpc.NewServer(lRPCContext)
lTLSConfig, err := lRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
lln, err := util.ListenAndServe(stopper, lserver, laddr, lTLSConfig)
if err != nil {
t.Fatal(err)
}
local = New(lRPCContext, TestBootstrap, stopper)
local.start(lserver, lln.Addr())
rclock := hlc.NewClock(hlc.UnixNano)
rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper)
raddr := util.CreateTestAddr("tcp")
rserver := rpc.NewServer(rRPCContext)
rTLSConfig, err := rRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
rln, err := util.ListenAndServe(stopper, rserver, raddr, rTLSConfig)
if err != nil {
t.Fatal(err)
}
if remote, err = newFakeGossipServer(rserver, stopper); err != nil {
t.Fatal(err)
}
addr := rln.Addr()
remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String())
time.Sleep(time.Millisecond)
return
}
// createTestNode creates an rpc server using the specified address,
// gossip instance, KV database and a node using the specified slice
// of engines. The server, clock and node are returned. If gossipBS is
// not nil, the gossip bootstrap address is set to gossipBS.
func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) (
*rpc.Server, net.Addr, *hlc.Clock, *Node, *stop.Stopper) {
ctx := storage.StoreContext{}
stopper := stop.NewStopper()
ctx.Clock = hlc.NewClock(hlc.UnixNano)
nodeRPCContext := rpc.NewContext(nodeTestBaseContext, ctx.Clock, stopper)
ctx.ScanInterval = 10 * time.Hour
rpcServer := rpc.NewServer(nodeRPCContext)
grpcServer := grpc.NewServer()
tlsConfig, err := nodeRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
ln, err := util.ListenAndServe(stopper, grpcutil.GRPCHandlerFunc(grpcServer, rpcServer), addr, tlsConfig)
if err != nil {
t.Fatal(err)
}
g := gossip.New(nodeRPCContext, testContext.GossipBootstrapResolvers, stopper)
if gossipBS != nil {
// Handle possibility of a :0 port specification.
if gossipBS.Network() == addr.Network() && gossipBS.String() == addr.String() {
gossipBS = ln.Addr()
}
r, err := resolver.NewResolverFromAddress(gossipBS)
if err != nil {
t.Fatalf("bad gossip address %s: %s", gossipBS, err)
}
g.SetResolvers([]resolver.Resolver{r})
g.Start(grpcServer, ln.Addr())
}
ctx.Gossip = g
retryOpts := kv.GetDefaultDistSenderRetryOptions()
retryOpts.Closer = stopper.ShouldDrain()
distSender := kv.NewDistSender(&kv.DistSenderContext{
Clock: ctx.Clock,
RPCContext: nodeRPCContext,
RPCRetryOptions: &retryOpts,
}, g)
tracer := tracing.NewTracer()
sender := kv.NewTxnCoordSender(distSender, ctx.Clock, false, tracer, stopper)
ctx.DB = client.NewDB(sender)
// TODO(bdarnell): arrange to have the transport closed.
// (or attach LocalRPCTransport.Close to the stopper)
ctx.Transport = storage.NewLocalRPCTransport(stopper)
ctx.EventFeed = util.NewFeed(stopper)
ctx.Tracer = tracer
node := NewNode(ctx, metric.NewRegistry(), stopper, nil)
return rpcServer, ln.Addr(), ctx.Clock, node, stopper
}
// Start starts the server on the specified port, starts gossip and
// initializes the node using the engines from the server's context.
func (s *Server) Start() error {
tlsConfig, err := s.ctx.GetServerTLSConfig()
if err != nil {
return err
}
unresolvedAddr := util.NewUnresolvedAddr("tcp", s.ctx.Addr)
ln, err := util.ListenAndServe(s.stopper, s, unresolvedAddr, tlsConfig)
if err != nil {
return err
}
s.listener = ln // Only used in tests.
if err := officializeAddr(unresolvedAddr, ln.Addr()); err != nil {
return err
}
s.rpcContext.SetLocalServer(s.rpc, unresolvedAddr.String())
s.grpc = grpc.NewServer()
s.gossip.Start(s.grpc, unresolvedAddr)
if err := s.node.start(s.rpc, unresolvedAddr, s.ctx.Engines, s.ctx.NodeAttributes); err != nil {
return err
}
// Begin recording runtime statistics.
runtime := status.NewRuntimeStatRecorder(s.node.Descriptor.NodeID, s.clock)
s.tsDB.PollSource(runtime, s.ctx.MetricsFrequency, ts.Resolution10s, s.stopper)
// Begin recording time series data collected by the status monitor.
s.recorder = status.NewNodeStatusRecorder(s.node.status, s.clock)
s.tsDB.PollSource(s.recorder, s.ctx.MetricsFrequency, ts.Resolution10s, s.stopper)
// Begin recording status summaries.
s.startWriteSummaries()
s.sqlServer.SetNodeID(s.node.Descriptor.NodeID)
// Create and start the schema change manager only after a NodeID
// has been assigned.
s.schemaChangeManager = sql.NewSchemaChangeManager(*s.db, s.gossip, s.leaseMgr)
s.schemaChangeManager.Start(s.stopper)
s.status = newStatusServer(s.db, s.gossip, s.registry, s.ctx)
log.Infof("starting %s server at %s", s.ctx.HTTPRequestScheme(), unresolvedAddr)
s.initHTTP()
return s.pgServer.Start(util.NewUnresolvedAddr("tcp", s.ctx.PGAddr))
}
// TestClientRegisterInitNodeID verifies two client's gossip request with NodeID 0.
func TestClientRegisterWithInitNodeID(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
// Create three gossip nodes, and connect to the first with NodeID 0.
var g []*Gossip
var gossipAddr string
for i := 0; i < 3; i++ {
clock := hlc.NewClock(hlc.UnixNano)
RPCContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper)
addr := util.CreateTestAddr("tcp")
server := grpc.NewServer()
TLSConfig, err := RPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
ln, err := util.ListenAndServe(stopper, server, addr, TLSConfig)
if err != nil {
t.Fatal(err)
}
// Connect to the first gossip node.
if gossipAddr == "" {
gossipAddr = ln.Addr().String()
}
var resolvers []resolver.Resolver
resolver, _ := resolver.NewResolver(&RPCContext.Context, gossipAddr)
resolvers = append(resolvers, resolver)
gnode := New(RPCContext, resolvers, stopper)
// node ID must be non-zero
gnode.SetNodeID(roachpb.NodeID(i + 1))
g = append(g, gnode)
gnode.Start(server, ln.Addr())
}
util.SucceedsSoon(t, func() error {
// The first gossip node should have two gossip client address
// in nodeMap if these three gossip nodes registered success.
g[0].mu.Lock()
defer g[0].mu.Unlock()
if a, e := len(g[0].nodeMap), 2; a != e {
return util.Errorf("expected %s to contain %d nodes, got %d", g[0].nodeMap, e, a)
}
return nil
})
}
func newTestServer(t *testing.T, ctx *rpc.Context) (*rpc.Server, net.Listener) {
s := rpc.NewServer(ctx)
tlsConfig, err := ctx.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
addr := util.CreateTestAddr("tcp")
ln, err := util.ListenAndServe(ctx.Stopper, s, addr, tlsConfig)
if err != nil {
t.Fatal(err)
}
return s, ln
}
// TestClientRegisterInitNodeID verifies two client's gossip request with NodeID 0.
func TestClientRegisterWithInitNodeID(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
// Create three gossip nodes, and connect to the first with NodeID 0.
var g []*Gossip
var gossipAddr string
for i := 0; i < 3; i++ {
clock := hlc.NewClock(hlc.UnixNano)
RPCContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper)
addr := util.CreateTestAddr("tcp")
server := rpc.NewServer(RPCContext)
TLSConfig, err := RPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
ln, err := util.ListenAndServe(stopper, server, addr, TLSConfig)
if err != nil {
t.Fatal(err)
}
// Connect to the first gossip node.
if gossipAddr == "" {
gossipAddr = ln.Addr().String()
}
var resolvers []resolver.Resolver
resolver, _ := resolver.NewResolver(&RPCContext.Context, gossipAddr)
resolvers = append(resolvers, resolver)
gnode := New(RPCContext, resolvers, stopper)
g = append(g, gnode)
gnode.Start(server, ln.Addr())
}
util.SucceedsWithin(t, 5*time.Second, func() error {
// The first gossip node should have two gossip client address
// in lAddrMap if these three gossip nodes registered success.
g[0].mu.Lock()
defer g[0].mu.Unlock()
if len(g[0].lAddrMap) == 2 {
return nil
}
return util.Errorf("gossip client register fail.")
})
}
func (lt *localRPCTransport) Listen(id roachpb.StoreID, handler RaftMessageHandler) error {
ctx := crpc.Context{
Context: base.Context{
Insecure: true,
},
Stopper: lt.stopper,
DisableCache: true,
}
rpcServer := crpc.NewServer(&ctx)
err := rpcServer.RegisterAsync(raftMessageName, false, /*not public*/
func(argsI proto.Message, callback func(proto.Message, error)) {
defer func() {
// TODO(bdarnell): the http/rpc code is swallowing panics somewhere.
if p := recover(); p != nil {
log.Fatalf("caught panic: %s", p)
}
}()
args := argsI.(*RaftMessageRequest)
err := handler(args)
callback(&RaftMessageResponse{}, err)
}, &RaftMessageRequest{})
if err != nil {
return err
}
tlsConfig, err := ctx.GetServerTLSConfig()
if err != nil {
return err
}
addr := util.CreateTestAddr("tcp")
ln, err := util.ListenAndServe(ctx.Stopper, rpcServer, addr, tlsConfig)
if err != nil {
return err
}
lt.mu.Lock()
if _, ok := lt.servers[id]; ok {
log.Fatalf("node %d already listening", id)
}
lt.servers[id] = serverWithAddr{rpcServer, ln.Addr()}
lt.mu.Unlock()
return nil
}
func newTestServer(t *testing.T, ctx *rpc.Context, manual bool) (*rpc.Server, net.Listener) {
var s *rpc.Server
if manual {
s = rpc.NewManualServer(ctx)
} else {
s = rpc.NewServer(ctx)
}
tlsConfig, err := ctx.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
addr := util.CreateTestAddr("tcp")
ln, err := util.ListenAndServe(ctx.Stopper, s, addr, tlsConfig)
if err != nil {
t.Fatal(err)
}
return s, ln
}
func newTestServer(t *testing.T, ctx *rpc.Context) (*rpc.Server, net.Listener) {
s := rpc.NewServer(ctx)
tlsConfig, err := ctx.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
// We may be called in a loop, meaning tlsConfig is may be in used by a
// running server during a call to `util.ListenAndServe`, which may
// mutate it (due to http2.ConfigureServer). Make a copy to avoid trouble.
tlsConfigCopy := *tlsConfig
addr := util.CreateTestAddr("tcp")
ln, err := util.ListenAndServe(ctx.Stopper, s, addr, &tlsConfigCopy)
if err != nil {
t.Fatal(err)
}
return s, ln
}
// createTestNode creates an rpc server using the specified address,
// gossip instance, KV database and a node using the specified slice
// of engines. The server, clock and node are returned. If gossipBS is
// not nil, the gossip bootstrap address is set to gossipBS.
func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) (
*rpc.Server, net.Addr, *hlc.Clock, *Node, *stop.Stopper) {
ctx := storage.StoreContext{}
stopper := stop.NewStopper()
ctx.Clock = hlc.NewClock(hlc.UnixNano)
nodeRPCContext := rpc.NewContext(nodeTestBaseContext, ctx.Clock, stopper)
ctx.ScanInterval = 10 * time.Hour
rpcServer := rpc.NewServer(nodeRPCContext)
tlsConfig, err := nodeRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
ln, err := util.ListenAndServe(stopper, rpcServer, addr, tlsConfig)
if err != nil {
t.Fatal(err)
}
g := gossip.New(nodeRPCContext, testContext.GossipBootstrapResolvers)
if gossipBS != nil {
// Handle possibility of a :0 port specification.
if gossipBS == addr {
gossipBS = ln.Addr()
}
r, err := resolver.NewResolverFromAddress(gossipBS)
if err != nil {
t.Fatalf("bad gossip address %s: %s", gossipBS, err)
}
g.SetResolvers([]resolver.Resolver{r})
g.Start(rpcServer, ln.Addr(), stopper)
}
ctx.Gossip = g
sender := kv.NewDistSender(&kv.DistSenderContext{Clock: ctx.Clock, RPCContext: nodeRPCContext}, g)
ctx.DB = client.NewDB(sender)
// TODO(bdarnell): arrange to have the transport closed.
// (or attach LocalRPCTransport.Close to the stopper)
ctx.Transport = storage.NewLocalRPCTransport(stopper)
ctx.EventFeed = util.NewFeed(stopper)
node := NewNode(ctx, metric.NewRegistry(), stopper)
return rpcServer, ln.Addr(), ctx.Clock, node, stopper
}
func (lt *localRPCTransport) Listen(id roachpb.StoreID, server ServerInterface) error {
ctx := crpc.Context{
Context: base.Context{
Insecure: true,
},
Stopper: lt.stopper,
DisableCache: true,
}
rpcServer := crpc.NewServer(&ctx)
err := rpcServer.RegisterAsync(raftMessageName, false, /*not public*/
func(argsI proto.Message, callback func(proto.Message, error)) {
args := argsI.(*RaftMessageRequest)
resp, err := server.RaftMessage(args)
callback(resp, err)
}, &RaftMessageRequest{})
if err != nil {
return err
}
tlsConfig, err := ctx.GetServerTLSConfig()
if err != nil {
return err
}
addr := util.CreateTestAddr("tcp")
ln, err := util.ListenAndServe(ctx.Stopper, rpcServer, addr, tlsConfig)
if err != nil {
return err
}
lt.mu.Lock()
if _, ok := lt.servers[id]; ok {
log.Fatalf("node %d already listening", id)
}
lt.servers[id] = serverWithAddr{rpcServer, ln.Addr()}
lt.mu.Unlock()
return nil
}
func TestSendAndReceive(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
nodeRPCContext := rpc.NewContext(nodeTestBaseContext, hlc.NewClock(hlc.UnixNano), stopper)
g := gossip.New(nodeRPCContext, gossip.TestBootstrap, stopper)
g.SetNodeID(roachpb.NodeID(1))
// Create several servers, each of which has two stores (A raft
// node ID addresses a store). Node 1 has stores 1 and 2, node 2 has
// stores 3 and 4, etc.
//
// We suppose that range 1 is replicated across the odd-numbered
// stores in reverse order to ensure that the various IDs are not
// equal: replica 1 is store 5, replica 2 is store 3, and replica 3
// is store 1.
const numServers = 3
const storesPerServer = 2
const numStores = numServers * storesPerServer
nextNodeID := roachpb.NodeID(2)
nextStoreID := roachpb.StoreID(2)
// Per-node state.
transports := map[roachpb.NodeID]storage.RaftTransport{}
// Per-store state.
storeNodes := map[roachpb.StoreID]roachpb.NodeID{}
channels := map[roachpb.StoreID]channelServer{}
replicaIDs := map[roachpb.StoreID]roachpb.ReplicaID{
1: 3,
3: 2,
5: 1,
}
for serverIndex := 0; serverIndex < numServers; serverIndex++ {
nodeID := nextNodeID
nextNodeID++
rpcServer := rpc.NewServer(nodeRPCContext)
grpcServer := grpc.NewServer()
tlsConfig, err := nodeRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
ln, err := util.ListenAndServe(stopper, grpcutil.GRPCHandlerFunc(grpcServer, rpcServer), util.CreateTestAddr("tcp"), tlsConfig)
if err != nil {
t.Fatal(err)
}
addr := ln.Addr()
// Have to call g.SetNodeID before call g.AddInfo
g.SetNodeID(roachpb.NodeID(nodeID))
if err := g.AddInfoProto(gossip.MakeNodeIDKey(nodeID),
&roachpb.NodeDescriptor{
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
},
time.Hour); err != nil {
t.Fatal(err)
}
transport := newRPCTransport(g, grpcServer, nodeRPCContext)
defer transport.Close()
transports[nodeID] = transport
for store := 0; store < storesPerServer; store++ {
storeID := nextStoreID
nextStoreID++
storeNodes[storeID] = nodeID
channel := newChannelServer(10, 0)
if err := transport.Listen(storeID, channel.RaftMessage); err != nil {
t.Fatal(err)
}
channels[storeID] = channel
}
}
// Heartbeat messages: Each store sends one message to each store.
for fromStoreID, fromNodeID := range storeNodes {
for toStoreID, toNodeID := range storeNodes {
req := &storage.RaftMessageRequest{
GroupID: 0,
Message: raftpb.Message{
Type: raftpb.MsgHeartbeat,
From: uint64(fromStoreID),
To: uint64(toStoreID),
},
FromReplica: roachpb.ReplicaDescriptor{
NodeID: fromNodeID,
StoreID: fromStoreID,
ReplicaID: 0,
},
ToReplica: roachpb.ReplicaDescriptor{
NodeID: toNodeID,
StoreID: toStoreID,
ReplicaID: 0,
},
}
if err := transports[fromNodeID].Send(req); err != nil {
t.Errorf("Unable to send message from %d to %d: %s", fromNodeID, toNodeID, err)
}
}
}
// Read all the messages from the channels. Note that the transport
// does not guarantee in-order delivery between independent
// transports, so we just verify that the right number of messages
// end up in each channel.
for toStoreID := range storeNodes {
for range storeNodes {
select {
case req := <-channels[toStoreID].ch:
if req.Message.To != uint64(toStoreID) {
t.Errorf("invalid message received on channel %d: %+v",
toStoreID, req)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for message")
}
}
select {
case req := <-channels[toStoreID].ch:
t.Errorf("got unexpected message %+v on channel %d", req, toStoreID)
default:
}
}
// Real raft messages have different node/store/replica IDs.
// Send a message from replica 2 (on store 3, node 2) to replica 1 (on store 5, node 3)
fromStoreID := roachpb.StoreID(3)
toStoreID := roachpb.StoreID(5)
req := &storage.RaftMessageRequest{
GroupID: 1,
Message: raftpb.Message{
Type: raftpb.MsgApp,
From: uint64(replicaIDs[fromStoreID]),
To: uint64(replicaIDs[toStoreID]),
},
FromReplica: roachpb.ReplicaDescriptor{
NodeID: storeNodes[fromStoreID],
StoreID: fromStoreID,
ReplicaID: replicaIDs[fromStoreID],
},
ToReplica: roachpb.ReplicaDescriptor{
NodeID: storeNodes[toStoreID],
StoreID: toStoreID,
ReplicaID: replicaIDs[toStoreID],
},
}
if err := transports[storeNodes[fromStoreID]].Send(req); err != nil {
t.Errorf("Unable to send message from %d to %d: %s", fromStoreID, toStoreID, err)
}
select {
case req2 := <-channels[toStoreID].ch:
if !reflect.DeepEqual(req, req2) {
t.Errorf("got unexpected message %+v", req2)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for message")
}
select {
case req := <-channels[toStoreID].ch:
t.Errorf("got unexpected message %+v on channel %d", req, toStoreID)
default:
}
}
// TestInOrderDelivery verifies that for a given pair of nodes, raft
// messages are delivered in order.
func TestInOrderDelivery(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
nodeRPCContext := rpc.NewContext(nodeTestBaseContext, hlc.NewClock(hlc.UnixNano), stopper)
g := gossip.New(nodeRPCContext, gossip.TestBootstrap, stopper)
g.SetNodeID(roachpb.NodeID(1))
rpcServer := rpc.NewServer(nodeRPCContext)
grpcServer := grpc.NewServer()
tlsConfig, err := nodeRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
ln, err := util.ListenAndServe(stopper, grpcutil.GRPCHandlerFunc(grpcServer, rpcServer), util.CreateTestAddr("tcp"), tlsConfig)
if err != nil {
t.Fatal(err)
}
const numMessages = 100
nodeID := roachpb.NodeID(roachpb.NodeID(2))
serverTransport := newRPCTransport(g, grpcServer, nodeRPCContext)
defer serverTransport.Close()
serverChannel := newChannelServer(numMessages, 10*time.Millisecond)
if err := serverTransport.Listen(roachpb.StoreID(nodeID), serverChannel.RaftMessage); err != nil {
t.Fatal(err)
}
addr := ln.Addr()
// Have to set gossip.NodeID before call gossip.AddInofXXX
g.SetNodeID(nodeID)
if err := g.AddInfoProto(gossip.MakeNodeIDKey(nodeID),
&roachpb.NodeDescriptor{
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
},
time.Hour); err != nil {
t.Fatal(err)
}
clientNodeID := roachpb.NodeID(2)
clientTransport := newRPCTransport(g, nil, nodeRPCContext)
defer clientTransport.Close()
for i := 0; i < numMessages; i++ {
req := &storage.RaftMessageRequest{
GroupID: 1,
Message: raftpb.Message{
To: uint64(nodeID),
From: uint64(clientNodeID),
Commit: uint64(i),
},
ToReplica: roachpb.ReplicaDescriptor{
NodeID: nodeID,
StoreID: roachpb.StoreID(nodeID),
ReplicaID: roachpb.ReplicaID(nodeID),
},
FromReplica: roachpb.ReplicaDescriptor{
NodeID: clientNodeID,
StoreID: roachpb.StoreID(clientNodeID),
ReplicaID: roachpb.ReplicaID(clientNodeID),
},
}
if err := clientTransport.Send(req); err != nil {
t.Errorf("failed to send message %d: %s", i, err)
}
}
for i := 0; i < numMessages; i++ {
req := <-serverChannel.ch
if req.Message.Commit != uint64(i) {
t.Errorf("messages out of order: got %d while expecting %d", req.Message.Commit, i)
}
}
}