// startFakeServerGossip creates local gossip instances and remote faked gossip instance. // The remote gossip instance launches its faked gossip service just for // check the client message. func startFakeServerGossip(t *testing.T) (local *Gossip, remote *fakeGossipServer, stopper *stop.Stopper) { lclock := hlc.NewClock(hlc.UnixNano) stopper = stop.NewStopper() lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper) laddr := util.CreateTestAddr("tcp") lserver := rpc.NewServer(laddr, lRPCContext) if err := lserver.Start(); err != nil { t.Fatal(err) } local = New(lRPCContext, TestBootstrap) local.start(lserver, stopper) rclock := hlc.NewClock(hlc.UnixNano) raddr := util.CreateTestAddr("tcp") rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper) rserver := rpc.NewServer(raddr, rRPCContext) if err := rserver.Start(); err != nil { t.Fatal(err) } remote, err := newFakeGossipServer(rserver, stopper) if err != nil { t.Fatal(err) } addr := rserver.Addr() remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String()) time.Sleep(time.Millisecond) return }
// TestNodeJoin verifies a new node is able to join a bootstrapped // cluster consisting of one node. func TestNodeJoin(t *testing.T) { defer leaktest.AfterTest(t) engineStopper := stop.NewStopper() defer engineStopper.Stop() e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper) stopper := stop.NewStopper() _, err := BootstrapCluster("cluster-1", []engine.Engine{e}, stopper) if err != nil { t.Fatal(err) } stopper.Stop() // Set an aggressive gossip interval to make sure information is exchanged tout de suite. testContext.GossipInterval = gossip.TestInterval // Start the bootstrap node. engines1 := []engine.Engine{e} addr1 := util.CreateTestAddr("tcp") server1, node1, stopper1 := createAndStartTestNode(addr1, engines1, addr1, t) defer stopper1.Stop() // Create a new node. engines2 := []engine.Engine{engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)} server2, node2, stopper2 := createAndStartTestNode(util.CreateTestAddr("tcp"), engines2, server1.Addr(), t) defer stopper2.Stop() // Verify new node is able to bootstrap its store. if err := util.IsTrueWithin(func() bool { return node2.lSender.GetStoreCount() == 1 }, 50*time.Millisecond); err != nil { t.Fatal(err) } // Verify node1 sees node2 via gossip and vice versa. node1Key := gossip.MakeNodeIDKey(node1.Descriptor.NodeID) node2Key := gossip.MakeNodeIDKey(node2.Descriptor.NodeID) if err := util.IsTrueWithin(func() bool { nodeDesc1 := &roachpb.NodeDescriptor{} if err := node1.ctx.Gossip.GetInfoProto(node2Key, nodeDesc1); err != nil { return false } if addr2 := nodeDesc1.Address.AddressField; addr2 != server2.Addr().String() { t.Errorf("addr2 gossip %s doesn't match addr2 address %s", addr2, server2.Addr().String()) } nodeDesc2 := &roachpb.NodeDescriptor{} if err := node2.ctx.Gossip.GetInfoProto(node1Key, nodeDesc2); err != nil { return false } if addr1 := nodeDesc2.Address.AddressField; addr1 != server1.Addr().String() { t.Errorf("addr1 gossip %s doesn't match addr1 address %s", addr1, server1.Addr().String()) } return true }, 50*time.Millisecond); err != nil { t.Error(err) } }
// startGossip creates local and remote gossip instances. // Both remote and local instances launch the gossip service. func startGossip(t *testing.T) (local, remote *Gossip, stopper *stop.Stopper) { stopper = stop.NewStopper() lclock := hlc.NewClock(hlc.UnixNano) lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper) laddr := util.CreateTestAddr("tcp") lserver := rpc.NewServer(lRPCContext) lTLSConfig, err := lRPCContext.GetServerTLSConfig() if err != nil { t.Fatal(err) } lln, err := util.ListenAndServe(stopper, lserver, laddr, lTLSConfig) if err != nil { t.Fatal(err) } local = New(lRPCContext, TestBootstrap) local.SetNodeID(1) if err := local.SetNodeDescriptor(&roachpb.NodeDescriptor{ NodeID: 1, Address: util.MakeUnresolvedAddr(laddr.Network(), laddr.String()), }); err != nil { t.Fatal(err) } rclock := hlc.NewClock(hlc.UnixNano) rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper) raddr := util.CreateTestAddr("tcp") rserver := rpc.NewServer(rRPCContext) rTLSConfig, err := rRPCContext.GetServerTLSConfig() if err != nil { t.Fatal(err) } rln, err := util.ListenAndServe(stopper, rserver, raddr, rTLSConfig) if err != nil { t.Fatal(err) } remote = New(rRPCContext, TestBootstrap) remote.SetNodeID(2) if err := remote.SetNodeDescriptor(&roachpb.NodeDescriptor{ NodeID: 2, Address: util.MakeUnresolvedAddr(raddr.Network(), raddr.String()), }); err != nil { t.Fatal(err) } local.start(lserver, lln.Addr(), stopper) remote.start(rserver, rln.Addr(), stopper) time.Sleep(time.Millisecond) return }
// startGossip creates local and remote gossip instances. // The remote gossip instance launches its gossip service. func startGossip(t *testing.T) (local, remote *Gossip, lserver, rserver *rpc.Server) { laddr := util.CreateTestAddr("unix") lserver = rpc.NewServer(laddr) lserver.Start() local = New() raddr := util.CreateTestAddr("unix") rserver = rpc.NewServer(raddr) rserver.Start() remote = New() local.start(lserver) remote.start(rserver) time.Sleep(time.Millisecond) return }
// TestNodeJoin verifies a new node is able to join a bootstrapped // cluster consisting of one node. func TestNodeJoin(t *testing.T) { defer leaktest.AfterTest(t)() engineStopper := stop.NewStopper() defer engineStopper.Stop() e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper) if _, err := bootstrapCluster([]engine.Engine{e}, kv.NewTxnMetrics(metric.NewRegistry())); err != nil { t.Fatal(err) } // Start the bootstrap node. engines1 := []engine.Engine{e} addr1 := util.CreateTestAddr("tcp") _, server1Addr, node1, stopper1 := createAndStartTestNode(addr1, engines1, addr1, t) defer stopper1.Stop() // Create a new node. engines2 := []engine.Engine{engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)} addr2 := util.CreateTestAddr("tcp") _, server2Addr, node2, stopper2 := createAndStartTestNode(addr2, engines2, server1Addr, t) defer stopper2.Stop() // Verify new node is able to bootstrap its store. util.SucceedsSoon(t, func() error { if sc := node2.stores.GetStoreCount(); sc != 1 { return util.Errorf("GetStoreCount() expected 1; got %d", sc) } return nil }) // Verify node1 sees node2 via gossip and vice versa. node1Key := gossip.MakeNodeIDKey(node1.Descriptor.NodeID) node2Key := gossip.MakeNodeIDKey(node2.Descriptor.NodeID) util.SucceedsSoon(t, func() error { var nodeDesc1 roachpb.NodeDescriptor if err := node1.ctx.Gossip.GetInfoProto(node2Key, &nodeDesc1); err != nil { return err } if addr2Str, server2AddrStr := nodeDesc1.Address.String(), server2Addr.String(); addr2Str != server2AddrStr { return util.Errorf("addr2 gossip %s doesn't match addr2 address %s", addr2Str, server2AddrStr) } var nodeDesc2 roachpb.NodeDescriptor if err := node2.ctx.Gossip.GetInfoProto(node1Key, &nodeDesc2); err != nil { return err } if addr1Str, server1AddrStr := nodeDesc2.Address.String(), server1Addr.String(); addr1Str != server1AddrStr { return util.Errorf("addr1 gossip %s doesn't match addr1 address %s", addr1Str, server1AddrStr) } return nil }) }
// TestCorruptedClusterID verifies that a node fails to start when a // store's cluster ID is empty. func TestCorruptedClusterID(t *testing.T) { defer leaktest.AfterTest(t) engineStopper := stop.NewStopper() e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper) defer engineStopper.Stop() if _, err := bootstrapCluster([]engine.Engine{e}); err != nil { t.Fatal(err) } // Set the cluster ID to the empty UUID. sIdent := roachpb.StoreIdent{ ClusterID: *uuid.EmptyUUID, NodeID: 1, StoreID: 1, } if err := engine.MVCCPutProto(e, nil, keys.StoreIdentKey(), roachpb.ZeroTimestamp, nil, &sIdent); err != nil { t.Fatal(err) } engines := []engine.Engine{e} server, serverAddr, _, node, stopper := createTestNode(util.CreateTestAddr("tcp"), engines, nil, t) stopper.Stop() if err := node.start(server, serverAddr, engines, roachpb.Attributes{}); !testutils.IsError(err, "unidentified store") { t.Errorf("unexpected error %v", err) } }
// createAndStartNewServer creates and starts a new server with a test address. func createAndStartNewServer(t *testing.T, ctx *Context) *Server { s := NewServer(util.CreateTestAddr("tcp"), ctx) if err := s.Start(); err != nil { t.Fatal(err) } return s }
// startGossip creates and starts a gossip instance. func startGossip(nodeID roachpb.NodeID, stopper *stop.Stopper, t *testing.T) *Gossip { clock := hlc.NewClock(hlc.UnixNano) rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper) addr := util.CreateTestAddr("tcp") server := rpc.NewServer(rpcContext) tlsConfig, err := rpcContext.GetServerTLSConfig() if err != nil { t.Fatal(err) } ln, err := util.ListenAndServe(stopper, server, addr, tlsConfig) if err != nil { t.Fatal(err) } g := New(rpcContext, TestBootstrap, stopper) g.SetNodeID(nodeID) if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{ NodeID: nodeID, Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()), }); err != nil { t.Fatal(err) } g.start(server, ln.Addr()) time.Sleep(time.Millisecond) return g }
func newTestServer(t *testing.T, ctx *Context, manual bool) (*Server, net.Listener) { var s *Server if manual { s = &Server{ insecure: ctx.Insecure, activeConns: make(map[net.Conn]struct{}), methods: map[string]method{}, } } else { s = NewServer(ctx) } tlsConfig, err := ctx.GetServerTLSConfig() if err != nil { t.Fatal(err) } addr := util.CreateTestAddr("tcp") ln, err := util.ListenAndServe(ctx.Stopper, s, addr, tlsConfig) if err != nil { t.Fatal(err) } return s, ln }
// TestCorruptedClusterID verifies that a node fails to start when a // store's cluster ID is empty. func TestCorruptedClusterID(t *testing.T) { defer leaktest.AfterTest(t) eagerStopper := stop.NewStopper() e := engine.NewInMem(proto.Attributes{}, 1<<20) _, err := BootstrapCluster("cluster-1", []engine.Engine{e}, eagerStopper) if err != nil { t.Fatal(err) } eagerStopper.Stop() // Set the cluster ID to an empty string. sIdent := proto.StoreIdent{ ClusterID: "", NodeID: 1, StoreID: 1, } if err = engine.MVCCPutProto(e, nil, keys.StoreIdentKey(), proto.ZeroTimestamp, nil, &sIdent); err != nil { t.Fatal(err) } engines := []engine.Engine{e} server, _, node, stopper := createTestNode(util.CreateTestAddr("tcp"), engines, nil, t) if err := node.start(server, engines, proto.Attributes{}, stopper); err == nil { t.Errorf("unexpected success") } stopper.Stop() }
// TestClientHeartbeatBadServer verifies that the client is not marked // as "ready" until a heartbeat request succeeds. func TestClientHeartbeatBadServer(t *testing.T) { tlsConfig, err := LoadTestTLSConfig("..") if err != nil { t.Fatal(err) } addr := util.CreateTestAddr("tcp") // Create a server which doesn't support heartbeats. s := &Server{ Server: rpc.NewServer(), tlsConfig: tlsConfig, addr: addr, closeCallbacks: make([]func(conn net.Conn), 0, 1), } if err := s.Start(); err != nil { t.Fatal(err) } // Now, create a client. It should attempt a heartbeat and fail, // causing retry loop to activate. c := NewClient(s.Addr(), nil, tlsConfig) select { case <-c.Ready: t.Error("unexpected client heartbeat success") case <-c.Closed: } s.Close() }
// TestBootstrapNewStore starts a cluster with two unbootstrapped // stores and verifies both stores are added. func TestBootstrapNewStore(t *testing.T) { e := engine.NewInMem(engine.Attributes{}, 1<<20) localDB, err := BootstrapCluster("cluster-1", e) if err != nil { t.Fatal(err) } localDB.Close() // Start a new node with two new stores which will require bootstrapping. engines := []engine.Engine{ e, engine.NewInMem(engine.Attributes{}, 1<<20), engine.NewInMem(engine.Attributes{}, 1<<20), } server, node := createTestNode(util.CreateTestAddr("tcp"), engines, nil, t) defer server.Close() // Non-initialized stores (in this case the new in-memory-based // store) will be bootstrapped by the node upon start. This happens // in a goroutine, so we'll have to wait a bit (maximum 10ms) until // we can find the new node. if err := util.IsTrueWithin(func() bool { return node.localDB.GetStoreCount() == 3 }, 50*time.Millisecond); err != nil { t.Error(err) } }
func (lt *localRPCTransport) Listen(id roachpb.StoreID, server ServerInterface) error { addr := util.CreateTestAddr("tcp") rpcServer := crpc.NewServer(addr, &crpc.Context{ Context: base.Context{ Insecure: true, }, Stopper: lt.stopper, DisableCache: true, }) err := rpcServer.RegisterAsync(raftMessageName, false, /*not public*/ func(argsI proto.Message, callback func(proto.Message, error)) { args := argsI.(*RaftMessageRequest) resp, err := server.RaftMessage(args) callback(resp, err) }, &RaftMessageRequest{}) if err != nil { return err } lt.mu.Lock() if _, ok := lt.servers[id]; ok { log.Fatalf("node %d already listening", id) } lt.servers[id] = rpcServer lt.mu.Unlock() return rpcServer.Start() }
func (lt *localRPCTransport) Listen(id proto.RaftNodeID, server ServerInterface) error { addr := util.CreateTestAddr("tcp") rpcServer := crpc.NewServer(addr, &crpc.Context{ Context: base.Context{ Insecure: true, }, Stopper: lt.stopper, DisableCache: true, }) err := rpcServer.RegisterAsync(raftMessageName, func(argsI gogoproto.Message, callback func(gogoproto.Message, error)) { protoArgs := argsI.(*proto.RaftMessageRequest) args := RaftMessageRequest{ GroupID: protoArgs.GroupID, } if err := args.Message.Unmarshal(protoArgs.Msg); err != nil { callback(nil, err) } err := server.RaftMessage(&args, &RaftMessageResponse{}) callback(&proto.RaftMessageResponse{}, err) }, &proto.RaftMessageRequest{}) if err != nil { return err } lt.mu.Lock() if _, ok := lt.servers[id]; ok { log.Fatalf("node %d already listening", id) } lt.servers[id] = rpcServer lt.mu.Unlock() return rpcServer.Start() }
// SimulateNetwork creates nodeCount gossip nodes. The network should // be set to either "tcp" or "unix". The gossipInterval should be set // to a compressed simulation timescale, though large enough to give // the concurrent goroutines enough time to pass data back and forth // in order to yield accurate estimates of how old data actually ends // up being at the various nodes. After each gossipInterval period, // simCallback is invoked; when it returns false, the simulation // ends. If it returns true, the simulation continues another cycle. // // Node0 gossips the node count as well as the gossip sentinel. The // gossip bootstrap hosts are set to the first three nodes (or fewer if // less than three are available). // // At each cycle of the simulation, node 0 gossips the sentinel. If // the simulation requires other nodes to gossip, this should be done // via simCallback. // // The simulation callback receives a map of nodes, keyed by node address. func SimulateNetwork(nodeCount int, network string, gossipInterval time.Duration, simCallback func(cycle int, nodes map[string]*Gossip) bool) { // seed the random number generator for non-determinism across // multiple runs. rand.Seed(time.Now().UTC().UnixNano()) tlsConfig := rpc.LoadInsecureTLSConfig() log.Infof("simulating network with %d nodes", nodeCount) servers := make([]*rpc.Server, nodeCount) addrs := make([]net.Addr, nodeCount) for i := 0; i < nodeCount; i++ { addr := util.CreateTestAddr(network) servers[i] = rpc.NewServer(addr, tlsConfig) if err := servers[i].Start(); err != nil { log.Fatal(err) } addrs[i] = servers[i].Addr() } var bootstrap []net.Addr if nodeCount < 3 { bootstrap = addrs } else { bootstrap = addrs[:3] } nodes := make(map[string]*Gossip, nodeCount) for i := 0; i < nodeCount; i++ { node := New(tlsConfig) node.Name = fmt.Sprintf("Node%d", i) node.SetBootstrap(bootstrap) node.SetInterval(gossipInterval) node.Start(servers[i]) // Node 0 gossips node count. if i == 0 { node.AddInfo(KeyNodeCount, int64(nodeCount), time.Hour) } nodes[addrs[i].String()] = node } gossipTimeout := time.Tick(gossipInterval) var complete bool for cycle := 0; !complete; cycle++ { select { case <-gossipTimeout: // Node 0 gossips sentinel every cycle. nodes[addrs[0].String()].AddInfo(KeySentinel, int64(cycle), time.Hour) if !simCallback(cycle, nodes) { complete = true } } } // Stop all servers & nodes. for i := 0; i < nodeCount; i++ { servers[i].Close() nodes[addrs[i].String()].Stop() } }
// TestNodeJoin verifies a new node is able to join a bootstrapped // cluster consisting of one node. func TestNodeJoin(t *testing.T) { e := engine.NewInMem(engine.Attributes{}, 1<<20) localDB, err := BootstrapCluster("cluster-1", e) if err != nil { t.Fatal(err) } localDB.Close() // Set an aggressive gossip interval to make sure information is exchanged tout de suite. *gossip.GossipInterval = 10 * time.Millisecond // Start the bootstrap node. engines1 := []engine.Engine{e} addr1 := util.CreateTestAddr("tcp") server1, node1 := createTestNode(addr1, engines1, addr1, t) defer server1.Close() // Create a new node. engines2 := []engine.Engine{engine.NewInMem(engine.Attributes{}, 1<<20)} server2, node2 := createTestNode(util.CreateTestAddr("tcp"), engines2, server1.Addr(), t) defer server2.Close() // Verify new node is able to bootstrap its store. if err := util.IsTrueWithin(func() bool { return node2.localDB.GetStoreCount() == 1 }, 50*time.Millisecond); err != nil { t.Fatal(err) } // Verify node1 sees node2 via gossip and vice versa. node1Key := gossip.MakeNodeIDGossipKey(node1.Descriptor.NodeID) node2Key := gossip.MakeNodeIDGossipKey(node2.Descriptor.NodeID) if err := util.IsTrueWithin(func() bool { if val, err := node1.gossip.GetInfo(node2Key); err != nil { return false } else if val.(net.Addr).String() != server2.Addr().String() { t.Error("addr2 gossip %s doesn't match addr2 address %s", val.(net.Addr).String(), server2.Addr().String()) } if val, err := node2.gossip.GetInfo(node1Key); err != nil { return false } else if val.(net.Addr).String() != server1.Addr().String() { t.Error("addr1 gossip %s doesn't match addr1 address %s", val.(net.Addr).String(), server1.Addr().String()) } return true }, 50*time.Millisecond); err != nil { t.Error(err) } }
func TestUpdateOffsetOnHeartbeat(t *testing.T) { defer leaktest.AfterTest(t) stopper := stop.NewStopper() defer stopper.Stop() nodeContext := NewNodeTestContext(nil, stopper) serverAddr := util.CreateTestAddr("tcp") // Start heartbeat. s := NewServer(serverAddr, nodeContext) if err := s.Start(); err != nil { t.Fatal(err) } // Create a client and set its remote offset. On first heartbeat, // it will update the server's remote clocks map. We create the // client manually here to allow us to set the remote offset // before the first heartbeat. client := &Client{ addr: s.Addr(), Ready: make(chan struct{}), Closed: make(chan struct{}), clock: nodeContext.localClock, remoteClocks: nodeContext.RemoteClocks, offset: proto.RemoteOffset{ Offset: 10, Uncertainty: 5, MeasuredAt: 20, }, } if err := client.connect(nil, nodeContext); err != nil { t.Fatal(err) } nodeContext.RemoteClocks.mu.Lock() remoteAddr := client.Addr().String() o := nodeContext.RemoteClocks.offsets[remoteAddr] nodeContext.RemoteClocks.mu.Unlock() expServerOffset := proto.RemoteOffset{Offset: -10, Uncertainty: 5, MeasuredAt: 20} if o.Equal(expServerOffset) { t.Errorf("expected updated offset %v, instead %v", expServerOffset, o) } s.Close() // Remove the offset from RemoteClocks and simulate the remote end // closing the client connection. A new offset for the server should // not be added to the clock monitor. nodeContext.RemoteClocks.mu.Lock() delete(nodeContext.RemoteClocks.offsets, remoteAddr) client.Client.Close() nodeContext.RemoteClocks.mu.Unlock() nodeContext.RemoteClocks.mu.Lock() if offset, ok := nodeContext.RemoteClocks.offsets[remoteAddr]; ok { t.Errorf("unexpected updated offset: %v", offset) } nodeContext.RemoteClocks.mu.Unlock() }
// NewNetwork creates nodeCount gossip nodes. The networkType should // be set to either "tcp" or "unix". The gossipInterval should be set // to a compressed simulation timescale, though large enough to give // the concurrent goroutines enough time to pass data back and forth // in order to yield accurate estimates of how old data actually ends // up being at the various nodes (e.g. DefaultTestGossipInterval). // TODO: This method should take `stopper` as an argument. func NewNetwork(nodeCount int, networkType string, gossipInterval time.Duration) *Network { clock := hlc.NewClock(hlc.UnixNano) log.Infof("simulating gossip network with %d nodes", nodeCount) stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper) nodes := make([]*Node, nodeCount) for i := range nodes { server := rpc.NewServer(util.CreateTestAddr(networkType), rpcContext) if err := server.Start(); err != nil { log.Fatal(err) } nodes[i] = &Node{Server: server} } var numResolvers int if len(nodes) > 3 { numResolvers = 3 } else { numResolvers = len(nodes) } for i, leftNode := range nodes { // Build new resolvers for each instance or we'll get data races. var resolvers []resolver.Resolver for _, rightNode := range nodes[:numResolvers] { resolvers = append(resolvers, resolver.NewResolverFromAddress(rightNode.Server.Addr())) } gossipNode := gossip.New(rpcContext, gossipInterval, resolvers) addr := leftNode.Server.Addr() if err := gossipNode.SetNodeDescriptor(&roachpb.NodeDescriptor{ NodeID: roachpb.NodeID(i + 1), Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()), }); err != nil { log.Fatal(err) } gossipNode.Start(leftNode.Server, stopper) stopper.AddCloser(leftNode.Server) leftNode.Gossip = gossipNode } return &Network{ Nodes: nodes, NetworkType: networkType, GossipInterval: gossipInterval, Stopper: stopper, } }
// NewNetwork creates nodeCount gossip nodes. The networkType should // be set to either "tcp" or "unix". func NewNetwork(nodeCount int, networkType string) *Network { clock := hlc.NewClock(hlc.UnixNano) log.Infof("simulating gossip network with %d nodes", nodeCount) stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper) tlsConfig, err := rpcContext.GetServerTLSConfig() if err != nil { log.Fatal(err) } nodes := make([]*Node, nodeCount) for i := range nodes { server := rpc.NewServer(rpcContext) testAddr := util.CreateTestAddr(networkType) ln, err := util.ListenAndServe(stopper, server, testAddr, tlsConfig) if err != nil { log.Fatal(err) } nodes[i] = &Node{Server: server, Addr: ln.Addr()} } for i, leftNode := range nodes { // Build new resolvers for each instance or we'll get data races. resolvers := []resolver.Resolver{resolver.NewResolverFromAddress(nodes[0].Addr)} gossipNode := gossip.New(rpcContext, resolvers) addr := leftNode.Addr gossipNode.SetNodeID(roachpb.NodeID(i + 1)) if err := gossipNode.SetNodeDescriptor(&roachpb.NodeDescriptor{ NodeID: roachpb.NodeID(i + 1), Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()), }); err != nil { log.Fatal(err) } if err := gossipNode.AddInfo(addr.String(), encoding.EncodeUint64(nil, 0), time.Hour); err != nil { log.Fatal(err) } gossipNode.Start(leftNode.Server, addr, stopper) gossipNode.EnableSimulationCycler(true) leftNode.Gossip = gossipNode } return &Network{ Nodes: nodes, NetworkType: networkType, Stopper: stopper, } }
// startGossip creates local and remote gossip instances. // The remote gossip instance launches its gossip service. func startGossip(t *testing.T) (local, remote *Gossip, lserver, rserver *rpc.Server) { tlsConfig := rpc.LoadInsecureTLSConfig() laddr := util.CreateTestAddr("unix") lserver = rpc.NewServer(laddr, tlsConfig) if err := lserver.Start(); err != nil { t.Fatal(err) } local = New(tlsConfig) raddr := util.CreateTestAddr("unix") rserver = rpc.NewServer(raddr, tlsConfig) if err := rserver.Start(); err != nil { t.Fatal(err) } remote = New(tlsConfig) local.start(lserver) remote.start(rserver) time.Sleep(time.Millisecond) return }
// TestBootstrapNewStore starts a cluster with two unbootstrapped // stores and verifies both stores are added and started. func TestBootstrapNewStore(t *testing.T) { defer leaktest.AfterTest(t) engineStopper := stop.NewStopper() defer engineStopper.Stop() e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper) eagerStopper := stop.NewStopper() if _, err := BootstrapCluster("cluster-1", []engine.Engine{e}, eagerStopper); err != nil { t.Fatal(err) } eagerStopper.Stop() // Start a new node with two new stores which will require bootstrapping. engines := []engine.Engine{ e, engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper), engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper), } _, _, node, stopper := createAndStartTestNode(util.CreateTestAddr("tcp"), engines, util.CreateTestAddr("tcp"), t) defer stopper.Stop() // Non-initialized stores (in this case the new in-memory-based // store) will be bootstrapped by the node upon start. This happens // in a goroutine, so we'll have to wait a bit until we can find the // new node. util.SucceedsWithin(t, testTimeout, func() error { if n := node.stores.GetStoreCount(); n != 3 { return util.Errorf("expected 3 stores but got %d", n) } return nil }) // Check whether all stores are started properly. if err := node.stores.VisitStores(func(s *storage.Store) error { if s.IsStarted() == false { return util.Errorf("fail to start store: %s", s) } return nil }); err != nil { t.Error(err) } }
// startGossip creates local and remote gossip instances. // The remote gossip instance launches its gossip service. // We use insecure contexts since we do not have certificates for unix sockets. func startGossip(t *testing.T) (local, remote *Gossip, stopper *util.Stopper) { lclock := hlc.NewClock(hlc.UnixNano) stopper = util.NewStopper() lRPCContext := rpc.NewContext(insecureTestBaseContext, lclock, stopper) laddr := util.CreateTestAddr("unix") lserver := rpc.NewServer(laddr, lRPCContext) if err := lserver.Start(); err != nil { t.Fatal(err) } local = New(lRPCContext, gossipInterval, TestBootstrap) if err := local.SetNodeDescriptor(&proto.NodeDescriptor{ NodeID: 1, Address: proto.Addr{ Network: laddr.Network(), Address: laddr.String(), }}); err != nil { t.Fatal(err) } rclock := hlc.NewClock(hlc.UnixNano) raddr := util.CreateTestAddr("unix") rRPCContext := rpc.NewContext(insecureTestBaseContext, rclock, stopper) rserver := rpc.NewServer(raddr, rRPCContext) if err := rserver.Start(); err != nil { t.Fatal(err) } remote = New(rRPCContext, gossipInterval, TestBootstrap) if err := local.SetNodeDescriptor(&proto.NodeDescriptor{ NodeID: 2, Address: proto.Addr{ Network: raddr.Network(), Address: raddr.String(), }, }); err != nil { t.Fatal(err) } local.start(lserver, stopper) remote.start(rserver, stopper) time.Sleep(time.Millisecond) return }
func TestClientHeartbeat(t *testing.T) { addr := util.CreateTestAddr("tcp") s := NewServer(addr) s.Start() c := NewClient(s.Addr(), nil) time.Sleep(heartbeatInterval * 2) if c != NewClient(s.Addr(), nil) { t.Error("expected cached client to be returned while healthy") } <-c.Ready s.Close() }
// CreateNode creates a simulation node and starts an RPC server for it. func (n *Network) CreateNode() (*Node, error) { server := rpc.NewServer(n.rpcContext) testAddr := util.CreateTestAddr("tcp") ln, err := util.ListenAndServe(n.Stopper, server, testAddr, n.tlsConfig) if err != nil { return nil, err } node := &Node{Server: server, Addr: ln.Addr()} node.Gossip = gossip.New(n.rpcContext, nil) n.Nodes = append(n.Nodes, node) return node, nil }
// startFakeServerGossips creates local gossip instances and remote // faked gossip instance. The remote gossip instance launches its // faked gossip service just for check the client message. func startFakeServerGossips(t *testing.T) (local *Gossip, remote *fakeGossipServer, stopper *stop.Stopper) { stopper = stop.NewStopper() lclock := hlc.NewClock(hlc.UnixNano) lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper) laddr := util.CreateTestAddr("tcp") lserver := rpc.NewServer(lRPCContext) lTLSConfig, err := lRPCContext.GetServerTLSConfig() if err != nil { t.Fatal(err) } lln, err := util.ListenAndServe(stopper, lserver, laddr, lTLSConfig) if err != nil { t.Fatal(err) } local = New(lRPCContext, TestBootstrap, stopper) local.start(lserver, lln.Addr()) rclock := hlc.NewClock(hlc.UnixNano) rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper) raddr := util.CreateTestAddr("tcp") rserver := rpc.NewServer(rRPCContext) rTLSConfig, err := rRPCContext.GetServerTLSConfig() if err != nil { t.Fatal(err) } rln, err := util.ListenAndServe(stopper, rserver, raddr, rTLSConfig) if err != nil { t.Fatal(err) } if remote, err = newFakeGossipServer(rserver, stopper); err != nil { t.Fatal(err) } addr := rln.Addr() remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String()) time.Sleep(time.Millisecond) return }
func TestDuplicateRegistration(t *testing.T) { defer leaktest.AfterTest(t) stopper := stop.NewStopper() defer stopper.Stop() s := NewServer(util.CreateTestAddr("tcp"), NewNodeTestContext(nil, stopper)) heartbeat := &Heartbeat{} if err := s.Register("Foo.Bar", heartbeat.Ping, &proto.PingRequest{}); err != nil { t.Fatalf("unexpected failure on first registration: %s", s) } if err := s.Register("Foo.Bar", heartbeat.Ping, &proto.PingRequest{}); err == nil { t.Fatalf("unexpected success on second registration") } }
// TestNodeJoinSelf verifies that an uninitialized node trying to join // itself will fail. func TestNodeJoinSelf(t *testing.T) { defer leaktest.AfterTest(t) engineStopper := stop.NewStopper() defer engineStopper.Stop() engines := []engine.Engine{engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)} addr := util.CreateTestAddr("tcp") rpcServer, addr, _, node, stopper := createTestNode(addr, engines, addr, t) defer stopper.Stop() err := node.start(rpcServer, addr, engines, roachpb.Attributes{}) if err != errCannotJoinSelf { t.Fatalf("expected err %s; got %s", errCannotJoinSelf, err) } }
// TestClientRegisterInitNodeID verifies two client's gossip request with NodeID 0. func TestClientRegisterWithInitNodeID(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() // Create three gossip nodes, and connect to the first with NodeID 0. var g []*Gossip var gossipAddr string for i := 0; i < 3; i++ { clock := hlc.NewClock(hlc.UnixNano) RPCContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper) addr := util.CreateTestAddr("tcp") server := grpc.NewServer() TLSConfig, err := RPCContext.GetServerTLSConfig() if err != nil { t.Fatal(err) } ln, err := util.ListenAndServe(stopper, server, addr, TLSConfig) if err != nil { t.Fatal(err) } // Connect to the first gossip node. if gossipAddr == "" { gossipAddr = ln.Addr().String() } var resolvers []resolver.Resolver resolver, _ := resolver.NewResolver(&RPCContext.Context, gossipAddr) resolvers = append(resolvers, resolver) gnode := New(RPCContext, resolvers, stopper) // node ID must be non-zero gnode.SetNodeID(roachpb.NodeID(i + 1)) g = append(g, gnode) gnode.Start(server, ln.Addr()) } util.SucceedsSoon(t, func() error { // The first gossip node should have two gossip client address // in nodeMap if these three gossip nodes registered success. g[0].mu.Lock() defer g[0].mu.Unlock() if a, e := len(g[0].nodeMap), 2; a != e { return util.Errorf("expected %s to contain %d nodes, got %d", g[0].nodeMap, e, a) } return nil }) }
// startGossip creates local and remote gossip instances. // The remote gossip instance launches its gossip service. func startGossip(t *testing.T) (local, remote *Gossip, stopper *stop.Stopper) { lclock := hlc.NewClock(hlc.UnixNano) stopper = stop.NewStopper() lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper) laddr := util.CreateTestAddr("tcp") lserver := rpc.NewServer(laddr, lRPCContext) if err := lserver.Start(); err != nil { t.Fatal(err) } local = New(lRPCContext, gossipInterval, TestBootstrap) if err := local.SetNodeDescriptor(&proto.NodeDescriptor{ NodeID: 1, Address: util.MakeUnresolvedAddr(laddr.Network(), laddr.String()), }); err != nil { t.Fatal(err) } rclock := hlc.NewClock(hlc.UnixNano) raddr := util.CreateTestAddr("tcp") rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper) rserver := rpc.NewServer(raddr, rRPCContext) if err := rserver.Start(); err != nil { t.Fatal(err) } remote = New(rRPCContext, gossipInterval, TestBootstrap) if err := local.SetNodeDescriptor(&proto.NodeDescriptor{ NodeID: 2, Address: util.MakeUnresolvedAddr(raddr.Network(), raddr.String()), }); err != nil { t.Fatal(err) } local.start(lserver, stopper) remote.start(rserver, stopper) time.Sleep(time.Millisecond) return }
// createTestServer creates and starts a new server with a test tlsConfig and // addr. Be sure to close the server when done. Building the server manually // like this allows for manual registration of the heartbeat service. func createTestServer(serverClock *hlc.Clock, stopper *stop.Stopper, t *testing.T) *Server { // Create a test context, but override the clock. nodeContext := NewNodeTestContext(serverClock, stopper) // Create the server so that we can register a manual clock. addr := util.CreateTestAddr("tcp") s := &Server{ context: nodeContext, addr: addr, methods: map[string]method{}, } if err := s.Start(); err != nil { t.Fatal(err) } return s }