// startFakeServerGossip creates local gossip instances and remote faked gossip instance.
// The remote gossip instance launches its faked gossip service just for
// check the client message.
func startFakeServerGossip(t *testing.T) (local *Gossip, remote *fakeGossipServer, stopper *stop.Stopper) {
lclock := hlc.NewClock(hlc.UnixNano)
stopper = stop.NewStopper()
lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper)
laddr := util.CreateTestAddr("tcp")
lserver := rpc.NewServer(laddr, lRPCContext)
if err := lserver.Start(); err != nil {
t.Fatal(err)
}
local = New(lRPCContext, TestBootstrap)
local.start(lserver, stopper)
rclock := hlc.NewClock(hlc.UnixNano)
raddr := util.CreateTestAddr("tcp")
rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper)
rserver := rpc.NewServer(raddr, rRPCContext)
if err := rserver.Start(); err != nil {
t.Fatal(err)
}
remote, err := newFakeGossipServer(rserver, stopper)
if err != nil {
t.Fatal(err)
}
addr := rserver.Addr()
remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String())
time.Sleep(time.Millisecond)
return
}
// TestNodeJoin verifies a new node is able to join a bootstrapped
// cluster consisting of one node.
func TestNodeJoin(t *testing.T) {
defer leaktest.AfterTest(t)
engineStopper := stop.NewStopper()
defer engineStopper.Stop()
e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)
stopper := stop.NewStopper()
_, err := BootstrapCluster("cluster-1", []engine.Engine{e}, stopper)
if err != nil {
t.Fatal(err)
}
stopper.Stop()
// Set an aggressive gossip interval to make sure information is exchanged tout de suite.
testContext.GossipInterval = gossip.TestInterval
// Start the bootstrap node.
engines1 := []engine.Engine{e}
addr1 := util.CreateTestAddr("tcp")
server1, node1, stopper1 := createAndStartTestNode(addr1, engines1, addr1, t)
defer stopper1.Stop()
// Create a new node.
engines2 := []engine.Engine{engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)}
server2, node2, stopper2 := createAndStartTestNode(util.CreateTestAddr("tcp"), engines2, server1.Addr(), t)
defer stopper2.Stop()
// Verify new node is able to bootstrap its store.
if err := util.IsTrueWithin(func() bool { return node2.lSender.GetStoreCount() == 1 }, 50*time.Millisecond); err != nil {
t.Fatal(err)
}
// Verify node1 sees node2 via gossip and vice versa.
node1Key := gossip.MakeNodeIDKey(node1.Descriptor.NodeID)
node2Key := gossip.MakeNodeIDKey(node2.Descriptor.NodeID)
if err := util.IsTrueWithin(func() bool {
nodeDesc1 := &roachpb.NodeDescriptor{}
if err := node1.ctx.Gossip.GetInfoProto(node2Key, nodeDesc1); err != nil {
return false
}
if addr2 := nodeDesc1.Address.AddressField; addr2 != server2.Addr().String() {
t.Errorf("addr2 gossip %s doesn't match addr2 address %s", addr2, server2.Addr().String())
}
nodeDesc2 := &roachpb.NodeDescriptor{}
if err := node2.ctx.Gossip.GetInfoProto(node1Key, nodeDesc2); err != nil {
return false
}
if addr1 := nodeDesc2.Address.AddressField; addr1 != server1.Addr().String() {
t.Errorf("addr1 gossip %s doesn't match addr1 address %s", addr1, server1.Addr().String())
}
return true
}, 50*time.Millisecond); err != nil {
t.Error(err)
}
}
// startGossip creates local and remote gossip instances.
// Both remote and local instances launch the gossip service.
func startGossip(t *testing.T) (local, remote *Gossip, stopper *stop.Stopper) {
stopper = stop.NewStopper()
lclock := hlc.NewClock(hlc.UnixNano)
lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper)
laddr := util.CreateTestAddr("tcp")
lserver := rpc.NewServer(lRPCContext)
lTLSConfig, err := lRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
lln, err := util.ListenAndServe(stopper, lserver, laddr, lTLSConfig)
if err != nil {
t.Fatal(err)
}
local = New(lRPCContext, TestBootstrap)
local.SetNodeID(1)
if err := local.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: 1,
Address: util.MakeUnresolvedAddr(laddr.Network(), laddr.String()),
}); err != nil {
t.Fatal(err)
}
rclock := hlc.NewClock(hlc.UnixNano)
rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper)
raddr := util.CreateTestAddr("tcp")
rserver := rpc.NewServer(rRPCContext)
rTLSConfig, err := rRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
rln, err := util.ListenAndServe(stopper, rserver, raddr, rTLSConfig)
if err != nil {
t.Fatal(err)
}
remote = New(rRPCContext, TestBootstrap)
remote.SetNodeID(2)
if err := remote.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: 2,
Address: util.MakeUnresolvedAddr(raddr.Network(), raddr.String()),
}); err != nil {
t.Fatal(err)
}
local.start(lserver, lln.Addr(), stopper)
remote.start(rserver, rln.Addr(), stopper)
time.Sleep(time.Millisecond)
return
}
// startGossip creates local and remote gossip instances.
// The remote gossip instance launches its gossip service.
func startGossip(t *testing.T) (local, remote *Gossip, lserver, rserver *rpc.Server) {
laddr := util.CreateTestAddr("unix")
lserver = rpc.NewServer(laddr)
lserver.Start()
local = New()
raddr := util.CreateTestAddr("unix")
rserver = rpc.NewServer(raddr)
rserver.Start()
remote = New()
local.start(lserver)
remote.start(rserver)
time.Sleep(time.Millisecond)
return
}
// TestNodeJoin verifies a new node is able to join a bootstrapped
// cluster consisting of one node.
func TestNodeJoin(t *testing.T) {
defer leaktest.AfterTest(t)()
engineStopper := stop.NewStopper()
defer engineStopper.Stop()
e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)
if _, err := bootstrapCluster([]engine.Engine{e}, kv.NewTxnMetrics(metric.NewRegistry())); err != nil {
t.Fatal(err)
}
// Start the bootstrap node.
engines1 := []engine.Engine{e}
addr1 := util.CreateTestAddr("tcp")
_, server1Addr, node1, stopper1 := createAndStartTestNode(addr1, engines1, addr1, t)
defer stopper1.Stop()
// Create a new node.
engines2 := []engine.Engine{engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)}
addr2 := util.CreateTestAddr("tcp")
_, server2Addr, node2, stopper2 := createAndStartTestNode(addr2, engines2, server1Addr, t)
defer stopper2.Stop()
// Verify new node is able to bootstrap its store.
util.SucceedsSoon(t, func() error {
if sc := node2.stores.GetStoreCount(); sc != 1 {
return util.Errorf("GetStoreCount() expected 1; got %d", sc)
}
return nil
})
// Verify node1 sees node2 via gossip and vice versa.
node1Key := gossip.MakeNodeIDKey(node1.Descriptor.NodeID)
node2Key := gossip.MakeNodeIDKey(node2.Descriptor.NodeID)
util.SucceedsSoon(t, func() error {
var nodeDesc1 roachpb.NodeDescriptor
if err := node1.ctx.Gossip.GetInfoProto(node2Key, &nodeDesc1); err != nil {
return err
}
if addr2Str, server2AddrStr := nodeDesc1.Address.String(), server2Addr.String(); addr2Str != server2AddrStr {
return util.Errorf("addr2 gossip %s doesn't match addr2 address %s", addr2Str, server2AddrStr)
}
var nodeDesc2 roachpb.NodeDescriptor
if err := node2.ctx.Gossip.GetInfoProto(node1Key, &nodeDesc2); err != nil {
return err
}
if addr1Str, server1AddrStr := nodeDesc2.Address.String(), server1Addr.String(); addr1Str != server1AddrStr {
return util.Errorf("addr1 gossip %s doesn't match addr1 address %s", addr1Str, server1AddrStr)
}
return nil
})
}
// TestCorruptedClusterID verifies that a node fails to start when a
// store's cluster ID is empty.
func TestCorruptedClusterID(t *testing.T) {
defer leaktest.AfterTest(t)
engineStopper := stop.NewStopper()
e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)
defer engineStopper.Stop()
if _, err := bootstrapCluster([]engine.Engine{e}); err != nil {
t.Fatal(err)
}
// Set the cluster ID to the empty UUID.
sIdent := roachpb.StoreIdent{
ClusterID: *uuid.EmptyUUID,
NodeID: 1,
StoreID: 1,
}
if err := engine.MVCCPutProto(e, nil, keys.StoreIdentKey(), roachpb.ZeroTimestamp, nil, &sIdent); err != nil {
t.Fatal(err)
}
engines := []engine.Engine{e}
server, serverAddr, _, node, stopper := createTestNode(util.CreateTestAddr("tcp"), engines, nil, t)
stopper.Stop()
if err := node.start(server, serverAddr, engines, roachpb.Attributes{}); !testutils.IsError(err, "unidentified store") {
t.Errorf("unexpected error %v", err)
}
}
// createAndStartNewServer creates and starts a new server with a test address.
func createAndStartNewServer(t *testing.T, ctx *Context) *Server {
s := NewServer(util.CreateTestAddr("tcp"), ctx)
if err := s.Start(); err != nil {
t.Fatal(err)
}
return s
}
// startGossip creates and starts a gossip instance.
func startGossip(nodeID roachpb.NodeID, stopper *stop.Stopper, t *testing.T) *Gossip {
clock := hlc.NewClock(hlc.UnixNano)
rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper)
addr := util.CreateTestAddr("tcp")
server := rpc.NewServer(rpcContext)
tlsConfig, err := rpcContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
ln, err := util.ListenAndServe(stopper, server, addr, tlsConfig)
if err != nil {
t.Fatal(err)
}
g := New(rpcContext, TestBootstrap, stopper)
g.SetNodeID(nodeID)
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: nodeID,
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}); err != nil {
t.Fatal(err)
}
g.start(server, ln.Addr())
time.Sleep(time.Millisecond)
return g
}
func newTestServer(t *testing.T, ctx *Context, manual bool) (*Server, net.Listener) {
var s *Server
if manual {
s = &Server{
insecure: ctx.Insecure,
activeConns: make(map[net.Conn]struct{}),
methods: map[string]method{},
}
} else {
s = NewServer(ctx)
}
tlsConfig, err := ctx.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
addr := util.CreateTestAddr("tcp")
ln, err := util.ListenAndServe(ctx.Stopper, s, addr, tlsConfig)
if err != nil {
t.Fatal(err)
}
return s, ln
}
// TestCorruptedClusterID verifies that a node fails to start when a
// store's cluster ID is empty.
func TestCorruptedClusterID(t *testing.T) {
defer leaktest.AfterTest(t)
eagerStopper := stop.NewStopper()
e := engine.NewInMem(proto.Attributes{}, 1<<20)
_, err := BootstrapCluster("cluster-1", []engine.Engine{e}, eagerStopper)
if err != nil {
t.Fatal(err)
}
eagerStopper.Stop()
// Set the cluster ID to an empty string.
sIdent := proto.StoreIdent{
ClusterID: "",
NodeID: 1,
StoreID: 1,
}
if err = engine.MVCCPutProto(e, nil, keys.StoreIdentKey(), proto.ZeroTimestamp, nil, &sIdent); err != nil {
t.Fatal(err)
}
engines := []engine.Engine{e}
server, _, node, stopper := createTestNode(util.CreateTestAddr("tcp"), engines, nil, t)
if err := node.start(server, engines, proto.Attributes{}, stopper); err == nil {
t.Errorf("unexpected success")
}
stopper.Stop()
}
// TestClientHeartbeatBadServer verifies that the client is not marked
// as "ready" until a heartbeat request succeeds.
func TestClientHeartbeatBadServer(t *testing.T) {
tlsConfig, err := LoadTestTLSConfig("..")
if err != nil {
t.Fatal(err)
}
addr := util.CreateTestAddr("tcp")
// Create a server which doesn't support heartbeats.
s := &Server{
Server: rpc.NewServer(),
tlsConfig: tlsConfig,
addr: addr,
closeCallbacks: make([]func(conn net.Conn), 0, 1),
}
if err := s.Start(); err != nil {
t.Fatal(err)
}
// Now, create a client. It should attempt a heartbeat and fail,
// causing retry loop to activate.
c := NewClient(s.Addr(), nil, tlsConfig)
select {
case <-c.Ready:
t.Error("unexpected client heartbeat success")
case <-c.Closed:
}
s.Close()
}
// TestBootstrapNewStore starts a cluster with two unbootstrapped
// stores and verifies both stores are added.
func TestBootstrapNewStore(t *testing.T) {
e := engine.NewInMem(engine.Attributes{}, 1<<20)
localDB, err := BootstrapCluster("cluster-1", e)
if err != nil {
t.Fatal(err)
}
localDB.Close()
// Start a new node with two new stores which will require bootstrapping.
engines := []engine.Engine{
e,
engine.NewInMem(engine.Attributes{}, 1<<20),
engine.NewInMem(engine.Attributes{}, 1<<20),
}
server, node := createTestNode(util.CreateTestAddr("tcp"), engines, nil, t)
defer server.Close()
// Non-initialized stores (in this case the new in-memory-based
// store) will be bootstrapped by the node upon start. This happens
// in a goroutine, so we'll have to wait a bit (maximum 10ms) until
// we can find the new node.
if err := util.IsTrueWithin(func() bool { return node.localDB.GetStoreCount() == 3 }, 50*time.Millisecond); err != nil {
t.Error(err)
}
}
func (lt *localRPCTransport) Listen(id roachpb.StoreID, server ServerInterface) error {
addr := util.CreateTestAddr("tcp")
rpcServer := crpc.NewServer(addr, &crpc.Context{
Context: base.Context{
Insecure: true,
},
Stopper: lt.stopper,
DisableCache: true,
})
err := rpcServer.RegisterAsync(raftMessageName, false, /*not public*/
func(argsI proto.Message, callback func(proto.Message, error)) {
args := argsI.(*RaftMessageRequest)
resp, err := server.RaftMessage(args)
callback(resp, err)
}, &RaftMessageRequest{})
if err != nil {
return err
}
lt.mu.Lock()
if _, ok := lt.servers[id]; ok {
log.Fatalf("node %d already listening", id)
}
lt.servers[id] = rpcServer
lt.mu.Unlock()
return rpcServer.Start()
}
func (lt *localRPCTransport) Listen(id proto.RaftNodeID, server ServerInterface) error {
addr := util.CreateTestAddr("tcp")
rpcServer := crpc.NewServer(addr, &crpc.Context{
Context: base.Context{
Insecure: true,
},
Stopper: lt.stopper,
DisableCache: true,
})
err := rpcServer.RegisterAsync(raftMessageName,
func(argsI gogoproto.Message, callback func(gogoproto.Message, error)) {
protoArgs := argsI.(*proto.RaftMessageRequest)
args := RaftMessageRequest{
GroupID: protoArgs.GroupID,
}
if err := args.Message.Unmarshal(protoArgs.Msg); err != nil {
callback(nil, err)
}
err := server.RaftMessage(&args, &RaftMessageResponse{})
callback(&proto.RaftMessageResponse{}, err)
}, &proto.RaftMessageRequest{})
if err != nil {
return err
}
lt.mu.Lock()
if _, ok := lt.servers[id]; ok {
log.Fatalf("node %d already listening", id)
}
lt.servers[id] = rpcServer
lt.mu.Unlock()
return rpcServer.Start()
}
// SimulateNetwork creates nodeCount gossip nodes. The network should
// be set to either "tcp" or "unix". The gossipInterval should be set
// to a compressed simulation timescale, though large enough to give
// the concurrent goroutines enough time to pass data back and forth
// in order to yield accurate estimates of how old data actually ends
// up being at the various nodes. After each gossipInterval period,
// simCallback is invoked; when it returns false, the simulation
// ends. If it returns true, the simulation continues another cycle.
//
// Node0 gossips the node count as well as the gossip sentinel. The
// gossip bootstrap hosts are set to the first three nodes (or fewer if
// less than three are available).
//
// At each cycle of the simulation, node 0 gossips the sentinel. If
// the simulation requires other nodes to gossip, this should be done
// via simCallback.
//
// The simulation callback receives a map of nodes, keyed by node address.
func SimulateNetwork(nodeCount int, network string, gossipInterval time.Duration,
simCallback func(cycle int, nodes map[string]*Gossip) bool) {
// seed the random number generator for non-determinism across
// multiple runs.
rand.Seed(time.Now().UTC().UnixNano())
tlsConfig := rpc.LoadInsecureTLSConfig()
log.Infof("simulating network with %d nodes", nodeCount)
servers := make([]*rpc.Server, nodeCount)
addrs := make([]net.Addr, nodeCount)
for i := 0; i < nodeCount; i++ {
addr := util.CreateTestAddr(network)
servers[i] = rpc.NewServer(addr, tlsConfig)
if err := servers[i].Start(); err != nil {
log.Fatal(err)
}
addrs[i] = servers[i].Addr()
}
var bootstrap []net.Addr
if nodeCount < 3 {
bootstrap = addrs
} else {
bootstrap = addrs[:3]
}
nodes := make(map[string]*Gossip, nodeCount)
for i := 0; i < nodeCount; i++ {
node := New(tlsConfig)
node.Name = fmt.Sprintf("Node%d", i)
node.SetBootstrap(bootstrap)
node.SetInterval(gossipInterval)
node.Start(servers[i])
// Node 0 gossips node count.
if i == 0 {
node.AddInfo(KeyNodeCount, int64(nodeCount), time.Hour)
}
nodes[addrs[i].String()] = node
}
gossipTimeout := time.Tick(gossipInterval)
var complete bool
for cycle := 0; !complete; cycle++ {
select {
case <-gossipTimeout:
// Node 0 gossips sentinel every cycle.
nodes[addrs[0].String()].AddInfo(KeySentinel, int64(cycle), time.Hour)
if !simCallback(cycle, nodes) {
complete = true
}
}
}
// Stop all servers & nodes.
for i := 0; i < nodeCount; i++ {
servers[i].Close()
nodes[addrs[i].String()].Stop()
}
}
// TestNodeJoin verifies a new node is able to join a bootstrapped
// cluster consisting of one node.
func TestNodeJoin(t *testing.T) {
e := engine.NewInMem(engine.Attributes{}, 1<<20)
localDB, err := BootstrapCluster("cluster-1", e)
if err != nil {
t.Fatal(err)
}
localDB.Close()
// Set an aggressive gossip interval to make sure information is exchanged tout de suite.
*gossip.GossipInterval = 10 * time.Millisecond
// Start the bootstrap node.
engines1 := []engine.Engine{e}
addr1 := util.CreateTestAddr("tcp")
server1, node1 := createTestNode(addr1, engines1, addr1, t)
defer server1.Close()
// Create a new node.
engines2 := []engine.Engine{engine.NewInMem(engine.Attributes{}, 1<<20)}
server2, node2 := createTestNode(util.CreateTestAddr("tcp"), engines2, server1.Addr(), t)
defer server2.Close()
// Verify new node is able to bootstrap its store.
if err := util.IsTrueWithin(func() bool { return node2.localDB.GetStoreCount() == 1 }, 50*time.Millisecond); err != nil {
t.Fatal(err)
}
// Verify node1 sees node2 via gossip and vice versa.
node1Key := gossip.MakeNodeIDGossipKey(node1.Descriptor.NodeID)
node2Key := gossip.MakeNodeIDGossipKey(node2.Descriptor.NodeID)
if err := util.IsTrueWithin(func() bool {
if val, err := node1.gossip.GetInfo(node2Key); err != nil {
return false
} else if val.(net.Addr).String() != server2.Addr().String() {
t.Error("addr2 gossip %s doesn't match addr2 address %s", val.(net.Addr).String(), server2.Addr().String())
}
if val, err := node2.gossip.GetInfo(node1Key); err != nil {
return false
} else if val.(net.Addr).String() != server1.Addr().String() {
t.Error("addr1 gossip %s doesn't match addr1 address %s", val.(net.Addr).String(), server1.Addr().String())
}
return true
}, 50*time.Millisecond); err != nil {
t.Error(err)
}
}
func TestUpdateOffsetOnHeartbeat(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
nodeContext := NewNodeTestContext(nil, stopper)
serverAddr := util.CreateTestAddr("tcp")
// Start heartbeat.
s := NewServer(serverAddr, nodeContext)
if err := s.Start(); err != nil {
t.Fatal(err)
}
// Create a client and set its remote offset. On first heartbeat,
// it will update the server's remote clocks map. We create the
// client manually here to allow us to set the remote offset
// before the first heartbeat.
client := &Client{
addr: s.Addr(),
Ready: make(chan struct{}),
Closed: make(chan struct{}),
clock: nodeContext.localClock,
remoteClocks: nodeContext.RemoteClocks,
offset: proto.RemoteOffset{
Offset: 10,
Uncertainty: 5,
MeasuredAt: 20,
},
}
if err := client.connect(nil, nodeContext); err != nil {
t.Fatal(err)
}
nodeContext.RemoteClocks.mu.Lock()
remoteAddr := client.Addr().String()
o := nodeContext.RemoteClocks.offsets[remoteAddr]
nodeContext.RemoteClocks.mu.Unlock()
expServerOffset := proto.RemoteOffset{Offset: -10, Uncertainty: 5, MeasuredAt: 20}
if o.Equal(expServerOffset) {
t.Errorf("expected updated offset %v, instead %v", expServerOffset, o)
}
s.Close()
// Remove the offset from RemoteClocks and simulate the remote end
// closing the client connection. A new offset for the server should
// not be added to the clock monitor.
nodeContext.RemoteClocks.mu.Lock()
delete(nodeContext.RemoteClocks.offsets, remoteAddr)
client.Client.Close()
nodeContext.RemoteClocks.mu.Unlock()
nodeContext.RemoteClocks.mu.Lock()
if offset, ok := nodeContext.RemoteClocks.offsets[remoteAddr]; ok {
t.Errorf("unexpected updated offset: %v", offset)
}
nodeContext.RemoteClocks.mu.Unlock()
}
// NewNetwork creates nodeCount gossip nodes. The networkType should
// be set to either "tcp" or "unix". The gossipInterval should be set
// to a compressed simulation timescale, though large enough to give
// the concurrent goroutines enough time to pass data back and forth
// in order to yield accurate estimates of how old data actually ends
// up being at the various nodes (e.g. DefaultTestGossipInterval).
// TODO: This method should take `stopper` as an argument.
func NewNetwork(nodeCount int, networkType string,
gossipInterval time.Duration) *Network {
clock := hlc.NewClock(hlc.UnixNano)
log.Infof("simulating gossip network with %d nodes", nodeCount)
stopper := stop.NewStopper()
rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper)
nodes := make([]*Node, nodeCount)
for i := range nodes {
server := rpc.NewServer(util.CreateTestAddr(networkType), rpcContext)
if err := server.Start(); err != nil {
log.Fatal(err)
}
nodes[i] = &Node{Server: server}
}
var numResolvers int
if len(nodes) > 3 {
numResolvers = 3
} else {
numResolvers = len(nodes)
}
for i, leftNode := range nodes {
// Build new resolvers for each instance or we'll get data races.
var resolvers []resolver.Resolver
for _, rightNode := range nodes[:numResolvers] {
resolvers = append(resolvers, resolver.NewResolverFromAddress(rightNode.Server.Addr()))
}
gossipNode := gossip.New(rpcContext, gossipInterval, resolvers)
addr := leftNode.Server.Addr()
if err := gossipNode.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(i + 1),
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}); err != nil {
log.Fatal(err)
}
gossipNode.Start(leftNode.Server, stopper)
stopper.AddCloser(leftNode.Server)
leftNode.Gossip = gossipNode
}
return &Network{
Nodes: nodes,
NetworkType: networkType,
GossipInterval: gossipInterval,
Stopper: stopper,
}
}
// NewNetwork creates nodeCount gossip nodes. The networkType should
// be set to either "tcp" or "unix".
func NewNetwork(nodeCount int, networkType string) *Network {
clock := hlc.NewClock(hlc.UnixNano)
log.Infof("simulating gossip network with %d nodes", nodeCount)
stopper := stop.NewStopper()
rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper)
tlsConfig, err := rpcContext.GetServerTLSConfig()
if err != nil {
log.Fatal(err)
}
nodes := make([]*Node, nodeCount)
for i := range nodes {
server := rpc.NewServer(rpcContext)
testAddr := util.CreateTestAddr(networkType)
ln, err := util.ListenAndServe(stopper, server, testAddr, tlsConfig)
if err != nil {
log.Fatal(err)
}
nodes[i] = &Node{Server: server, Addr: ln.Addr()}
}
for i, leftNode := range nodes {
// Build new resolvers for each instance or we'll get data races.
resolvers := []resolver.Resolver{resolver.NewResolverFromAddress(nodes[0].Addr)}
gossipNode := gossip.New(rpcContext, resolvers)
addr := leftNode.Addr
gossipNode.SetNodeID(roachpb.NodeID(i + 1))
if err := gossipNode.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(i + 1),
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}); err != nil {
log.Fatal(err)
}
if err := gossipNode.AddInfo(addr.String(), encoding.EncodeUint64(nil, 0), time.Hour); err != nil {
log.Fatal(err)
}
gossipNode.Start(leftNode.Server, addr, stopper)
gossipNode.EnableSimulationCycler(true)
leftNode.Gossip = gossipNode
}
return &Network{
Nodes: nodes,
NetworkType: networkType,
Stopper: stopper,
}
}
// startGossip creates local and remote gossip instances.
// The remote gossip instance launches its gossip service.
func startGossip(t *testing.T) (local, remote *Gossip, lserver, rserver *rpc.Server) {
tlsConfig := rpc.LoadInsecureTLSConfig()
laddr := util.CreateTestAddr("unix")
lserver = rpc.NewServer(laddr, tlsConfig)
if err := lserver.Start(); err != nil {
t.Fatal(err)
}
local = New(tlsConfig)
raddr := util.CreateTestAddr("unix")
rserver = rpc.NewServer(raddr, tlsConfig)
if err := rserver.Start(); err != nil {
t.Fatal(err)
}
remote = New(tlsConfig)
local.start(lserver)
remote.start(rserver)
time.Sleep(time.Millisecond)
return
}
// TestBootstrapNewStore starts a cluster with two unbootstrapped
// stores and verifies both stores are added and started.
func TestBootstrapNewStore(t *testing.T) {
defer leaktest.AfterTest(t)
engineStopper := stop.NewStopper()
defer engineStopper.Stop()
e := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)
eagerStopper := stop.NewStopper()
if _, err := BootstrapCluster("cluster-1", []engine.Engine{e}, eagerStopper); err != nil {
t.Fatal(err)
}
eagerStopper.Stop()
// Start a new node with two new stores which will require bootstrapping.
engines := []engine.Engine{
e,
engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper),
engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper),
}
_, _, node, stopper := createAndStartTestNode(util.CreateTestAddr("tcp"), engines, util.CreateTestAddr("tcp"), t)
defer stopper.Stop()
// Non-initialized stores (in this case the new in-memory-based
// store) will be bootstrapped by the node upon start. This happens
// in a goroutine, so we'll have to wait a bit until we can find the
// new node.
util.SucceedsWithin(t, testTimeout, func() error {
if n := node.stores.GetStoreCount(); n != 3 {
return util.Errorf("expected 3 stores but got %d", n)
}
return nil
})
// Check whether all stores are started properly.
if err := node.stores.VisitStores(func(s *storage.Store) error {
if s.IsStarted() == false {
return util.Errorf("fail to start store: %s", s)
}
return nil
}); err != nil {
t.Error(err)
}
}
// startGossip creates local and remote gossip instances.
// The remote gossip instance launches its gossip service.
// We use insecure contexts since we do not have certificates for unix sockets.
func startGossip(t *testing.T) (local, remote *Gossip, stopper *util.Stopper) {
lclock := hlc.NewClock(hlc.UnixNano)
stopper = util.NewStopper()
lRPCContext := rpc.NewContext(insecureTestBaseContext, lclock, stopper)
laddr := util.CreateTestAddr("unix")
lserver := rpc.NewServer(laddr, lRPCContext)
if err := lserver.Start(); err != nil {
t.Fatal(err)
}
local = New(lRPCContext, gossipInterval, TestBootstrap)
if err := local.SetNodeDescriptor(&proto.NodeDescriptor{
NodeID: 1,
Address: proto.Addr{
Network: laddr.Network(),
Address: laddr.String(),
}}); err != nil {
t.Fatal(err)
}
rclock := hlc.NewClock(hlc.UnixNano)
raddr := util.CreateTestAddr("unix")
rRPCContext := rpc.NewContext(insecureTestBaseContext, rclock, stopper)
rserver := rpc.NewServer(raddr, rRPCContext)
if err := rserver.Start(); err != nil {
t.Fatal(err)
}
remote = New(rRPCContext, gossipInterval, TestBootstrap)
if err := local.SetNodeDescriptor(&proto.NodeDescriptor{
NodeID: 2,
Address: proto.Addr{
Network: raddr.Network(),
Address: raddr.String(),
},
}); err != nil {
t.Fatal(err)
}
local.start(lserver, stopper)
remote.start(rserver, stopper)
time.Sleep(time.Millisecond)
return
}
func TestClientHeartbeat(t *testing.T) {
addr := util.CreateTestAddr("tcp")
s := NewServer(addr)
s.Start()
c := NewClient(s.Addr(), nil)
time.Sleep(heartbeatInterval * 2)
if c != NewClient(s.Addr(), nil) {
t.Error("expected cached client to be returned while healthy")
}
<-c.Ready
s.Close()
}
// CreateNode creates a simulation node and starts an RPC server for it.
func (n *Network) CreateNode() (*Node, error) {
server := rpc.NewServer(n.rpcContext)
testAddr := util.CreateTestAddr("tcp")
ln, err := util.ListenAndServe(n.Stopper, server, testAddr, n.tlsConfig)
if err != nil {
return nil, err
}
node := &Node{Server: server, Addr: ln.Addr()}
node.Gossip = gossip.New(n.rpcContext, nil)
n.Nodes = append(n.Nodes, node)
return node, nil
}
// startFakeServerGossips creates local gossip instances and remote
// faked gossip instance. The remote gossip instance launches its
// faked gossip service just for check the client message.
func startFakeServerGossips(t *testing.T) (local *Gossip, remote *fakeGossipServer, stopper *stop.Stopper) {
stopper = stop.NewStopper()
lclock := hlc.NewClock(hlc.UnixNano)
lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper)
laddr := util.CreateTestAddr("tcp")
lserver := rpc.NewServer(lRPCContext)
lTLSConfig, err := lRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
lln, err := util.ListenAndServe(stopper, lserver, laddr, lTLSConfig)
if err != nil {
t.Fatal(err)
}
local = New(lRPCContext, TestBootstrap, stopper)
local.start(lserver, lln.Addr())
rclock := hlc.NewClock(hlc.UnixNano)
rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper)
raddr := util.CreateTestAddr("tcp")
rserver := rpc.NewServer(rRPCContext)
rTLSConfig, err := rRPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
rln, err := util.ListenAndServe(stopper, rserver, raddr, rTLSConfig)
if err != nil {
t.Fatal(err)
}
if remote, err = newFakeGossipServer(rserver, stopper); err != nil {
t.Fatal(err)
}
addr := rln.Addr()
remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String())
time.Sleep(time.Millisecond)
return
}
func TestDuplicateRegistration(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
s := NewServer(util.CreateTestAddr("tcp"), NewNodeTestContext(nil, stopper))
heartbeat := &Heartbeat{}
if err := s.Register("Foo.Bar", heartbeat.Ping, &proto.PingRequest{}); err != nil {
t.Fatalf("unexpected failure on first registration: %s", s)
}
if err := s.Register("Foo.Bar", heartbeat.Ping, &proto.PingRequest{}); err == nil {
t.Fatalf("unexpected success on second registration")
}
}
// TestNodeJoinSelf verifies that an uninitialized node trying to join
// itself will fail.
func TestNodeJoinSelf(t *testing.T) {
defer leaktest.AfterTest(t)
engineStopper := stop.NewStopper()
defer engineStopper.Stop()
engines := []engine.Engine{engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)}
addr := util.CreateTestAddr("tcp")
rpcServer, addr, _, node, stopper := createTestNode(addr, engines, addr, t)
defer stopper.Stop()
err := node.start(rpcServer, addr, engines, roachpb.Attributes{})
if err != errCannotJoinSelf {
t.Fatalf("expected err %s; got %s", errCannotJoinSelf, err)
}
}
// TestClientRegisterInitNodeID verifies two client's gossip request with NodeID 0.
func TestClientRegisterWithInitNodeID(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
// Create three gossip nodes, and connect to the first with NodeID 0.
var g []*Gossip
var gossipAddr string
for i := 0; i < 3; i++ {
clock := hlc.NewClock(hlc.UnixNano)
RPCContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper)
addr := util.CreateTestAddr("tcp")
server := grpc.NewServer()
TLSConfig, err := RPCContext.GetServerTLSConfig()
if err != nil {
t.Fatal(err)
}
ln, err := util.ListenAndServe(stopper, server, addr, TLSConfig)
if err != nil {
t.Fatal(err)
}
// Connect to the first gossip node.
if gossipAddr == "" {
gossipAddr = ln.Addr().String()
}
var resolvers []resolver.Resolver
resolver, _ := resolver.NewResolver(&RPCContext.Context, gossipAddr)
resolvers = append(resolvers, resolver)
gnode := New(RPCContext, resolvers, stopper)
// node ID must be non-zero
gnode.SetNodeID(roachpb.NodeID(i + 1))
g = append(g, gnode)
gnode.Start(server, ln.Addr())
}
util.SucceedsSoon(t, func() error {
// The first gossip node should have two gossip client address
// in nodeMap if these three gossip nodes registered success.
g[0].mu.Lock()
defer g[0].mu.Unlock()
if a, e := len(g[0].nodeMap), 2; a != e {
return util.Errorf("expected %s to contain %d nodes, got %d", g[0].nodeMap, e, a)
}
return nil
})
}
// startGossip creates local and remote gossip instances.
// The remote gossip instance launches its gossip service.
func startGossip(t *testing.T) (local, remote *Gossip, stopper *stop.Stopper) {
lclock := hlc.NewClock(hlc.UnixNano)
stopper = stop.NewStopper()
lRPCContext := rpc.NewContext(&base.Context{Insecure: true}, lclock, stopper)
laddr := util.CreateTestAddr("tcp")
lserver := rpc.NewServer(laddr, lRPCContext)
if err := lserver.Start(); err != nil {
t.Fatal(err)
}
local = New(lRPCContext, gossipInterval, TestBootstrap)
if err := local.SetNodeDescriptor(&proto.NodeDescriptor{
NodeID: 1,
Address: util.MakeUnresolvedAddr(laddr.Network(), laddr.String()),
}); err != nil {
t.Fatal(err)
}
rclock := hlc.NewClock(hlc.UnixNano)
raddr := util.CreateTestAddr("tcp")
rRPCContext := rpc.NewContext(&base.Context{Insecure: true}, rclock, stopper)
rserver := rpc.NewServer(raddr, rRPCContext)
if err := rserver.Start(); err != nil {
t.Fatal(err)
}
remote = New(rRPCContext, gossipInterval, TestBootstrap)
if err := local.SetNodeDescriptor(&proto.NodeDescriptor{
NodeID: 2,
Address: util.MakeUnresolvedAddr(raddr.Network(), raddr.String()),
}); err != nil {
t.Fatal(err)
}
local.start(lserver, stopper)
remote.start(rserver, stopper)
time.Sleep(time.Millisecond)
return
}
// createTestServer creates and starts a new server with a test tlsConfig and
// addr. Be sure to close the server when done. Building the server manually
// like this allows for manual registration of the heartbeat service.
func createTestServer(serverClock *hlc.Clock, stopper *stop.Stopper, t *testing.T) *Server {
// Create a test context, but override the clock.
nodeContext := NewNodeTestContext(serverClock, stopper)
// Create the server so that we can register a manual clock.
addr := util.CreateTestAddr("tcp")
s := &Server{
context: nodeContext,
addr: addr,
methods: map[string]method{},
}
if err := s.Start(); err != nil {
t.Fatal(err)
}
return s
}
