// TestGossipStorageCleanup verifies that bad resolvers are purged
// from the bootstrap info after gossip has successfully connected.
func TestGossipStorageCleanup(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
const numNodes = 3
network := simulation.NewNetwork(stopper, numNodes, false)
const notReachableAddr = "localhost:0"
const invalidAddr = "10.0.0.1000:3333333"
// Set storage for each of the nodes.
addresses := make(unresolvedAddrSlice, len(network.Nodes))
stores := make([]testStorage, len(network.Nodes))
for i, n := range network.Nodes {
addresses[i] = util.MakeUnresolvedAddr(n.Addr().Network(), n.Addr().String())
// Pre-add an invalid address to each gossip storage.
if err := stores[i].WriteBootstrapInfo(&gossip.BootstrapInfo{
Addresses: []util.UnresolvedAddr{
util.MakeUnresolvedAddr("tcp", network.Nodes[(i+1)%numNodes].Addr().String()), // node i+1 address
util.MakeUnresolvedAddr("tcp", notReachableAddr), // unreachable address
util.MakeUnresolvedAddr("tcp", invalidAddr), // invalid address
},
}); err != nil {
t.Fatal(err)
}
if err := n.Gossip.SetStorage(&stores[i]); err != nil {
t.Fatal(err)
}
n.Gossip.SetStallInterval(1 * time.Millisecond)
n.Gossip.SetBootstrapInterval(1 * time.Millisecond)
}
// Wait for the gossip network to connect.
network.RunUntilFullyConnected()
// Let the gossip network continue running in the background without the
// simulation cycler preventing it from operating.
for _, node := range network.Nodes {
node.Gossip.EnableSimulationCycler(false)
}
// Wait long enough for storage to get the expected number of
// addresses and no pending cleanups.
testutils.SucceedsSoon(t, func() error {
for i := range stores {
p := &stores[i]
if expected, actual := len(network.Nodes)-1 /* -1 is ourself */, p.Len(); expected != actual {
return errors.Errorf("expected %v, got %v (info: %#v)", expected, actual, p.Info().Addresses)
}
for _, addr := range p.Info().Addresses {
if addr.String() == invalidAddr {
return errors.Errorf("node %d still needs bootstrap cleanup", i)
}
}
}
return nil
})
}
// TestStoresGossipStorageReadLatest verifies that the latest
// bootstrap info from multiple stores is returned on Read.
func TestStoresGossipStorageReadLatest(t *testing.T) {
defer leaktest.AfterTest(t)()
manual, stores, ls, stopper := createStores(2, t)
defer stopper.Stop()
ls.AddStore(stores[0])
// Set clock to 1.
manual.Set(1)
// Add a fake address and write.
var bi gossip.BootstrapInfo
bi.Addresses = append(bi.Addresses, util.MakeUnresolvedAddr("tcp", "127.0.0.1:8001"))
if err := ls.WriteBootstrapInfo(&bi); err != nil {
t.Fatal(err)
}
// Now remove store 0 and add store 1.
ls.RemoveStore(stores[0])
ls.AddStore(stores[1])
// Increment clock, add another address and write.
manual.Increment(1)
bi.Addresses = append(bi.Addresses, util.MakeUnresolvedAddr("tcp", "127.0.0.1:8002"))
if err := ls.WriteBootstrapInfo(&bi); err != nil {
t.Fatal(err)
}
// Create a new stores object to freshly read. Should get latest
// version from store 1.
manual.Increment(1)
ls2 := NewStores(log.AmbientContext{}, ls.clock)
ls2.AddStore(stores[0])
ls2.AddStore(stores[1])
var verifyBI gossip.BootstrapInfo
if err := ls2.ReadBootstrapInfo(&verifyBI); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(bi, verifyBI) {
t.Errorf("bootstrap info %+v not equal to expected %+v", verifyBI, bi)
}
// Verify that stores[0], which had old info, was updated with
// latest bootstrap info during the read.
ls3 := NewStores(log.AmbientContext{}, ls.clock)
ls3.AddStore(stores[0])
verifyBI.Reset()
if err := ls2.ReadBootstrapInfo(&verifyBI); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(bi, verifyBI) {
t.Errorf("bootstrap info %+v not equal to expected %+v", verifyBI, bi)
}
}
func startGossipAtAddr(
nodeID roachpb.NodeID,
addr net.Addr,
stopper *stop.Stopper,
t *testing.T,
registry *metric.Registry,
) *Gossip {
rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper)
server := rpc.NewServer(rpcContext)
g := NewTest(nodeID, rpcContext, server, nil, stopper, registry)
ln, err := netutil.ListenAndServeGRPC(stopper, server, addr)
if err != nil {
t.Fatal(err)
}
addr = ln.Addr()
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: nodeID,
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}); err != nil {
t.Fatal(err)
}
g.start(addr)
time.Sleep(time.Millisecond)
return g
}
// startFakeServerGossips creates local gossip instances and remote
// faked gossip instance. The remote gossip instance launches its
// faked gossip service just for check the client message.
func startFakeServerGossips(
t *testing.T, localNodeID roachpb.NodeID,
) (*Gossip, *fakeGossipServer, *stop.Stopper) {
stopper := stop.NewStopper()
lRPCContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper)
lserver := rpc.NewServer(lRPCContext)
local := NewTest(localNodeID, lRPCContext, lserver, nil, stopper, metric.NewRegistry())
lln, err := netutil.ListenAndServeGRPC(stopper, lserver, util.IsolatedTestAddr)
if err != nil {
t.Fatal(err)
}
local.start(lln.Addr())
rRPCContext := rpc.NewContext(log.AmbientContext{}, &base.Config{Insecure: true}, nil, stopper)
rserver := rpc.NewServer(rRPCContext)
rln, err := netutil.ListenAndServeGRPC(stopper, rserver, util.IsolatedTestAddr)
if err != nil {
t.Fatal(err)
}
remote := newFakeGossipServer(rserver, stopper)
addr := rln.Addr()
remote.nodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String())
return local, remote, stopper
}
// start initializes the infostore with the rpc server address and
// then begins processing connecting clients in an infinite select
// loop via goroutine. Periodically, clients connected and awaiting
// the next round of gossip are awoken via the conditional variable.
func (s *server) start(addr net.Addr) {
s.mu.Lock()
defer s.mu.Unlock()
s.mu.is.NodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String())
broadcast := func() {
// Close the old ready and open a new one. This will broadcast to all
// receivers and setup a fresh channel to replace the closed one.
s.mu.Lock()
defer s.mu.Unlock()
ready := make(chan struct{})
close(s.mu.ready)
s.mu.ready = ready
}
unregister := s.mu.is.registerCallback(".*", func(_ string, _ roachpb.Value) {
broadcast()
})
s.stopper.RunWorker(func() {
<-s.stopper.ShouldQuiesce()
s.mu.Lock()
unregister()
s.mu.Unlock()
broadcast()
})
}
// TestGossipOverwriteNode verifies that if a new node is added with the same
// address as an old node, that old node is removed from the cluster.
func TestGossipOverwriteNode(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
rpcContext := newInsecureRPCContext(stopper)
g := NewTest(1, rpcContext, rpc.NewServer(rpcContext), nil, stopper, metric.NewRegistry())
node1 := &roachpb.NodeDescriptor{NodeID: 1, Address: util.MakeUnresolvedAddr("tcp", "1.1.1.1:1")}
node2 := &roachpb.NodeDescriptor{NodeID: 2, Address: util.MakeUnresolvedAddr("tcp", "2.2.2.2:2")}
if err := g.SetNodeDescriptor(node1); err != nil {
t.Fatal(err)
}
if err := g.SetNodeDescriptor(node2); err != nil {
t.Fatal(err)
}
if val, err := g.GetNodeDescriptor(node1.NodeID); err != nil {
t.Error(err)
} else if val.NodeID != node1.NodeID {
t.Errorf("expected node %d, got %+v", node1.NodeID, val)
}
if val, err := g.GetNodeDescriptor(node2.NodeID); err != nil {
t.Error(err)
} else if val.NodeID != node2.NodeID {
t.Errorf("expected node %d, got %+v", node2.NodeID, val)
}
// Give node3 the same address as node1, which should cause node1 to be
// removed from the cluster.
node3 := &roachpb.NodeDescriptor{NodeID: 3, Address: node1.Address}
if err := g.SetNodeDescriptor(node3); err != nil {
t.Fatal(err)
}
if val, err := g.GetNodeDescriptor(node3.NodeID); err != nil {
t.Error(err)
} else if val.NodeID != node3.NodeID {
t.Errorf("expected node %d, got %+v", node3.NodeID, val)
}
// Quiesce the stopper now to ensure that the update has propagated before
// checking whether node 1 has been removed from the infoStore.
stopper.Quiesce()
expectedErr := "unable to look up descriptor for node"
if val, err := g.GetNodeDescriptor(node1.NodeID); !testutils.IsError(err, expectedErr) {
t.Errorf("expected error %q fetching node %d; got error %v and node %+v",
expectedErr, node1.NodeID, err, val)
}
}
func makeReplicas(addrs ...net.Addr) ReplicaSlice {
replicas := make(ReplicaSlice, len(addrs))
for i, addr := range addrs {
replicas[i].NodeDesc = &roachpb.NodeDescriptor{
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}
}
return replicas
}
// GossipNode gossips the node's address, which is necessary before
// any messages can be sent to it. Normally done automatically by
// AddNode.
func (rttc *raftTransportTestContext) GossipNode(nodeID roachpb.NodeID, addr net.Addr) {
if err := rttc.gossip.AddInfoProto(gossip.MakeNodeIDKey(nodeID),
&roachpb.NodeDescriptor{
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
},
time.Hour); err != nil {
rttc.t.Fatal(err)
}
}
// TestStoresGossipStorage verifies reading and writing of bootstrap info.
func TestStoresGossipStorage(t *testing.T) {
defer leaktest.AfterTest(t)()
manual, stores, ls, stopper := createStores(2, t)
defer stopper.Stop()
ls.AddStore(stores[0])
manual.Set(1)
// Verify initial read is empty.
var bi gossip.BootstrapInfo
if err := ls.ReadBootstrapInfo(&bi); err != nil {
t.Fatal(err)
}
if len(bi.Addresses) != 0 {
t.Errorf("expected empty bootstrap info: %+v", bi)
}
// Add a fake address and write.
manual.Increment(1)
bi.Addresses = append(bi.Addresses, util.MakeUnresolvedAddr("tcp", "127.0.0.1:8001"))
if err := ls.WriteBootstrapInfo(&bi); err != nil {
t.Fatal(err)
}
// Verify on read.
manual.Increment(1)
var newBI gossip.BootstrapInfo
if err := ls.ReadBootstrapInfo(&newBI); err != nil {
t.Fatal(err)
}
if len(newBI.Addresses) != 1 {
t.Errorf("expected single bootstrap info address: %+v", newBI)
}
// Add another store and verify it has bootstrap info written.
ls.AddStore(stores[1])
// Create a new stores object to verify read.
ls2 := NewStores(log.AmbientContext{}, ls.clock)
ls2.AddStore(stores[1])
var verifyBI gossip.BootstrapInfo
if err := ls2.ReadBootstrapInfo(&verifyBI); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(bi, verifyBI) {
t.Errorf("bootstrap info %+v not equal to expected %+v", verifyBI, bi)
}
}
// StartNode initializes a gossip instance for the simulation node and
// starts it.
func (n *Network) StartNode(node *Node) error {
node.Gossip.Start(node.Addr())
node.Gossip.EnableSimulationCycler(true)
n.nodeIDAllocator++
node.Gossip.NodeID.Set(context.TODO(), n.nodeIDAllocator)
if err := node.Gossip.SetNodeDescriptor(&roachpb.NodeDescriptor{
NodeID: node.Gossip.NodeID.Get(),
Address: util.MakeUnresolvedAddr(node.Addr().Network(), node.Addr().String()),
}); err != nil {
return err
}
if err := node.Gossip.AddInfo(node.Addr().String(),
encoding.EncodeUint64Ascending(nil, 0), time.Hour); err != nil {
return err
}
n.Stopper.RunWorker(func() {
netutil.FatalIfUnexpected(node.Server.Serve(node.Listener))
})
return nil
}
// TestGossipStorage verifies that a gossip node can join the cluster
// using the bootstrap hosts in a gossip.Storage object.
func TestGossipStorage(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
network := simulation.NewNetwork(stopper, 3, true)
// Set storage for each of the nodes.
addresses := make(unresolvedAddrSlice, len(network.Nodes))
stores := make([]testStorage, len(network.Nodes))
for i, n := range network.Nodes {
addresses[i] = util.MakeUnresolvedAddr(n.Addr().Network(), n.Addr().String())
if err := n.Gossip.SetStorage(&stores[i]); err != nil {
t.Fatal(err)
}
}
// Wait for the gossip network to connect.
network.RunUntilFullyConnected()
// Wait long enough for storage to get the expected number of addresses.
testutils.SucceedsSoon(t, func() error {
for i := range stores {
p := &stores[i]
if expected, actual := len(network.Nodes)-1 /* -1 is ourself */, p.Len(); expected != actual {
return errors.Errorf("expected %v, got %v (info: %#v)", expected, actual, p.Info().Addresses)
}
}
return nil
})
for i := range stores {
p := &stores[i]
if !p.isRead() {
t.Errorf("%d: expected read from storage", i)
}
if !p.isWrite() {
t.Errorf("%d: expected write from storage", i)
}
p.Lock()
gotAddresses := unresolvedAddrSlice(p.info.Addresses)
sort.Sort(gotAddresses)
var expectedAddresses unresolvedAddrSlice
for j, addr := range addresses {
if i != j { // skip node's own address
expectedAddresses = append(expectedAddresses, addr)
}
}
sort.Sort(expectedAddresses)
// Verify all gossip addresses are written to each persistent store.
if !reflect.DeepEqual(gotAddresses, expectedAddresses) {
t.Errorf("%d: expected addresses: %s, got: %s", i, expectedAddresses, gotAddresses)
}
p.Unlock()
}
// Create an unaffiliated gossip node with only itself as a resolver,
// leaving it no way to reach the gossip network.
node, err := network.CreateNode()
if err != nil {
t.Fatal(err)
}
node.Gossip.SetBootstrapInterval(1 * time.Millisecond)
r, err := resolver.NewResolverFromAddress(node.Addr())
if err != nil {
t.Fatal(err)
}
node.Gossip.SetResolvers([]resolver.Resolver{r})
if err := network.StartNode(node); err != nil {
t.Fatal(err)
}
// Wait for a bit to ensure no connection.
select {
case <-time.After(10 * time.Millisecond):
// expected outcome...
case <-node.Gossip.Connected:
t.Fatal("unexpectedly connected to gossip")
}
// Give the new node storage with info established from a node
// in the established network.
var ts2 testStorage
if err := stores[0].ReadBootstrapInfo(&ts2.info); err != nil {
t.Fatal(err)
}
if err := node.Gossip.SetStorage(&ts2); err != nil {
t.Fatal(err)
}
network.SimulateNetwork(func(cycle int, network *simulation.Network) bool {
if cycle > 1000 {
t.Fatal("failed to connect to gossip")
}
select {
case <-node.Gossip.Connected:
return false
default:
return true
}
})
testutils.SucceedsSoon(t, func() error {
if expected, actual := len(network.Nodes)-1 /* -1 is ourself */, ts2.Len(); expected != actual {
return errors.Errorf("expected %v, got %v (info: %#v)", expected, actual, ts2.Info().Addresses)
}
return nil
})
}
// initDescriptor initializes the node descriptor with the server
// address, the node attributes and locality.
func (n *Node) initDescriptor(addr net.Addr, attrs roachpb.Attributes, locality roachpb.Locality) {
n.Descriptor.Address = util.MakeUnresolvedAddr(addr.Network(), addr.String())
n.Descriptor.Attrs = attrs
n.Descriptor.Locality = locality
}
"time"
"golang.org/x/net/context"
"github.com/cockroachdb/cockroach/pkg/base"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/util"
"github.com/cockroachdb/cockroach/pkg/util/leaktest"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/metric"
"github.com/cockroachdb/cockroach/pkg/util/stop"
"github.com/cockroachdb/cockroach/pkg/util/syncutil"
"github.com/gogo/protobuf/proto"
)
var emptyAddr = util.MakeUnresolvedAddr("test", "<test-addr>")
func newTestInfoStore() (*infoStore, *stop.Stopper) {
stopper := stop.NewStopper()
nc := &base.NodeIDContainer{}
nc.Set(context.TODO(), 1)
is := newInfoStore(log.AmbientContext{}, nc, emptyAddr, stopper)
return is, stopper
}
// TestZeroDuration verifies that specifying a zero duration sets
// TTLStamp to max int64.
func TestZeroDuration(t *testing.T) {
defer leaktest.AfterTest(t)()
is, stopper := newTestInfoStore()
defer stopper.Stop()