// TestGossipStorageCleanup verifies that bad resolvers are purged
// from the bootstrap info after gossip has successfully connected.
func TestGossipStorageCleanup(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
const numNodes = 3
network := simulation.NewNetwork(stopper, numNodes, false)
const notReachableAddr = "localhost:0"
const invalidAddr = "10.0.0.1000:3333333"
// Set storage for each of the nodes.
addresses := make(unresolvedAddrSlice, len(network.Nodes))
stores := make([]testStorage, len(network.Nodes))
for i, n := range network.Nodes {
addresses[i] = util.MakeUnresolvedAddr(n.Addr().Network(), n.Addr().String())
// Pre-add an invalid address to each gossip storage.
if err := stores[i].WriteBootstrapInfo(&gossip.BootstrapInfo{
Addresses: []util.UnresolvedAddr{
util.MakeUnresolvedAddr("tcp", network.Nodes[(i+1)%numNodes].Addr().String()), // node i+1 address
util.MakeUnresolvedAddr("tcp", notReachableAddr), // unreachable address
util.MakeUnresolvedAddr("tcp", invalidAddr), // invalid address
},
}); err != nil {
t.Fatal(err)
}
if err := n.Gossip.SetStorage(&stores[i]); err != nil {
t.Fatal(err)
}
n.Gossip.SetStallInterval(1 * time.Millisecond)
n.Gossip.SetBootstrapInterval(1 * time.Millisecond)
}
// Wait for the gossip network to connect.
network.RunUntilFullyConnected()
// Let the gossip network continue running in the background without the
// simulation cycler preventing it from operating.
for _, node := range network.Nodes {
node.Gossip.EnableSimulationCycler(false)
}
// Wait long enough for storage to get the expected number of
// addresses and no pending cleanups.
testutils.SucceedsSoon(t, func() error {
for i := range stores {
p := &stores[i]
if expected, actual := len(network.Nodes)-1 /* -1 is ourself */, p.Len(); expected != actual {
return errors.Errorf("expected %v, got %v (info: %#v)", expected, actual, p.Info().Addresses)
}
for _, addr := range p.Info().Addresses {
if addr.String() == invalidAddr {
return errors.Errorf("node %d still needs bootstrap cleanup", i)
}
}
}
return nil
})
}
func main() {
// Seed the random number generator for non-determinism across
// multiple runs.
randutil.SeedForTests()
if f := flag.Lookup("alsologtostderr"); f != nil {
fmt.Println("Starting simulation. Add -alsologtostderr to see progress.")
}
flag.Parse()
dirName, err := ioutil.TempDir("", "gossip-simulation-")
if err != nil {
log.Fatalf(context.TODO(), "could not create temporary directory for gossip simulation output: %s", err)
}
// Simulation callbacks to run the simulation for cycleCount
// cycles. At each cycle % outputEvery, a dot file showing the
// state of the network graph is output.
nodeCount := 3
switch *size {
case "tiny":
// Use default parameters.
case "small":
nodeCount = 10
case "medium":
nodeCount = 25
case "large":
nodeCount = 50
case "huge":
nodeCount = 100
case "ginormous":
nodeCount = 250
default:
log.Fatalf(context.TODO(), "unknown simulation size: %s", *size)
}
edgeSet := make(map[string]edge)
stopper := stop.NewStopper()
defer stopper.Stop()
n := simulation.NewNetwork(stopper, nodeCount, true)
n.SimulateNetwork(
func(cycle int, network *simulation.Network) bool {
// Output dot graph.
dotFN := fmt.Sprintf("%s/sim-cycle-%03d.dot", dirName, cycle)
_, quiescent := outputDotFile(dotFN, cycle, network, edgeSet)
// Run until network has quiesced.
return !quiescent
},
)
// Output instructions for viewing graphs.
fmt.Printf("To view simulation graph output run (you must install graphviz):\n\nfor f in %s/*.dot ; do circo $f -Tpng -o $f.png ; echo $f.png ; done\n", dirName)
}
// TestConvergence verifies a 10 node gossip network converges within
// a fixed number of simulation cycles. It's really difficult to
// determine the right number for cycles because different things can
// happen during a single cycle, depending on how much CPU time is
// available. Eliminating this variability by getting more
// synchronization primitives in place for the simulation is possible,
// though two attempts so far have introduced more complexity into the
// actual production gossip code than seems worthwhile for a unittest.
func TestConvergence(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
network := simulation.NewNetwork(stopper, 10, true)
const maxCycles = 100
if connectedCycle := network.RunUntilFullyConnected(); connectedCycle > maxCycles {
log.Warningf(context.TODO(), "expected a fully-connected network within %d cycles; took %d",
maxCycles, connectedCycle)
}
}
// TestGossipStorage verifies that a gossip node can join the cluster
// using the bootstrap hosts in a gossip.Storage object.
func TestGossipStorage(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
network := simulation.NewNetwork(stopper, 3, true)
// Set storage for each of the nodes.
addresses := make(unresolvedAddrSlice, len(network.Nodes))
stores := make([]testStorage, len(network.Nodes))
for i, n := range network.Nodes {
addresses[i] = util.MakeUnresolvedAddr(n.Addr().Network(), n.Addr().String())
if err := n.Gossip.SetStorage(&stores[i]); err != nil {
t.Fatal(err)
}
}
// Wait for the gossip network to connect.
network.RunUntilFullyConnected()
// Wait long enough for storage to get the expected number of addresses.
testutils.SucceedsSoon(t, func() error {
for i := range stores {
p := &stores[i]
if expected, actual := len(network.Nodes)-1 /* -1 is ourself */, p.Len(); expected != actual {
return errors.Errorf("expected %v, got %v (info: %#v)", expected, actual, p.Info().Addresses)
}
}
return nil
})
for i := range stores {
p := &stores[i]
if !p.isRead() {
t.Errorf("%d: expected read from storage", i)
}
if !p.isWrite() {
t.Errorf("%d: expected write from storage", i)
}
p.Lock()
gotAddresses := unresolvedAddrSlice(p.info.Addresses)
sort.Sort(gotAddresses)
var expectedAddresses unresolvedAddrSlice
for j, addr := range addresses {
if i != j { // skip node's own address
expectedAddresses = append(expectedAddresses, addr)
}
}
sort.Sort(expectedAddresses)
// Verify all gossip addresses are written to each persistent store.
if !reflect.DeepEqual(gotAddresses, expectedAddresses) {
t.Errorf("%d: expected addresses: %s, got: %s", i, expectedAddresses, gotAddresses)
}
p.Unlock()
}
// Create an unaffiliated gossip node with only itself as a resolver,
// leaving it no way to reach the gossip network.
node, err := network.CreateNode()
if err != nil {
t.Fatal(err)
}
node.Gossip.SetBootstrapInterval(1 * time.Millisecond)
r, err := resolver.NewResolverFromAddress(node.Addr())
if err != nil {
t.Fatal(err)
}
node.Gossip.SetResolvers([]resolver.Resolver{r})
if err := network.StartNode(node); err != nil {
t.Fatal(err)
}
// Wait for a bit to ensure no connection.
select {
case <-time.After(10 * time.Millisecond):
// expected outcome...
case <-node.Gossip.Connected:
t.Fatal("unexpectedly connected to gossip")
}
// Give the new node storage with info established from a node
// in the established network.
var ts2 testStorage
if err := stores[0].ReadBootstrapInfo(&ts2.info); err != nil {
t.Fatal(err)
}
if err := node.Gossip.SetStorage(&ts2); err != nil {
t.Fatal(err)
}
network.SimulateNetwork(func(cycle int, network *simulation.Network) bool {
if cycle > 1000 {
t.Fatal("failed to connect to gossip")
}
select {
case <-node.Gossip.Connected:
return false
default:
return true
}
})
testutils.SucceedsSoon(t, func() error {
if expected, actual := len(network.Nodes)-1 /* -1 is ourself */, ts2.Len(); expected != actual {
return errors.Errorf("expected %v, got %v (info: %#v)", expected, actual, ts2.Info().Addresses)
}
return nil
})
}