// TestGossipStorageCleanup verifies that bad resolvers are purged // from the bootstrap info after gossip has successfully connected. func TestGossipStorageCleanup(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() const numNodes = 3 network := simulation.NewNetwork(stopper, numNodes, false) const notReachableAddr = "localhost:0" const invalidAddr = "10.0.0.1000:3333333" // Set storage for each of the nodes. addresses := make(unresolvedAddrSlice, len(network.Nodes)) stores := make([]testStorage, len(network.Nodes)) for i, n := range network.Nodes { addresses[i] = util.MakeUnresolvedAddr(n.Addr().Network(), n.Addr().String()) // Pre-add an invalid address to each gossip storage. if err := stores[i].WriteBootstrapInfo(&gossip.BootstrapInfo{ Addresses: []util.UnresolvedAddr{ util.MakeUnresolvedAddr("tcp", network.Nodes[(i+1)%numNodes].Addr().String()), // node i+1 address util.MakeUnresolvedAddr("tcp", notReachableAddr), // unreachable address util.MakeUnresolvedAddr("tcp", invalidAddr), // invalid address }, }); err != nil { t.Fatal(err) } if err := n.Gossip.SetStorage(&stores[i]); err != nil { t.Fatal(err) } n.Gossip.SetStallInterval(1 * time.Millisecond) n.Gossip.SetBootstrapInterval(1 * time.Millisecond) } // Wait for the gossip network to connect. network.RunUntilFullyConnected() // Let the gossip network continue running in the background without the // simulation cycler preventing it from operating. for _, node := range network.Nodes { node.Gossip.EnableSimulationCycler(false) } // Wait long enough for storage to get the expected number of // addresses and no pending cleanups. testutils.SucceedsSoon(t, func() error { for i := range stores { p := &stores[i] if expected, actual := len(network.Nodes)-1 /* -1 is ourself */, p.Len(); expected != actual { return errors.Errorf("expected %v, got %v (info: %#v)", expected, actual, p.Info().Addresses) } for _, addr := range p.Info().Addresses { if addr.String() == invalidAddr { return errors.Errorf("node %d still needs bootstrap cleanup", i) } } } return nil }) }
func main() { // Seed the random number generator for non-determinism across // multiple runs. randutil.SeedForTests() if f := flag.Lookup("alsologtostderr"); f != nil { fmt.Println("Starting simulation. Add -alsologtostderr to see progress.") } flag.Parse() dirName, err := ioutil.TempDir("", "gossip-simulation-") if err != nil { log.Fatalf(context.TODO(), "could not create temporary directory for gossip simulation output: %s", err) } // Simulation callbacks to run the simulation for cycleCount // cycles. At each cycle % outputEvery, a dot file showing the // state of the network graph is output. nodeCount := 3 switch *size { case "tiny": // Use default parameters. case "small": nodeCount = 10 case "medium": nodeCount = 25 case "large": nodeCount = 50 case "huge": nodeCount = 100 case "ginormous": nodeCount = 250 default: log.Fatalf(context.TODO(), "unknown simulation size: %s", *size) } edgeSet := make(map[string]edge) stopper := stop.NewStopper() defer stopper.Stop() n := simulation.NewNetwork(stopper, nodeCount, true) n.SimulateNetwork( func(cycle int, network *simulation.Network) bool { // Output dot graph. dotFN := fmt.Sprintf("%s/sim-cycle-%03d.dot", dirName, cycle) _, quiescent := outputDotFile(dotFN, cycle, network, edgeSet) // Run until network has quiesced. return !quiescent }, ) // Output instructions for viewing graphs. fmt.Printf("To view simulation graph output run (you must install graphviz):\n\nfor f in %s/*.dot ; do circo $f -Tpng -o $f.png ; echo $f.png ; done\n", dirName) }
// TestConvergence verifies a 10 node gossip network converges within // a fixed number of simulation cycles. It's really difficult to // determine the right number for cycles because different things can // happen during a single cycle, depending on how much CPU time is // available. Eliminating this variability by getting more // synchronization primitives in place for the simulation is possible, // though two attempts so far have introduced more complexity into the // actual production gossip code than seems worthwhile for a unittest. func TestConvergence(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() network := simulation.NewNetwork(stopper, 10, true) const maxCycles = 100 if connectedCycle := network.RunUntilFullyConnected(); connectedCycle > maxCycles { log.Warningf(context.TODO(), "expected a fully-connected network within %d cycles; took %d", maxCycles, connectedCycle) } }
// TestGossipStorage verifies that a gossip node can join the cluster // using the bootstrap hosts in a gossip.Storage object. func TestGossipStorage(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop() network := simulation.NewNetwork(stopper, 3, true) // Set storage for each of the nodes. addresses := make(unresolvedAddrSlice, len(network.Nodes)) stores := make([]testStorage, len(network.Nodes)) for i, n := range network.Nodes { addresses[i] = util.MakeUnresolvedAddr(n.Addr().Network(), n.Addr().String()) if err := n.Gossip.SetStorage(&stores[i]); err != nil { t.Fatal(err) } } // Wait for the gossip network to connect. network.RunUntilFullyConnected() // Wait long enough for storage to get the expected number of addresses. testutils.SucceedsSoon(t, func() error { for i := range stores { p := &stores[i] if expected, actual := len(network.Nodes)-1 /* -1 is ourself */, p.Len(); expected != actual { return errors.Errorf("expected %v, got %v (info: %#v)", expected, actual, p.Info().Addresses) } } return nil }) for i := range stores { p := &stores[i] if !p.isRead() { t.Errorf("%d: expected read from storage", i) } if !p.isWrite() { t.Errorf("%d: expected write from storage", i) } p.Lock() gotAddresses := unresolvedAddrSlice(p.info.Addresses) sort.Sort(gotAddresses) var expectedAddresses unresolvedAddrSlice for j, addr := range addresses { if i != j { // skip node's own address expectedAddresses = append(expectedAddresses, addr) } } sort.Sort(expectedAddresses) // Verify all gossip addresses are written to each persistent store. if !reflect.DeepEqual(gotAddresses, expectedAddresses) { t.Errorf("%d: expected addresses: %s, got: %s", i, expectedAddresses, gotAddresses) } p.Unlock() } // Create an unaffiliated gossip node with only itself as a resolver, // leaving it no way to reach the gossip network. node, err := network.CreateNode() if err != nil { t.Fatal(err) } node.Gossip.SetBootstrapInterval(1 * time.Millisecond) r, err := resolver.NewResolverFromAddress(node.Addr()) if err != nil { t.Fatal(err) } node.Gossip.SetResolvers([]resolver.Resolver{r}) if err := network.StartNode(node); err != nil { t.Fatal(err) } // Wait for a bit to ensure no connection. select { case <-time.After(10 * time.Millisecond): // expected outcome... case <-node.Gossip.Connected: t.Fatal("unexpectedly connected to gossip") } // Give the new node storage with info established from a node // in the established network. var ts2 testStorage if err := stores[0].ReadBootstrapInfo(&ts2.info); err != nil { t.Fatal(err) } if err := node.Gossip.SetStorage(&ts2); err != nil { t.Fatal(err) } network.SimulateNetwork(func(cycle int, network *simulation.Network) bool { if cycle > 1000 { t.Fatal("failed to connect to gossip") } select { case <-node.Gossip.Connected: return false default: return true } }) testutils.SucceedsSoon(t, func() error { if expected, actual := len(network.Nodes)-1 /* -1 is ourself */, ts2.Len(); expected != actual { return errors.Errorf("expected %v, got %v (info: %#v)", expected, actual, ts2.Info().Addresses) } return nil }) }