func fillUpNodeInfo(node *types.NodeInfo, key types.StoreKey, i int) { node.Id = types.NodeId(strconv.Itoa(i)) node.LastUpdateTs = time.Now() node.Status = types.NODE_STATUS_UP node.Value = make(types.StoreMap) node.Value[types.StoreKey(CPU+key)] = node.Id node.Value[types.StoreKey(MEMORY+key)] = node.Id }
func TestGossiperZMisc(t *testing.T) { printTestInfo() g := NewGossiperImpl("0.0.0.0:9092", "1") g.selfCorrect = true key := types.StoreKey("someKey") g.UpdateSelf(key, "1") startTime := time.Now() // get the default value gossipIntvl := g.GossipInterval() if gossipIntvl == 0 { t.Error("Default gossip interval set to zero") } gossipDuration := 20 * time.Millisecond g.SetGossipInterval(gossipDuration) gossipIntvl = g.GossipInterval() if gossipIntvl != gossipDuration { t.Error("Set interval and get interval differ, got: ", gossipIntvl) } // get the default value deathIntvl := g.NodeDeathInterval() if deathIntvl == 0 { t.Error("Default death interval set to zero") } deathDuration := 20 * time.Millisecond g.SetNodeDeathInterval(deathDuration) deathIntvl = g.NodeDeathInterval() if deathIntvl != deathDuration { t.Error("Set death interval and get interval differ, got: ", deathIntvl) } // stay up for more than gossip interval and check // that we don't die because there is no one to gossip time.Sleep(18 * gossipDuration) // check that our value is self corrected gValues := g.GetStoreKeyValue(key) if len(gValues) > 1 { t.Error("More values returned than request ", gValues) } nodeValue, ok := gValues["1"] if !ok { t.Error("Could not get node value for self node") } else { if !nodeValue.LastUpdateTs.After(startTime) { t.Error("time not updated") } } g.Stop() }
func (c *ClusterManager) GetState() *ClusterState { gossipStoreKey := types.StoreKey(heartbeatKey + c.config.ClusterId) nodeValue := c.g.GetStoreKeyValue(gossipStoreKey) nodes := make([]types.NodeValue, len(nodeValue), len(nodeValue)) i := 0 for _, value := range nodeValue { nodes[i] = value i++ } history := c.g.GetGossipHistory() return &ClusterState{ History: history, NodeStatus: nodes} }
func TestGossipStoreUpdateSelf(t *testing.T) { printTestInfo() // emtpy store g := NewGossipStore(ID) id := g.NodeId() if id != ID { t.Error("Incorrect NodeId(), got: ", id, " expected: ", ID) } value := "string" key1 := types.StoreKey("key1") // key absent, id absent g.UpdateSelf(key1, value) nodeInfo, ok := g.nodeMap[ID] if !ok || nodeInfo.Value == nil { t.Error("UpdateSelf adding new id failed") } else { nodeValue, ok := nodeInfo.Value[key1] if !ok { t.Error("UpdateSelf adding new key failed, after update state: ", g.nodeMap) } else { if nodeValue != value || nodeInfo.Id != ID { t.Error("UpdateSelf failed, got value: ", nodeInfo.Value, " got: ", value) } } } // key present id present prevTs := time.Now() time.Sleep(1 * time.Second) value = "newValue" g.UpdateSelf(key1, value) nodeInfo = g.nodeMap[ID] nodeValue := nodeInfo.Value[key1] if !prevTs.Before(nodeInfo.LastUpdateTs) { t.Error("UpdateSelf failed to update timestamp, prev: ", prevTs, " got: ", nodeInfo) } if nodeValue != value || nodeInfo.Id != ID { t.Error("UpdateSelf failed, got value: ", nodeInfo, " got: ", value, " expected id: ", ID) } }
func TestGossipStoreGetStoreKeyValue(t *testing.T) { printTestInfo() // Case: emtpy store // Case: key absent g := NewGossipStore(ID) keyList := []types.StoreKey{"key1", "key2"} nodeInfoMap := g.GetStoreKeyValue(keyList[0]) if len(nodeInfoMap) != 0 { t.Error("Expected empty node info list, got: ", nodeInfoMap) } // Case: key present with nodes with holes in node ids fillUpNodeInfoMap(g.nodeMap, keyList[0], 6) if len(g.nodeMap) != 6 { t.Error("Failed to fillup node info map properly, got: ", g.nodeMap) } keyCheck := types.StoreKey(CPU + keyList[0]) delete(g.nodeMap["0"].Value, keyCheck) delete(g.nodeMap["2"].Value, keyCheck) delete(g.nodeMap["4"].Value, keyCheck) nodeInfoMap = g.GetStoreKeyValue(keyCheck) if len(nodeInfoMap) != 3 { t.Error("Expected list with atleast 6 elements, got: ", nodeInfoMap) } for i := 0; i < len(nodeInfoMap); i++ { id := types.NodeId(strconv.Itoa(i)) if i%2 == 0 { if _, ok := nodeInfoMap[id]; ok { t.Error("No node expected, got: ", nodeInfoMap[id]) } continue } infoMap := nodeInfoMap[id].Value.(types.NodeId) if nodeInfoMap[id].Id != id || nodeInfoMap[id].Status != types.NODE_STATUS_UP || infoMap != id { t.Error("Invalid node content received, got: ", nodeInfoMap[id]) } } }
func (c *ClusterManager) startHeartBeat() { gossipStoreKey := types.StoreKey(heartbeatKey + c.config.ClusterId) node := c.getCurrentState() c.gossip.UpdateSelf(gossipStoreKey, *node) c.gossip.Start() lastUpdateTs := time.Now() for { node = c.getCurrentState() currTime := time.Now() diffTime := currTime.Sub(lastUpdateTs) if diffTime > 10*time.Second { dlog.Warnln("No gossip update for ", diffTime.Seconds(), "s") } c.gossip.UpdateSelf(gossipStoreKey, *node) lastUpdateTs = currTime time.Sleep(2 * time.Second) } }
func (c *ClusterManager) heartBeat() { for { node := c.getCurrentState() c.nodeCache[node.Id] = *node c.g.UpdateSelf(gossiptypes.StoreKey(heartbeatKey+c.config.ClusterId), *node) // Process heartbeats from other nodes... gossipValues := c.g.GetStoreKeyValue(gossiptypes.StoreKey(heartbeatKey + c.config.ClusterId)) for _, nodeInfo := range gossipValues { n, ok := nodeInfo.Value.(api.Node) if !ok { log.Warn("Received a bad broadcast packet: %v", nodeInfo.Value) continue } if n.Id == node.Id { continue } _, ok = c.nodeCache[n.Id] if ok { if n.Status != api.StatusOk { log.Warn("Detected node ", n.Id, " to be unhealthy.") for e := c.listeners.Front(); e != nil; e = e.Next() { err := e.Value.(ClusterListener).Update(&n) if err != nil { log.Warn("Failed to notify ", e.Value.(ClusterListener).String()) } } delete(c.nodeCache, n.Id) } else if time.Since(n.Timestamp) > 60*time.Second { log.Warn("Detected node ", n.Id, " to be offline due to inactivity.") n.Status = api.StatusOffline for e := c.listeners.Front(); e != nil; e = e.Next() { err := e.Value.(ClusterListener).Update(&n) if err != nil { log.Warn("Failed to notify ", e.Value.(ClusterListener).String()) } } delete(c.nodeCache, n.Id) } else { c.nodeCache[n.Id] = n } } else if time.Since(n.Timestamp) <= 60*time.Second { // A node discovered in the cluster. log.Warn("Detected node ", n.Id, " to be in the cluster.") c.nodeCache[n.Id] = n for e := c.listeners.Front(); e != nil; e = e.Next() { err := e.Value.(ClusterListener).Add(&n) if err != nil { log.Warn("Failed to notify ", e.Value.(ClusterListener).String()) } } } } time.Sleep(2 * time.Second) } }
func clearKey(nodes types.NodeInfoMap, key types.StoreKey, id int) { nodeId := types.NodeId(strconv.Itoa(id)) nodeInfo := nodes[nodeId] delete(nodeInfo.Value, types.StoreKey(CPU+key)) delete(nodeInfo.Value, types.StoreKey(MEMORY+key)) }
func (c *ClusterManager) heartBeat() { gossipStoreKey := types.StoreKey(heartbeatKey + c.config.ClusterId) lastUpdateTs := time.Now() for { node := c.getCurrentState() c.nodeCache[node.Id] = *node currTime := time.Now() if currTime.Sub(lastUpdateTs) > 10*time.Second { logrus.Warn("No gossip update for 10 seconds") } c.g.UpdateSelf(gossipStoreKey, *node) lastUpdateTs = currTime // Process heartbeats from other nodes... gossipValues := c.g.GetStoreKeyValue(gossipStoreKey) for id, nodeInfo := range gossipValues { if id == types.NodeId(node.Id) { continue } cachedNodeInfo, nodeFoundInCache := c.nodeCache[string(id)] n := cachedNodeInfo ok := false if nodeInfo.Value != nil { n, ok = nodeInfo.Value.(api.Node) if !ok { logrus.Error("Received a bad broadcast packet: %v", nodeInfo.Value) continue } } if nodeFoundInCache { if n.Status != api.StatusOk { logrus.Warn("Detected node ", n.Id, " to be unhealthy.") for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() { err := e.Value.(ClusterListener).Update(&n) if err != nil { logrus.Warn("Failed to notify ", e.Value.(ClusterListener).String()) } } delete(c.nodeCache, n.Id) continue } else if nodeInfo.Status == types.NODE_STATUS_DOWN { ne := c.getLatestNodeConfig(string(id)) if ne != nil && nodeInfo.GenNumber < ne.GenNumber { logrus.Warn("Detected stale update for node ", id, " going down, ignoring it") c.g.MarkNodeHasOldGen(id) delete(c.nodeCache, cachedNodeInfo.Id) continue } logrus.Warn("Detected node ", id, " to be offline due to inactivity.") n.Status = api.StatusOffline for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() { err := e.Value.(ClusterListener).Update(&n) if err != nil { logrus.Warn("Failed to notify ", e.Value.(ClusterListener).String()) } } delete(c.nodeCache, cachedNodeInfo.Id) } else if nodeInfo.Status == types.NODE_STATUS_DOWN_WAITING_FOR_NEW_UPDATE { logrus.Warn("Detected node ", n.Id, " to be offline due to inactivity.") n.Status = api.StatusOffline for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() { err := e.Value.(ClusterListener).Update(&n) if err != nil { logrus.Warn("Failed to notify ", e.Value.(ClusterListener).String()) } } delete(c.nodeCache, cachedNodeInfo.Id) } else { // node may be up or waiting for new update, // no need to tell listeners as yet. c.nodeCache[cachedNodeInfo.Id] = n } } else if nodeInfo.Status == types.NODE_STATUS_UP { // A node discovered in the cluster. logrus.Warn("Detected node ", n.Id, " to be in the cluster.") c.nodeCache[n.Id] = n for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() { err := e.Value.(ClusterListener).Add(&n) if err != nil { logrus.Warn("Failed to notify ", e.Value.(ClusterListener).String()) } } } } time.Sleep(2 * time.Second) } }
func (c *ClusterManager) updateClusterStatus() { gossipStoreKey := types.StoreKey(heartbeatKey + c.config.ClusterId) for { node := c.getCurrentState() c.nodeCache[node.Id] = *node // Process heartbeats from other nodes... gossipValues := c.gossip.GetStoreKeyValue(gossipStoreKey) numNodes := 0 for id, nodeInfo := range gossipValues { numNodes = numNodes + 1 // Check to make sure we are not exceeding the size of the cluster. if c.size > 0 && numNodes > c.size { dlog.Fatalf("Fatal, number of nodes in the cluster has"+ "exceeded the cluster size: %d > %d", numNodes, c.size) os.Exit(-1) } // Ignore updates from self node. if id == types.NodeId(node.Id) { continue } // Notify node status change if required. newNodeInfo := api.Node{} newNodeInfo.Id = string(id) newNodeInfo.Status = api.Status_STATUS_OK switch { case nodeInfo.Status == types.NODE_STATUS_DOWN: newNodeInfo.Status = api.Status_STATUS_OFFLINE lastStatus, ok := c.nodeStatuses[string(id)] if ok && lastStatus == newNodeInfo.Status { break } // Check if it is a stale update ne := c.getLatestNodeConfig(string(id)) if ne != nil && nodeInfo.GenNumber != 0 && nodeInfo.GenNumber < ne.GenNumber { dlog.Warnln("Detected stale update for node ", id, " going down, ignoring it") c.gossip.MarkNodeHasOldGen(id) break } c.nodeStatuses[string(id)] = newNodeInfo.Status dlog.Warnln("Detected node ", id, " to be offline due to inactivity.") for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() { err := e.Value.(ClusterListener).Update(&newNodeInfo) if err != nil { dlog.Warnln("Failed to notify ", e.Value.(ClusterListener).String()) } } case nodeInfo.Status == types.NODE_STATUS_DOWN_WAITING_FOR_NEW_UPDATE: newNodeInfo.Status = api.Status_STATUS_OFFLINE lastStatus, ok := c.nodeStatuses[string(id)] if ok && lastStatus == newNodeInfo.Status { break } c.nodeStatuses[string(id)] = newNodeInfo.Status dlog.Warnln("Detected node ", newNodeInfo.Id, " to be offline due to inactivity.") for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() { err := e.Value.(ClusterListener).Update(&newNodeInfo) if err != nil { dlog.Warnln("Failed to notify ", e.Value.(ClusterListener).String()) } } case nodeInfo.Status == types.NODE_STATUS_UP: newNodeInfo.Status = api.Status_STATUS_OK lastStatus, ok := c.nodeStatuses[string(id)] if ok && lastStatus == newNodeInfo.Status { break } c.nodeStatuses[string(id)] = newNodeInfo.Status // A node discovered in the cluster. dlog.Warnln("Detected node ", newNodeInfo.Id, " to be in the cluster.") for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() { err := e.Value.(ClusterListener).Add(&newNodeInfo) if err != nil { dlog.Warnln("Failed to notify ", e.Value.(ClusterListener).String()) } } } // Update cache. if nodeInfo.Value != nil { n, ok := nodeInfo.Value.(api.Node) if ok { n.Status = newNodeInfo.Status c.nodeCache[n.Id] = n } else { c.nodeCache[newNodeInfo.Id] = newNodeInfo } } else { newNodeInfo.Status = api.Status_STATUS_OFFLINE c.nodeCache[newNodeInfo.Id] = newNodeInfo } } time.Sleep(2 * time.Second) } }
func TestGossiperMultipleNodesGoingUpDown(t *testing.T) { printTestInfo() nodes := []string{"0.0.0.0:9152", "0.0.0.0:9153", "0.0.0.0:9154", "0.0.0.0:9155", "0.0.0.0:9156", "0.0.0.0:9157", "0.0.0.0:9158", "0.0.0.0:9159", "0.0.0.0:9160", "0.0.0.0:9161"} rand.Seed(time.Now().UnixNano()) gossipers := make(map[string]*GossiperImpl) for i, nodeId := range nodes { g := NewGossiperImpl(nodeId, types.NodeId(strconv.Itoa(i))) g.SetGossipInterval(time.Duration(1500+rand.Intn(200)) * time.Millisecond) // add one neighbor and 2 random peers if i < len(nodes)-1 { err := g.AddNode(nodes[i+1], types.NodeId(strconv.Itoa(i))) if err != nil { t.Error("Unexpected error adding node to id: ", nodeId, " node: ", nodes[i+1]) } } else { err := g.AddNode(nodes[0], types.NodeId(strconv.Itoa(0))) if err != nil { t.Error("Unexpected error adding node to id: ", nodeId, " node: ", nodes[0]) } } // to this gossiper, add two random peers for count := 0; count < 2; { randId := rand.Intn(len(nodes)) if randId == i { continue } err := g.AddNode(nodes[randId], types.NodeId(strconv.Itoa(randId))) if err != nil { t.Log("Unexpected error adding node to id: ", nodeId, " node: ", nodes[randId], " err: ", err) } else { count++ } } gossipers[nodeId] = g time.Sleep(2000 * time.Millisecond) } updateFunc := func(g *GossiperImpl, id string, max int, t *testing.T) { for i := 0; i < max; i++ { t.Log("Updting data for ", id) g.UpdateSelf("sameKey", strconv.Itoa(i)) g.UpdateSelf(types.StoreKey(g.NodeId()), strconv.Itoa(i*i)) time.Sleep(g.GossipInterval() + time.Duration(rand.Intn(100))) } } for id, g := range gossipers { go updateFunc(g, id, 10, t) } // Max duration for update is 1500 + 200 + 100 per update * 10 // = 1800 mil * 10 = 18000 mil. // To add go fork thread, 2000 mil on top. // Let gossip go on for another 10 seconds, after which it must settle time.Sleep(1 * time.Minute) // verify all of them are same for i := 1; i < len(nodes); i++ { t.Log("Checking equality of ", nodes[0], " and ", nodes[i]) verifyGossiperEquality(gossipers[nodes[0]], gossipers[nodes[i]], t) } // start another update round, however, we will shut down soem machines // in between for id, g := range gossipers { go updateFunc(g, id, 10, t) } shutdownNodes := make(map[int]bool) for { randId := rand.Intn(len(nodes)) if randId == 0 { continue } _, ok := shutdownNodes[randId] if ok == false { shutdownNodes[randId] = true gossipers[nodes[randId]].Stop() if len(shutdownNodes) == 3 { break } } } time.Sleep(1 * time.Minute) // verify all of them are same for i := 1; i < len(nodes); i++ { _, ok := shutdownNodes[i] if ok { continue } t.Log("Checking equality of ", nodes[0], " and ", nodes[i]) verifyGossiperEquality(gossipers[nodes[0]], gossipers[nodes[i]], t) g := gossipers[nodes[i]] keys := g.GetStoreKeys() for _, key := range keys { values := g.GetStoreKeyValue(key) for j, nodeInfo := range values { nodeId, _ := strconv.Atoi(string(j)) _, ok := shutdownNodes[nodeId] if ok && nodeInfo.Status == types.NODE_STATUS_UP { t.Error("Node not marked down: ", nodeInfo, " for node: ", nodes[i]) } } } } for i := 1; i < len(nodes); i++ { gossipers[nodes[i]].Stop() } }
func TestGossiperGossipMarkOldGenNode(t *testing.T) { printTestInfo() nodes := []string{"0.0.0.0:9225", "0.0.0.0:9226", "0.0.0.0:9227"} rand.Seed(time.Now().UnixNano()) gossipers := make(map[int]*GossiperImpl) for i, nodeId := range nodes { if i == 0 { // node 0 never comes up continue } id := types.NodeId(strconv.Itoa(i)) g := NewGossiperImpl(nodeId, id) g.SetGossipInterval(time.Duration(200+rand.Intn(200)) * time.Millisecond) for j, peer := range nodes { if i == j { continue } g.AddNode(peer, types.NodeId(j)) } gossipers[i] = g } // each node must mark node 0 as down key := types.StoreKey("somekey") value := "someValue" for i, g := range gossipers { g.UpdateSelf(key, value+strconv.Itoa(i)) } for i, g := range gossipers { res := g.GetStoreKeyValue(key) for nodeId, n := range res { if nodeId != n.Id { t.Error("Gossiper ", i, "Id does not match ", nodeId, " n:", n.Id) } nid, ok := strconv.Atoi(string(nodeId)) if ok != nil { t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id) } if nid == 0 { if n.Status == types.NODE_STATUS_DOWN { t.Error("Gossiper ", i, "Expected node status not to be down: ", nodeId, " n:", n) } } } } time.Sleep(2 * time.Second) for i, g := range gossipers { res := g.GetStoreKeyValue(key) for nodeId, n := range res { if nodeId != n.Id { t.Error("Gossiper ", i, "Id does not match ", nodeId, " n:", n.Id) } nid, ok := strconv.Atoi(string(nodeId)) if ok != nil { t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id) } if nid == 0 { if n.Status != types.NODE_STATUS_DOWN { t.Error("Gossiper ", i, "Expected node status to be down: ", nodeId, " n:", n) } } else { if n.Status != types.NODE_STATUS_UP { t.Error("Gossiper ", i, "Expected node to be up: ", nodeId, " n:", n) } } } } // Now Reset both node 0 and node 1 in node 2. nid0 := types.NodeId(strconv.Itoa(0)) nid1 := types.NodeId(strconv.Itoa(1)) nid2 := types.NodeId(strconv.Itoa(2)) g, _ := gossipers[2] g.MarkNodeHasOldGen(nid0) g.MarkNodeHasOldGen(nid1) // Update value in node 1 g, _ = gossipers[1] g.UpdateSelf(key, value+"__1") // Node must be up now g, _ = gossipers[2] res := g.GetStoreKeyValue(key) for nodeId, n := range res { if nid2 == nodeId { continue } if nodeId != n.Id { t.Error("Id does not match ", nodeId, " n:", n.Id) } _, ok := strconv.Atoi(string(nodeId)) if ok != nil { t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id) } if n.Status != types.NODE_STATUS_WAITING_FOR_NEW_UPDATE { t.Error("Expected node status to be down: ", nodeId, " n:", n) } } time.Sleep(800 * time.Millisecond) res = g.GetStoreKeyValue(key) g1Node, _ := res[nid1] if g1Node.Status != types.NODE_STATUS_UP { t.Error("Expected node to be up ", g1Node) } // Sleep for 15*(node death interval) i.e. 15*400 mil i.e. 6 seconds // , both nodes must me marked down time.Sleep(35 * time.Second) res = g.GetStoreKeyValue(key) for nodeId, n := range res { if nid2 == nodeId { continue } if nodeId != n.Id { t.Error("Id does not match ", nodeId, " n:", n.Id) } _, ok := strconv.Atoi(string(nodeId)) if ok != nil { t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id) } if n.Status != types.NODE_STATUS_DOWN { t.Error("Expected node to be down: ", nodeId, " n:", n) } } // Test that the node does not is marked down after reset g.MarkNodeHasOldGen(nid1) time.Sleep(35 * time.Second) res = g.GetStoreKeyValue(key) g1Node, _ = res[nid1] if g1Node.Status != types.NODE_STATUS_DOWN_WAITING_FOR_NEW_UPDATE { t.Error("Expected node to be down waiting for update ", g1Node) } for _, g := range gossipers { g.Stop() } }
func TestGossiperOneNodeNeverGossips(t *testing.T) { printTestInfo() nodes := []string{"0.0.0.0:9222", "0.0.0.0:9223", "0.0.0.0:9224"} rand.Seed(time.Now().UnixNano()) gossipers := make(map[int]*GossiperImpl) for i, nodeId := range nodes { if i == 0 { // node 0 never comes up continue } id := types.NodeId(strconv.Itoa(i)) g := NewGossiperImpl(nodeId, id) g.SetGossipInterval(time.Duration(200+rand.Intn(200)) * time.Millisecond) for j, peer := range nodes { if i == j { continue } g.AddNode(peer, types.NodeId(j)) } gossipers[i] = g } // each node must mark node 0 as down key := types.StoreKey("somekey") value := "someValue" for i, g := range gossipers { g.UpdateSelf(key, value+strconv.Itoa(i)) } for i, g := range gossipers { res := g.GetStoreKeyValue(key) for nodeId, n := range res { if nodeId != n.Id { t.Error("Gossiper ", i, "Id does not match ", nodeId, " n:", n.Id) } nid, ok := strconv.Atoi(string(nodeId)) if ok != nil { t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id) } if nid == 0 { if n.Status == types.NODE_STATUS_DOWN { t.Error("Gossiper ", i, "Expected node status not to be down: ", nodeId, " n:", n) } } } } time.Sleep(2 * time.Second) for i, g := range gossipers { res := g.GetStoreKeyValue(key) for nodeId, n := range res { if nodeId != n.Id { t.Error("Gossiper ", i, "Id does not match ", nodeId, " n:", n.Id) } nid, ok := strconv.Atoi(string(nodeId)) if ok != nil { t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id) } if nid == 0 { if n.Status != types.NODE_STATUS_DOWN { t.Error("Gossiper ", i, "Expected node status to be down: ", nodeId, " n:", n) } } else { if n.Status != types.NODE_STATUS_UP { t.Error("Gossiper ", i, "Expected node to be up: ", nodeId, " n:", n) } } } } for _, g := range gossipers { g.Stop() } }