func fillUpNodeInfo(node *types.NodeInfo, key types.StoreKey, i int) {
node.Id = types.NodeId(strconv.Itoa(i))
node.LastUpdateTs = time.Now()
node.Status = types.NODE_STATUS_UP
node.Value = make(types.StoreMap)
node.Value[types.StoreKey(CPU+key)] = node.Id
node.Value[types.StoreKey(MEMORY+key)] = node.Id
}
func TestGossiperZMisc(t *testing.T) {
printTestInfo()
g := NewGossiperImpl("0.0.0.0:9092", "1")
g.selfCorrect = true
key := types.StoreKey("someKey")
g.UpdateSelf(key, "1")
startTime := time.Now()
// get the default value
gossipIntvl := g.GossipInterval()
if gossipIntvl == 0 {
t.Error("Default gossip interval set to zero")
}
gossipDuration := 20 * time.Millisecond
g.SetGossipInterval(gossipDuration)
gossipIntvl = g.GossipInterval()
if gossipIntvl != gossipDuration {
t.Error("Set interval and get interval differ, got: ", gossipIntvl)
}
// get the default value
deathIntvl := g.NodeDeathInterval()
if deathIntvl == 0 {
t.Error("Default death interval set to zero")
}
deathDuration := 20 * time.Millisecond
g.SetNodeDeathInterval(deathDuration)
deathIntvl = g.NodeDeathInterval()
if deathIntvl != deathDuration {
t.Error("Set death interval and get interval differ, got: ", deathIntvl)
}
// stay up for more than gossip interval and check
// that we don't die because there is no one to gossip
time.Sleep(18 * gossipDuration)
// check that our value is self corrected
gValues := g.GetStoreKeyValue(key)
if len(gValues) > 1 {
t.Error("More values returned than request ", gValues)
}
nodeValue, ok := gValues["1"]
if !ok {
t.Error("Could not get node value for self node")
} else {
if !nodeValue.LastUpdateTs.After(startTime) {
t.Error("time not updated")
}
}
g.Stop()
}
func (c *ClusterManager) GetState() *ClusterState {
gossipStoreKey := types.StoreKey(heartbeatKey + c.config.ClusterId)
nodeValue := c.g.GetStoreKeyValue(gossipStoreKey)
nodes := make([]types.NodeValue, len(nodeValue), len(nodeValue))
i := 0
for _, value := range nodeValue {
nodes[i] = value
i++
}
history := c.g.GetGossipHistory()
return &ClusterState{
History: history, NodeStatus: nodes}
}
func TestGossipStoreUpdateSelf(t *testing.T) {
printTestInfo()
// emtpy store
g := NewGossipStore(ID)
id := g.NodeId()
if id != ID {
t.Error("Incorrect NodeId(), got: ", id,
" expected: ", ID)
}
value := "string"
key1 := types.StoreKey("key1")
// key absent, id absent
g.UpdateSelf(key1, value)
nodeInfo, ok := g.nodeMap[ID]
if !ok || nodeInfo.Value == nil {
t.Error("UpdateSelf adding new id failed")
} else {
nodeValue, ok := nodeInfo.Value[key1]
if !ok {
t.Error("UpdateSelf adding new key failed, after update state: ",
g.nodeMap)
} else {
if nodeValue != value || nodeInfo.Id != ID {
t.Error("UpdateSelf failed, got value: ", nodeInfo.Value,
" got: ", value)
}
}
}
// key present id present
prevTs := time.Now()
time.Sleep(1 * time.Second)
value = "newValue"
g.UpdateSelf(key1, value)
nodeInfo = g.nodeMap[ID]
nodeValue := nodeInfo.Value[key1]
if !prevTs.Before(nodeInfo.LastUpdateTs) {
t.Error("UpdateSelf failed to update timestamp, prev: ", prevTs,
" got: ", nodeInfo)
}
if nodeValue != value || nodeInfo.Id != ID {
t.Error("UpdateSelf failed, got value: ", nodeInfo,
" got: ", value, " expected id: ", ID)
}
}
func TestGossipStoreGetStoreKeyValue(t *testing.T) {
printTestInfo()
// Case: emtpy store
// Case: key absent
g := NewGossipStore(ID)
keyList := []types.StoreKey{"key1", "key2"}
nodeInfoMap := g.GetStoreKeyValue(keyList[0])
if len(nodeInfoMap) != 0 {
t.Error("Expected empty node info list, got: ", nodeInfoMap)
}
// Case: key present with nodes with holes in node ids
fillUpNodeInfoMap(g.nodeMap, keyList[0], 6)
if len(g.nodeMap) != 6 {
t.Error("Failed to fillup node info map properly, got: ",
g.nodeMap)
}
keyCheck := types.StoreKey(CPU + keyList[0])
delete(g.nodeMap["0"].Value, keyCheck)
delete(g.nodeMap["2"].Value, keyCheck)
delete(g.nodeMap["4"].Value, keyCheck)
nodeInfoMap = g.GetStoreKeyValue(keyCheck)
if len(nodeInfoMap) != 3 {
t.Error("Expected list with atleast 6 elements, got: ", nodeInfoMap)
}
for i := 0; i < len(nodeInfoMap); i++ {
id := types.NodeId(strconv.Itoa(i))
if i%2 == 0 {
if _, ok := nodeInfoMap[id]; ok {
t.Error("No node expected, got: ", nodeInfoMap[id])
}
continue
}
infoMap := nodeInfoMap[id].Value.(types.NodeId)
if nodeInfoMap[id].Id != id ||
nodeInfoMap[id].Status != types.NODE_STATUS_UP ||
infoMap != id {
t.Error("Invalid node content received, got: ", nodeInfoMap[id])
}
}
}
func (c *ClusterManager) startHeartBeat() {
gossipStoreKey := types.StoreKey(heartbeatKey + c.config.ClusterId)
node := c.getCurrentState()
c.gossip.UpdateSelf(gossipStoreKey, *node)
c.gossip.Start()
lastUpdateTs := time.Now()
for {
node = c.getCurrentState()
currTime := time.Now()
diffTime := currTime.Sub(lastUpdateTs)
if diffTime > 10*time.Second {
dlog.Warnln("No gossip update for ", diffTime.Seconds(), "s")
}
c.gossip.UpdateSelf(gossipStoreKey, *node)
lastUpdateTs = currTime
time.Sleep(2 * time.Second)
}
}
func (c *ClusterManager) heartBeat() {
for {
node := c.getCurrentState()
c.nodeCache[node.Id] = *node
c.g.UpdateSelf(gossiptypes.StoreKey(heartbeatKey+c.config.ClusterId), *node)
// Process heartbeats from other nodes...
gossipValues := c.g.GetStoreKeyValue(gossiptypes.StoreKey(heartbeatKey + c.config.ClusterId))
for _, nodeInfo := range gossipValues {
n, ok := nodeInfo.Value.(api.Node)
if !ok {
log.Warn("Received a bad broadcast packet: %v", nodeInfo.Value)
continue
}
if n.Id == node.Id {
continue
}
_, ok = c.nodeCache[n.Id]
if ok {
if n.Status != api.StatusOk {
log.Warn("Detected node ", n.Id, " to be unhealthy.")
for e := c.listeners.Front(); e != nil; e = e.Next() {
err := e.Value.(ClusterListener).Update(&n)
if err != nil {
log.Warn("Failed to notify ", e.Value.(ClusterListener).String())
}
}
delete(c.nodeCache, n.Id)
} else if time.Since(n.Timestamp) > 60*time.Second {
log.Warn("Detected node ", n.Id, " to be offline due to inactivity.")
n.Status = api.StatusOffline
for e := c.listeners.Front(); e != nil; e = e.Next() {
err := e.Value.(ClusterListener).Update(&n)
if err != nil {
log.Warn("Failed to notify ", e.Value.(ClusterListener).String())
}
}
delete(c.nodeCache, n.Id)
} else {
c.nodeCache[n.Id] = n
}
} else if time.Since(n.Timestamp) <= 60*time.Second {
// A node discovered in the cluster.
log.Warn("Detected node ", n.Id, " to be in the cluster.")
c.nodeCache[n.Id] = n
for e := c.listeners.Front(); e != nil; e = e.Next() {
err := e.Value.(ClusterListener).Add(&n)
if err != nil {
log.Warn("Failed to notify ", e.Value.(ClusterListener).String())
}
}
}
}
time.Sleep(2 * time.Second)
}
}
func clearKey(nodes types.NodeInfoMap, key types.StoreKey, id int) {
nodeId := types.NodeId(strconv.Itoa(id))
nodeInfo := nodes[nodeId]
delete(nodeInfo.Value, types.StoreKey(CPU+key))
delete(nodeInfo.Value, types.StoreKey(MEMORY+key))
}
func (c *ClusterManager) heartBeat() {
gossipStoreKey := types.StoreKey(heartbeatKey + c.config.ClusterId)
lastUpdateTs := time.Now()
for {
node := c.getCurrentState()
c.nodeCache[node.Id] = *node
currTime := time.Now()
if currTime.Sub(lastUpdateTs) > 10*time.Second {
logrus.Warn("No gossip update for 10 seconds")
}
c.g.UpdateSelf(gossipStoreKey, *node)
lastUpdateTs = currTime
// Process heartbeats from other nodes...
gossipValues := c.g.GetStoreKeyValue(gossipStoreKey)
for id, nodeInfo := range gossipValues {
if id == types.NodeId(node.Id) {
continue
}
cachedNodeInfo, nodeFoundInCache := c.nodeCache[string(id)]
n := cachedNodeInfo
ok := false
if nodeInfo.Value != nil {
n, ok = nodeInfo.Value.(api.Node)
if !ok {
logrus.Error("Received a bad broadcast packet: %v", nodeInfo.Value)
continue
}
}
if nodeFoundInCache {
if n.Status != api.StatusOk {
logrus.Warn("Detected node ", n.Id, " to be unhealthy.")
for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() {
err := e.Value.(ClusterListener).Update(&n)
if err != nil {
logrus.Warn("Failed to notify ", e.Value.(ClusterListener).String())
}
}
delete(c.nodeCache, n.Id)
continue
} else if nodeInfo.Status == types.NODE_STATUS_DOWN {
ne := c.getLatestNodeConfig(string(id))
if ne != nil && nodeInfo.GenNumber < ne.GenNumber {
logrus.Warn("Detected stale update for node ", id,
" going down, ignoring it")
c.g.MarkNodeHasOldGen(id)
delete(c.nodeCache, cachedNodeInfo.Id)
continue
}
logrus.Warn("Detected node ", id, " to be offline due to inactivity.")
n.Status = api.StatusOffline
for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() {
err := e.Value.(ClusterListener).Update(&n)
if err != nil {
logrus.Warn("Failed to notify ", e.Value.(ClusterListener).String())
}
}
delete(c.nodeCache, cachedNodeInfo.Id)
} else if nodeInfo.Status == types.NODE_STATUS_DOWN_WAITING_FOR_NEW_UPDATE {
logrus.Warn("Detected node ", n.Id, " to be offline due to inactivity.")
n.Status = api.StatusOffline
for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() {
err := e.Value.(ClusterListener).Update(&n)
if err != nil {
logrus.Warn("Failed to notify ", e.Value.(ClusterListener).String())
}
}
delete(c.nodeCache, cachedNodeInfo.Id)
} else {
// node may be up or waiting for new update,
// no need to tell listeners as yet.
c.nodeCache[cachedNodeInfo.Id] = n
}
} else if nodeInfo.Status == types.NODE_STATUS_UP {
// A node discovered in the cluster.
logrus.Warn("Detected node ", n.Id, " to be in the cluster.")
c.nodeCache[n.Id] = n
for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() {
err := e.Value.(ClusterListener).Add(&n)
if err != nil {
logrus.Warn("Failed to notify ", e.Value.(ClusterListener).String())
}
}
}
}
time.Sleep(2 * time.Second)
}
}
func (c *ClusterManager) updateClusterStatus() {
gossipStoreKey := types.StoreKey(heartbeatKey + c.config.ClusterId)
for {
node := c.getCurrentState()
c.nodeCache[node.Id] = *node
// Process heartbeats from other nodes...
gossipValues := c.gossip.GetStoreKeyValue(gossipStoreKey)
numNodes := 0
for id, nodeInfo := range gossipValues {
numNodes = numNodes + 1
// Check to make sure we are not exceeding the size of the cluster.
if c.size > 0 && numNodes > c.size {
dlog.Fatalf("Fatal, number of nodes in the cluster has"+
"exceeded the cluster size: %d > %d", numNodes, c.size)
os.Exit(-1)
}
// Ignore updates from self node.
if id == types.NodeId(node.Id) {
continue
}
// Notify node status change if required.
newNodeInfo := api.Node{}
newNodeInfo.Id = string(id)
newNodeInfo.Status = api.Status_STATUS_OK
switch {
case nodeInfo.Status == types.NODE_STATUS_DOWN:
newNodeInfo.Status = api.Status_STATUS_OFFLINE
lastStatus, ok := c.nodeStatuses[string(id)]
if ok && lastStatus == newNodeInfo.Status {
break
}
// Check if it is a stale update
ne := c.getLatestNodeConfig(string(id))
if ne != nil && nodeInfo.GenNumber != 0 &&
nodeInfo.GenNumber < ne.GenNumber {
dlog.Warnln("Detected stale update for node ", id,
" going down, ignoring it")
c.gossip.MarkNodeHasOldGen(id)
break
}
c.nodeStatuses[string(id)] = newNodeInfo.Status
dlog.Warnln("Detected node ", id,
" to be offline due to inactivity.")
for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() {
err := e.Value.(ClusterListener).Update(&newNodeInfo)
if err != nil {
dlog.Warnln("Failed to notify ",
e.Value.(ClusterListener).String())
}
}
case nodeInfo.Status == types.NODE_STATUS_DOWN_WAITING_FOR_NEW_UPDATE:
newNodeInfo.Status = api.Status_STATUS_OFFLINE
lastStatus, ok := c.nodeStatuses[string(id)]
if ok && lastStatus == newNodeInfo.Status {
break
}
c.nodeStatuses[string(id)] = newNodeInfo.Status
dlog.Warnln("Detected node ", newNodeInfo.Id,
" to be offline due to inactivity.")
for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() {
err := e.Value.(ClusterListener).Update(&newNodeInfo)
if err != nil {
dlog.Warnln("Failed to notify ",
e.Value.(ClusterListener).String())
}
}
case nodeInfo.Status == types.NODE_STATUS_UP:
newNodeInfo.Status = api.Status_STATUS_OK
lastStatus, ok := c.nodeStatuses[string(id)]
if ok && lastStatus == newNodeInfo.Status {
break
}
c.nodeStatuses[string(id)] = newNodeInfo.Status
// A node discovered in the cluster.
dlog.Warnln("Detected node ", newNodeInfo.Id,
" to be in the cluster.")
for e := c.listeners.Front(); e != nil && c.gEnabled; e = e.Next() {
err := e.Value.(ClusterListener).Add(&newNodeInfo)
if err != nil {
dlog.Warnln("Failed to notify ",
e.Value.(ClusterListener).String())
}
}
}
// Update cache.
if nodeInfo.Value != nil {
n, ok := nodeInfo.Value.(api.Node)
if ok {
n.Status = newNodeInfo.Status
c.nodeCache[n.Id] = n
} else {
c.nodeCache[newNodeInfo.Id] = newNodeInfo
}
} else {
newNodeInfo.Status = api.Status_STATUS_OFFLINE
c.nodeCache[newNodeInfo.Id] = newNodeInfo
}
}
time.Sleep(2 * time.Second)
}
}
func TestGossiperMultipleNodesGoingUpDown(t *testing.T) {
printTestInfo()
nodes := []string{"0.0.0.0:9152", "0.0.0.0:9153",
"0.0.0.0:9154", "0.0.0.0:9155",
"0.0.0.0:9156", "0.0.0.0:9157",
"0.0.0.0:9158", "0.0.0.0:9159",
"0.0.0.0:9160", "0.0.0.0:9161"}
rand.Seed(time.Now().UnixNano())
gossipers := make(map[string]*GossiperImpl)
for i, nodeId := range nodes {
g := NewGossiperImpl(nodeId, types.NodeId(strconv.Itoa(i)))
g.SetGossipInterval(time.Duration(1500+rand.Intn(200)) * time.Millisecond)
// add one neighbor and 2 random peers
if i < len(nodes)-1 {
err := g.AddNode(nodes[i+1], types.NodeId(strconv.Itoa(i)))
if err != nil {
t.Error("Unexpected error adding node to id: ", nodeId,
" node: ", nodes[i+1])
}
} else {
err := g.AddNode(nodes[0], types.NodeId(strconv.Itoa(0)))
if err != nil {
t.Error("Unexpected error adding node to id: ", nodeId,
" node: ", nodes[0])
}
}
// to this gossiper, add two random peers
for count := 0; count < 2; {
randId := rand.Intn(len(nodes))
if randId == i {
continue
}
err := g.AddNode(nodes[randId], types.NodeId(strconv.Itoa(randId)))
if err != nil {
t.Log("Unexpected error adding node to id: ", nodeId,
" node: ", nodes[randId], " err: ", err)
} else {
count++
}
}
gossipers[nodeId] = g
time.Sleep(2000 * time.Millisecond)
}
updateFunc := func(g *GossiperImpl, id string, max int, t *testing.T) {
for i := 0; i < max; i++ {
t.Log("Updting data for ", id)
g.UpdateSelf("sameKey", strconv.Itoa(i))
g.UpdateSelf(types.StoreKey(g.NodeId()), strconv.Itoa(i*i))
time.Sleep(g.GossipInterval() + time.Duration(rand.Intn(100)))
}
}
for id, g := range gossipers {
go updateFunc(g, id, 10, t)
}
// Max duration for update is 1500 + 200 + 100 per update * 10
// = 1800 mil * 10 = 18000 mil.
// To add go fork thread, 2000 mil on top.
// Let gossip go on for another 10 seconds, after which it must settle
time.Sleep(1 * time.Minute)
// verify all of them are same
for i := 1; i < len(nodes); i++ {
t.Log("Checking equality of ", nodes[0], " and ", nodes[i])
verifyGossiperEquality(gossipers[nodes[0]], gossipers[nodes[i]], t)
}
// start another update round, however, we will shut down soem machines
// in between
for id, g := range gossipers {
go updateFunc(g, id, 10, t)
}
shutdownNodes := make(map[int]bool)
for {
randId := rand.Intn(len(nodes))
if randId == 0 {
continue
}
_, ok := shutdownNodes[randId]
if ok == false {
shutdownNodes[randId] = true
gossipers[nodes[randId]].Stop()
if len(shutdownNodes) == 3 {
break
}
}
}
time.Sleep(1 * time.Minute)
// verify all of them are same
for i := 1; i < len(nodes); i++ {
_, ok := shutdownNodes[i]
if ok {
continue
}
t.Log("Checking equality of ", nodes[0], " and ", nodes[i])
verifyGossiperEquality(gossipers[nodes[0]], gossipers[nodes[i]], t)
g := gossipers[nodes[i]]
keys := g.GetStoreKeys()
for _, key := range keys {
values := g.GetStoreKeyValue(key)
for j, nodeInfo := range values {
nodeId, _ := strconv.Atoi(string(j))
_, ok := shutdownNodes[nodeId]
if ok && nodeInfo.Status == types.NODE_STATUS_UP {
t.Error("Node not marked down: ", nodeInfo, " for node: ", nodes[i])
}
}
}
}
for i := 1; i < len(nodes); i++ {
gossipers[nodes[i]].Stop()
}
}
func TestGossiperGossipMarkOldGenNode(t *testing.T) {
printTestInfo()
nodes := []string{"0.0.0.0:9225", "0.0.0.0:9226", "0.0.0.0:9227"}
rand.Seed(time.Now().UnixNano())
gossipers := make(map[int]*GossiperImpl)
for i, nodeId := range nodes {
if i == 0 {
// node 0 never comes up
continue
}
id := types.NodeId(strconv.Itoa(i))
g := NewGossiperImpl(nodeId, id)
g.SetGossipInterval(time.Duration(200+rand.Intn(200)) * time.Millisecond)
for j, peer := range nodes {
if i == j {
continue
}
g.AddNode(peer, types.NodeId(j))
}
gossipers[i] = g
}
// each node must mark node 0 as down
key := types.StoreKey("somekey")
value := "someValue"
for i, g := range gossipers {
g.UpdateSelf(key, value+strconv.Itoa(i))
}
for i, g := range gossipers {
res := g.GetStoreKeyValue(key)
for nodeId, n := range res {
if nodeId != n.Id {
t.Error("Gossiper ", i, "Id does not match ",
nodeId, " n:", n.Id)
}
nid, ok := strconv.Atoi(string(nodeId))
if ok != nil {
t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id)
}
if nid == 0 {
if n.Status == types.NODE_STATUS_DOWN {
t.Error("Gossiper ", i,
"Expected node status not to be down: ", nodeId, " n:", n)
}
}
}
}
time.Sleep(2 * time.Second)
for i, g := range gossipers {
res := g.GetStoreKeyValue(key)
for nodeId, n := range res {
if nodeId != n.Id {
t.Error("Gossiper ", i, "Id does not match ",
nodeId, " n:", n.Id)
}
nid, ok := strconv.Atoi(string(nodeId))
if ok != nil {
t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id)
}
if nid == 0 {
if n.Status != types.NODE_STATUS_DOWN {
t.Error("Gossiper ", i,
"Expected node status to be down: ", nodeId, " n:", n)
}
} else {
if n.Status != types.NODE_STATUS_UP {
t.Error("Gossiper ", i, "Expected node to be up: ", nodeId,
" n:", n)
}
}
}
}
// Now Reset both node 0 and node 1 in node 2.
nid0 := types.NodeId(strconv.Itoa(0))
nid1 := types.NodeId(strconv.Itoa(1))
nid2 := types.NodeId(strconv.Itoa(2))
g, _ := gossipers[2]
g.MarkNodeHasOldGen(nid0)
g.MarkNodeHasOldGen(nid1)
// Update value in node 1
g, _ = gossipers[1]
g.UpdateSelf(key, value+"__1")
// Node must be up now
g, _ = gossipers[2]
res := g.GetStoreKeyValue(key)
for nodeId, n := range res {
if nid2 == nodeId {
continue
}
if nodeId != n.Id {
t.Error("Id does not match ", nodeId, " n:", n.Id)
}
_, ok := strconv.Atoi(string(nodeId))
if ok != nil {
t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id)
}
if n.Status != types.NODE_STATUS_WAITING_FOR_NEW_UPDATE {
t.Error("Expected node status to be down: ", nodeId, " n:", n)
}
}
time.Sleep(800 * time.Millisecond)
res = g.GetStoreKeyValue(key)
g1Node, _ := res[nid1]
if g1Node.Status != types.NODE_STATUS_UP {
t.Error("Expected node to be up ", g1Node)
}
// Sleep for 15*(node death interval) i.e. 15*400 mil i.e. 6 seconds
// , both nodes must me marked down
time.Sleep(35 * time.Second)
res = g.GetStoreKeyValue(key)
for nodeId, n := range res {
if nid2 == nodeId {
continue
}
if nodeId != n.Id {
t.Error("Id does not match ", nodeId, " n:", n.Id)
}
_, ok := strconv.Atoi(string(nodeId))
if ok != nil {
t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id)
}
if n.Status != types.NODE_STATUS_DOWN {
t.Error("Expected node to be down: ", nodeId, " n:", n)
}
}
// Test that the node does not is marked down after reset
g.MarkNodeHasOldGen(nid1)
time.Sleep(35 * time.Second)
res = g.GetStoreKeyValue(key)
g1Node, _ = res[nid1]
if g1Node.Status != types.NODE_STATUS_DOWN_WAITING_FOR_NEW_UPDATE {
t.Error("Expected node to be down waiting for update ", g1Node)
}
for _, g := range gossipers {
g.Stop()
}
}
func TestGossiperOneNodeNeverGossips(t *testing.T) {
printTestInfo()
nodes := []string{"0.0.0.0:9222", "0.0.0.0:9223",
"0.0.0.0:9224"}
rand.Seed(time.Now().UnixNano())
gossipers := make(map[int]*GossiperImpl)
for i, nodeId := range nodes {
if i == 0 {
// node 0 never comes up
continue
}
id := types.NodeId(strconv.Itoa(i))
g := NewGossiperImpl(nodeId, id)
g.SetGossipInterval(time.Duration(200+rand.Intn(200)) * time.Millisecond)
for j, peer := range nodes {
if i == j {
continue
}
g.AddNode(peer, types.NodeId(j))
}
gossipers[i] = g
}
// each node must mark node 0 as down
key := types.StoreKey("somekey")
value := "someValue"
for i, g := range gossipers {
g.UpdateSelf(key, value+strconv.Itoa(i))
}
for i, g := range gossipers {
res := g.GetStoreKeyValue(key)
for nodeId, n := range res {
if nodeId != n.Id {
t.Error("Gossiper ", i, "Id does not match ",
nodeId, " n:", n.Id)
}
nid, ok := strconv.Atoi(string(nodeId))
if ok != nil {
t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id)
}
if nid == 0 {
if n.Status == types.NODE_STATUS_DOWN {
t.Error("Gossiper ", i,
"Expected node status not to be down: ", nodeId, " n:", n)
}
}
}
}
time.Sleep(2 * time.Second)
for i, g := range gossipers {
res := g.GetStoreKeyValue(key)
for nodeId, n := range res {
if nodeId != n.Id {
t.Error("Gossiper ", i, "Id does not match ",
nodeId, " n:", n.Id)
}
nid, ok := strconv.Atoi(string(nodeId))
if ok != nil {
t.Error("Failed to convert node to id ", nodeId, " n.Id", n.Id)
}
if nid == 0 {
if n.Status != types.NODE_STATUS_DOWN {
t.Error("Gossiper ", i,
"Expected node status to be down: ", nodeId, " n:", n)
}
} else {
if n.Status != types.NODE_STATUS_UP {
t.Error("Gossiper ", i, "Expected node to be up: ", nodeId,
" n:", n)
}
}
}
}
for _, g := range gossipers {
g.Stop()
}
}
