func (ckr *Checker) checkNodes() {
somethingheppens := false
nodes := ckr.topology.GetRelatives()
for _, nd := range nodes {
if _, ok := ckr.nodeError[nd.Node.Name]; !ok {
ckr.nodeError[nd.Node.Name] = 0
}
if err := ckr.outcomingQueue.Caller.Ping(cluster.GetCurrentNode().Node.Name, nd.Node); err != nil {
ckr.nodeError[nd.Node.Name] = ckr.nodeError[nd.Node.Name] + 1
} else {
ckr.nodeError[nd.Node.Name] = 0
}
}
// notify faults to other nodes
for name, count := range ckr.nodeError {
if count >= ErrorThreshold {
ckr.notifyFaultNotification(name)
somethingheppens = true
}
}
// clean inactive nodes
for _, nd := range ckr.topology.GetInactiveNodes() {
if nd.UpdateDate.Before(time.Now().Add(-1 * time.Minute)) {
ckr.topology.RemoveNode(nd.Node.Name)
somethingheppens = true
}
}
if somethingheppens {
// write conf
ckr.cnf.WriteTmp()
}
}
func (ckr *Checker) checkNodes() {
nodes := ckr.topology.GetRelatives()
for _, nd := range nodes {
if _, ok := ckr.nodeError[nd.Node.Name]; !ok {
ckr.nodeError[nd.Node.Name] = 0
}
if err := ckr.outcomingQueue.Caller.Ping(cluster.GetCurrentNode().Node.Name, nd.Node); err != nil {
ckr.nodeError[nd.Node.Name] = ckr.nodeError[nd.Node.Name] + 1
} else {
ckr.nodeError[nd.Node.Name] = 0
}
}
for name, count := range ckr.nodeError {
if count >= ErrorThreshold {
ckr.notifyFaultNotification(name)
}
}
}
// Execute remote operation on destination server
func (nc *NodeCaller) ExecuteOperation(obj *storage.MetaDataUpdObj, destination *cluster.OvoNode, operation string) error {
defer func() {
// executes normally even if there is a panic
if err2 := recover(); err2 != nil {
//remove the client
nc.deleteCaller(destination.Name)
}
}()
var client *rpc.Client
var ok bool
if client, ok = nc.getCaller(destination.Name); !ok {
client = nc.createClient(destination)
}
rpccmd := &command.RpcCommand{Source: cluster.GetCurrentNode().Node.Name, OpCode: operation, Obj: obj}
var reply int = 0
err := client.Call("InnerServer.ExecuteCommand", rpccmd, &reply)
if err != nil {
log.Println("InnerServer.ExecuteCommand error: ", err)
}
return err
}