// watchNode watch a named node for leader selection when failover
func watchCometNode(conn *zk.Conn, node, fpath string, retry, ping time.Duration, ch chan *CometNodeEvent) {
fpath = path.Join(fpath, node)
for {
nodes, watch, err := myzk.GetNodesW(conn, fpath)
if err == myzk.ErrNodeNotExist {
log.Warn("zk don't have node \"%s\"", fpath)
break
} else if err == myzk.ErrNoChild {
log.Warn("zk don't have any children in \"%s\", retry in %d second", fpath, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
} else if err != nil {
log.Error("zk path: \"%s\" getNodes error(%v), retry in %d second", fpath, err, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
}
// leader selection
sort.Strings(nodes)
if info, err := registerCometNode(conn, nodes[0], fpath, retry, ping, true); err != nil {
log.Error("zk path: \"%s\" registerCometNode error(%v)", fpath, err)
time.Sleep(waitNodeDelaySecond)
continue
} else {
// update node info
ch <- &CometNodeEvent{Event: eventNodeUpdate, Key: node, Value: info}
}
// blocking receive event
event := <-watch
log.Info("zk path: \"%s\" receive a event: (%v)", fpath, event)
}
// WARN, if no persistence node and comet rpc not config
log.Warn("zk path: \"%s\" never watch again till recreate", fpath)
}
// watchMessageRoot watch the message root path.
func watchMessageRoot(conn *zk.Conn, fpath string, ch chan *MessageNodeEvent) error {
for {
nodes, watch, err := myzk.GetNodesW(conn, fpath)
if err == myzk.ErrNodeNotExist {
glog.Warningf("zk don't have node \"%s\", retry in %d second", fpath, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
} else if err == myzk.ErrNoChild {
glog.Warningf("zk don't have any children in \"%s\", retry in %d second", fpath, waitNodeDelay)
// all child died, kick all the nodes
for _, client := range MessageRPC.Clients {
glog.V(1).Infof("node: \"%s\" send del node event", client.Addr)
ch <- &MessageNodeEvent{Event: eventNodeDel, Key: &RPCClient{Addr: client.Addr, Weight: client.Weight}}
}
time.Sleep(waitNodeDelaySecond)
continue
} else if err != nil {
glog.Errorf("getNodes error(%v), retry in %d second", err, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
}
nodesMap := map[string]bool{}
// handle new add nodes
for _, node := range nodes {
data, _, err := conn.Get(path.Join(fpath, node))
if err != nil {
glog.Errorf("zk.Get(\"%s\") error(%v)", path.Join(fpath, node), err)
continue
}
// may contains many addrs split by ,
//addrs := strings.Split(string(data), ",")
addrInfos, err := parseMessageAddr(string(data))
if err != nil {
glog.Errorf("parseMessageAddr(\"%s\") error(%v)", string(data), err)
continue
}
for _, addInfo := range addrInfos {
// if not exists in old map then trigger a add event
if _, ok := MessageRPC.Clients[addInfo.Addr]; !ok {
ch <- &MessageNodeEvent{Event: eventNodeAdd, Key: addInfo}
}
nodesMap[addInfo.Addr] = true
}
}
// handle delete nodes
for _, client := range MessageRPC.Clients {
if _, ok := nodesMap[client.Addr]; !ok {
ch <- &MessageNodeEvent{Event: eventNodeDel, Key: client}
}
}
// blocking wait node changed
event := <-watch
glog.Infof("zk path: \"%s\" receive a event %v", fpath, event)
}
}
// watchMessageRoot watch the message root path.
func watchMessageRoot(conn *zk.Conn, fpath string, ch chan *MessageNodeEvent) error {
for {
nodes, watch, err := myzk.GetNodesW(conn, fpath)
if err == myzk.ErrNodeNotExist {
log.Warn("zk don't have node \"%s\", retry in %d second", fpath, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
} else if err == myzk.ErrNoChild {
log.Warn("zk don't have any children in \"%s\", retry in %d second", fpath, waitNodeDelay)
// all child died, kick all the nodes
for _, client := range MessageRPC.Clients {
log.Debug("node: \"%s\" send del node event", client.Addr)
ch <- &MessageNodeEvent{Event: eventNodeDel, Key: &WeightRpc{Addr: client.Addr, Weight: client.Weight}}
}
time.Sleep(waitNodeDelaySecond)
continue
} else if err != nil {
log.Error("getNodes error(%v), retry in %d second", err, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
}
nodesMap := map[string]bool{}
// handle new add nodes
for _, node := range nodes {
data, _, err := conn.Get(path.Join(fpath, node))
if err != nil {
log.Error("zk.Get(\"%s\") error(%v)", path.Join(fpath, node), err)
continue
}
// parse message node info
nodeInfo := &MessageNodeInfo{}
if err := json.Unmarshal(data, nodeInfo); err != nil {
log.Error("json.Unmarshal(\"%s\", nodeInfo) error(%v)", string(data), err)
continue
}
for _, addr := range nodeInfo.Rpc {
// if not exists in old map then trigger a add event
if _, ok := MessageRPC.Clients[addr]; !ok {
ch <- &MessageNodeEvent{Event: eventNodeAdd, Key: &WeightRpc{Addr: addr, Weight: nodeInfo.Weight}}
}
nodesMap[addr] = true
}
}
// handle delete nodes
for _, client := range MessageRPC.Clients {
if _, ok := nodesMap[client.Addr]; !ok {
ch <- &MessageNodeEvent{Event: eventNodeDel, Key: client}
}
}
// blocking wait node changed
event := <-watch
log.Info("zk path: \"%s\" receive a event %v", fpath, event)
}
}
func watchCometRoot(conn *zk.Conn, fpath string, ch chan *CometNodeEvent) error {
for {
nodes, watch, err := myzk.GetNodesW(conn, fpath)
if err == myzk.ErrNodeNotExist {
glog.Warningf("zk don't have node \"%s\", retry in %d second", fpath, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
} else if err == myzk.ErrNoChild {
glog.Warningf("zk don't have any children in \"%s\", retry in %d second", fpath, waitNodeDelay)
for node, _ := range cometNodeInfoMap {
ch <- &CometNodeEvent{Event: eventNodeDel, Key: node}
}
time.Sleep(waitNodeDelaySecond)
continue
} else if err != nil {
glog.Errorf("getNodes error(%v), retry in %d second", err, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
}
nodesMap := map[string]bool{}
// handle new add nodes
for _, node := range nodes {
if _, ok := cometNodeInfoMap[node]; !ok {
ch <- &CometNodeEvent{Event: eventNodeAdd, Key: node}
}
nodesMap[node] = true
}
// handle delete nodes
for node, _ := range cometNodeInfoMap {
if _, ok := nodesMap[node]; !ok {
ch <- &CometNodeEvent{Event: eventNodeDel, Key: node}
}
}
// blocking wait node changed
event := <-watch
glog.Infof("zk path: \"%s\" receive a event %v", fpath, event)
}
}