// AddNode add a node and watch it, and notice Comet to migrate node
func AddNode(node string) error {
_, ok := NodeInfoMap[node]
if ok {
return ErrNodeExist
}
var nodes []string
for n, _ := range NodeInfoMap {
nodes = append(nodes, n)
}
nodes = append(nodes, node)
// Notice Comet to migrate node
if err := channelRPCMigrate(nodes, NodeInfoMap); err != nil {
return err
}
// Watch the node
go watchFirstService(node)
// Update Comet hash, because of nodes are changed
CometHash = hash.NewKetama2(nodes, 255)
return nil
}
// AddNode add a node and watch it
func AddNode(node string) error {
NodeInfoMapLock.RLock()
defer NodeInfoMapLock.RUnlock()
_, ok := NodeInfoMap[node]
if ok {
return ErrNodeExist
}
var nodes []string
for n, _ := range NodeInfoMap {
nodes = append(nodes, n)
}
nodes = append(nodes, node)
// Notice comet to migrate node
if err := ChannelRPCMigrate(nodes, NodeInfoMap); err != nil {
return err
}
go watchFirstServer(node)
CometHash = hash.NewKetama2(nodes, 255)
return nil
}
// DelNode disconnect and delete a node
func DelNode(node string) error {
var (
nodes []string
info *NodeInfo
)
NodeInfoMapLock.Lock()
defer NodeInfoMapLock.Unlock()
for n, c := range NodeInfoMap {
if n == node {
info = c
continue
}
nodes = append(nodes, n)
}
CometHash = hash.NewKetama2(nodes, 255)
delete(NodeInfoMap, node)
if info != nil && info.PubRPC != nil {
info.PubRPC.Close()
info.PubRPC = nil
}
// Notice comet to migrate node
if err := ChannelRPCMigrate(nodes, NodeInfoMap); err != nil {
return err
}
return nil
}
// BeginWatchNode start watch all of nodes
func BeginWatchNode() error {
nodes, err := getNodes(Conf.Zookeeper.RootPath)
if err != nil {
return err
}
CometHash = hash.NewKetama2(nodes, 255)
watchNodes(nodes)
return nil
}
// Publish expored a method for publishing a message for the channel
func (c *ChannelRPC) Migrate(args *myrpc.ChannelMigrateArgs, ret *int) error {
if len(args.Nodes) == 0 {
*ret = myrpc.ParamErr
return nil
}
// find current node exists in new nodes
has := false
for _, str := range args.Nodes {
if str == Conf.ZookeeperNode {
has = true
}
}
if !has {
Log.Crit("make sure your migrate nodes right, there is no %s in nodes, this will cause all the node hit miss", Conf.ZookeeperNode)
*ret = myrpc.MigrateErr
return nil
}
// init ketama
ketama := hash.NewKetama2(args.Nodes, args.Vnode)
channels := []Channel{}
keys := []string{}
// get all the channel lock
for i, c := range UserChannel.Channels {
c.Lock()
for k, v := range c.Data {
hn := ketama.Node(k)
if hn != Conf.ZookeeperNode {
channels = append(channels, v)
keys = append(keys, k)
Log.Debug("migrate key:\"%s\" hit node:\"%s\"", k, hn)
}
}
for _, k := range keys {
delete(c.Data, k)
Log.Info("migrate delete channel key \"%s\"", k)
}
c.Unlock()
Log.Info("migrate channel bucket:%d finished", i)
}
// close all the migrate channels
Log.Info("close all the migrate channels")
for _, channel := range channels {
if err := channel.Close(); err != nil {
Log.Error("channel.Close() error(%v)", err)
continue
}
}
*ret = myrpc.OK
Log.Info("close all the migrate channels finished")
return nil
}
// BeginWatchNode start watch all of nodes which registered in zookeeper
func BeginWatchNode() error {
nodes, err := getNodes(Conf.ZKCometPath)
if err != nil {
Log.Error("getNodes(\"%s\") error(%v)", Conf.ZKCometPath, err)
return err
}
// Update Comet hash, cause nodes are changed
CometHash = hash.NewKetama2(nodes, 255)
// Watch all of nodes
watchNodes(nodes)
return nil
}
// BeginWatchNode start watch all of nodes which registered in zookeeper
func BeginWatchNode() error {
nodes, err := getNodes(Conf.ZKCometPath)
if err != nil {
Log.Error("getNodes(\"%s\") error(%v)", Conf.ZKCometPath, err)
return err
}
for _, n := range nodes {
NodeInfoMap[n] = nil
}
// Update Comet hash
CometHash = hash.NewKetama2(nodes, 255)
// Watch all of nodes
watchNodes(nodes)
return nil
}
// DelNode disconnect and delete a node, and notice Comet to migrate node
func DelNode(node string) error {
var (
nodes []string
info *NodeInfo
)
if _, ok := NodeInfoMap[node]; !ok {
return nil
}
for n, c := range NodeInfoMap {
if n == node {
info = c
continue
}
nodes = append(nodes, n)
}
// Update Comet hash, cause nodes are changed
CometHash = hash.NewKetama2(nodes, 255)
// Delete node from map before call Migrate RPC interface of Comet, cause needn`t to notice deleted node
tmpMap := make(map[string]*NodeInfo)
for n, i := range NodeInfoMap {
tmpMap[n] = i
}
delete(tmpMap, node)
NodeInfoMap = tmpMap
if info != nil && info.PubRPC != nil {
info.PubRPC.Close()
}
// Notice Comet to migrate node
if err := channelRPCMigrate(nodes, NodeInfoMap); err != nil {
return err
}
return nil
}
// handleNodeEvent add and remove NodeInfo, copy the src map to a new map then replace the variable.
func handleNodeEvent(conn *zk.Conn, path string, ch chan *NodeEvent) {
for {
ev := <-ch
// copy map from src
tmpMap := make(map[string]*NodeInfo, len(NodeInfoMap))
for k, v := range NodeInfoMap {
tmpMap[k] = v
}
// handle event
if ev.Event == EventNodeAdd {
Log.Info("add node: \"%s\"", ev.Key)
tmpMap[ev.Key] = nil
go watchNode(conn, ev.Key, path, ch)
} else if ev.Event == EventNodeDel {
Log.Info("del node: \"%s\"", ev.Key)
delete(tmpMap, ev.Key)
} else if ev.Event == EventNodeUpdate {
Log.Info("update node: \"%s\"", ev.Key)
tmpMap[ev.Key] = ev.Value
} else {
Log.Crit("unknown node event: %d", ev.Event)
panic("unknown node event")
}
// if exist old node info, close
if info, ok := NodeInfoMap[ev.Key]; ok {
if info != nil {
info.Close()
Log.Info("close old node info: \"%s\"", ev.Key)
}
}
// use the tmpMap atomic replace the global NodeInfoMap
NodeInfoMap = tmpMap
// update comet hash, cause node has changed
nodes := make([]string, 0, len(tmpMap))
for k, _ := range tmpMap {
nodes = append(nodes, k)
}
cometHash = hash.NewKetama2(nodes, 255)
Log.Debug("NodeInfoMap len: %d", len(NodeInfoMap))
}
}
// handleCometNodeEvent add and remove CometNodeInfo, copy the src map to a new map then replace the variable.
func handleCometNodeEvent(conn *zk.Conn, fpath string, retry, ping time.Duration, ch chan *CometNodeEvent) {
for {
ev := <-ch
// copy map from src
tmpMap := make(map[string]*CometNodeInfo, len(cometNodeInfoMap))
for k, v := range cometNodeInfoMap {
tmpMap[k] = v
}
// handle event
if ev.Event == eventNodeAdd {
glog.Infof("add node: \"%s\"", ev.Key)
tmpMap[ev.Key] = nil
go watchCometNode(conn, ev.Key, fpath, retry, ping, ch)
} else if ev.Event == eventNodeDel {
glog.Infof("del node: \"%s\"", ev.Key)
delete(tmpMap, ev.Key)
} else if ev.Event == eventNodeUpdate {
glog.Infof("update node: \"%s\"", ev.Key)
tmpMap[ev.Key] = ev.Value
} else {
glog.Errorf("unknown node event: %d", ev.Event)
panic("unknown node event")
}
// if exist old node info, destroy
if info, ok := cometNodeInfoMap[ev.Key]; ok {
if info != nil {
info.CometRPC.Destroy()
}
}
// use the tmpMap atomic replace the global cometNodeInfoMap
cometNodeInfoMap = tmpMap
// update comet hash, cause node has changed
nodes := make([]string, 0, len(tmpMap))
for k, _ := range tmpMap {
nodes = append(nodes, k)
}
cometHash = hash.NewKetama2(nodes, 255)
glog.V(1).Infof("cometNodeInfoMap len: %d", len(cometNodeInfoMap))
}
}
// watchNodes watch the first service under the node, and keep rpc connecting with Comet RPC,
// the first service must be alive
func watchFirstService(node string) {
defer func() {
if err := recover(); err != nil {
Log.Error("watching node goroutine panic node:\"%s\" error(%v), stop watching", node, err)
var nodes []string
// Delete node from map
tmpMap := make(map[string]*NodeInfo)
for n, i := range NodeInfoMap {
if n == node {
continue
}
tmpMap[n] = i
nodes = append(nodes, n)
}
NodeInfoMap = tmpMap
// Update Comet hash, cause node has changed
CometHash = hash.NewKetama2(nodes, 255)
}
}()
path := fmt.Sprintf("%s/%s", Conf.ZKCometPath, node)
for {
subNodes, watch, err := getNodesW(path)
if err != nil {
// If no subNode, then check repeatedly
if err == ErrNoChild {
Log.Warn("get service of node:\"%s\" error(%v), recheck after 5 seconds", node, err)
time.Sleep(5 * time.Second)
continue
}
Log.Error("get service of node:\"%s\" error(%v), stop watching", node, err)
break
}
// Get service infomation
sort.Strings(subNodes)
data, _, err := zk.Get(fmt.Sprintf("%s/%s", path, subNodes[0]))
if err != nil {
Log.Error("watch node:\"%s\", subNode:\"%s\", error(%v)", node, subNodes[0], err)
time.Sleep(5 * time.Second)
continue
}
// Fecth and parse push service connection info
subAddr, err := parseZKData(data)
if err != nil {
Log.Error("get subNode data error node:\"%s\", subNode:\"%s\", error(%v)", node, subNodes[0], err)
time.Sleep(5 * time.Second)
continue
}
info := NodeInfoMap[node]
if info != nil {
info.SubAddr = subAddr
if info.PubRPC != nil {
info.PubRPC.Close()
}
} else {
info = &NodeInfo{SubAddr: subAddr}
}
tmpMap := make(map[string]*NodeInfo)
for n, i := range NodeInfoMap {
tmpMap[n] = i
}
tmpMap[node] = info
NodeInfoMap = tmpMap
// ReDial RPC
addr, ok := subAddr[ProtocolRPC]
if ok {
r, err := rpc.Dial("tcp", addr)
if err != nil {
Log.Error("rpc.Dial(\"tcp\", \"%s\") error(%v) node:\"%s\", subNode:\"%s\", recheck after 5 seconds", addr, err, node, subNodes[0])
time.Sleep(5 * time.Second)
continue
}
info.PubRPC = r
} else {
Log.Error("rpc.Dial(\"tcp\", \"%s\") error node:\"%s\", subNode:\"%s\" error(no rpc address)", addr, node, subNodes[0])
}
Log.Info("begin watching node:\"%s\" addr:\"%v\"", node, subAddr)
event := <-watch
Log.Warn("end watching node:\"%s\" addr:\"%v\" event:\"%v\", retry to watch", node, subAddr, event)
}
// Delete node
Log.Warn("stop watching node:\"%s\"", node)
if err := DelNode(node); err != nil {
Log.Error("stop watching node:\"%s\" error(%v)", node, err)
return
}
}