// NewExplorer creates a new Explorer instance with a name,
// application and balancer finders and zookeeper connection
// to persist versioning information
func NewExplorer(name string, af application.Finder, bf balancer.Finder, zc *zk.Conn, zp string) (*Explorer, error) {
s := state.State{}
ss, _, err := zc.Get(zp)
if err != nil {
if err != zk.ErrNoNode {
return nil, err
}
s.Versions = map[string]state.Versions{}
} else {
err := json.Unmarshal(ss, &s)
if err != nil {
return nil, err
}
}
return &Explorer{
name: name,
af: af,
bf: bf,
zookeeper: zc,
zp: zp,
state: s,
mutex: sync.Mutex{},
}, nil
}
func mirrorNode(conn *zk.Conn, node *NodeTree) error {
var err error
var children []string
nodePath := appendPath(node.PathPrefix, node.Name)
if children, _, err = conn.Children(nodePath); err != nil {
return err
}
for _, child := range children {
var data []byte
childPath := appendPath(nodePath, child)
if data, _, err = conn.Get(childPath); err != nil {
return err
}
childNode := NewNodeTree(child, nodePath, data)
node.AddChild(childNode)
if err := mirrorNode(conn, childNode); err != nil {
return err
}
}
return nil
}
func registerNode(conn *zk.Conn, node, path string) (*NodeInfo, error) {
path = path + "/" + node
data, _, err := conn.Get(path)
if err != nil {
Log.Error("zk.Get(\"%s\") error(%v)", path, err)
return nil, err
}
// fetch and parse comet info
addr, err := parseNode(string(data))
if err != nil {
Log.Error("parseNode(\"%s\") error(%v)", string(data), err)
return nil, err
}
info := &NodeInfo{Addr: addr}
rpcAddr, ok := addr[ProtocolRPC]
if !ok || len(rpcAddr) == 0 {
Log.Crit("zk nodes: \"%s\" don't have rpc addr", path)
return nil, ErrCometRPC
}
// init comet rpc
// TODO support many rpc
r, err := rpc.Dial("tcp", rpcAddr[0])
if err != nil {
Log.Crit("rpc.Dial(\"%s\") error(%v)", rpcAddr[0], err)
return nil, ErrCometRPC
}
info.PubRPC = r
Log.Info("zk path: \"%s\" register nodes: \"%s\"", path, node)
return info, nil
}
func DeleteNodebyData(path string, conn *zk.Conn, data []byte) error {
children, _, err := conn.Children(path)
if err != nil {
glog.Warning("Could not list children")
return err
}
for _, child := range children {
child_path := path + "/" + child
child_data, _, err := conn.Get(child_path)
if err != nil {
glog.Warning("Could not get data for %s", child_path)
continue
}
if bytes.Compare(data, child_data) == 0 {
for i := 0; i < 5; i++ {
_, stats, _ := conn.Get(child_path)
err = conn.Delete(child_path, stats.Version)
if err == nil || err == zk.ErrNoNode {
return nil
}
}
glog.Error("Could not delete matched node %s", child_path)
}
}
return nil
}
func recursiveDelete(c *zk.Conn, path string) error {
children, _, err := c.Children(path)
if err != nil && err != zk.ErrNoNode {
logError("err finding children of %s", path)
return err
}
for _, child := range children {
err := recursiveDelete(c, path+"/"+child)
if err != nil && err != zk.ErrNoNode {
logError("err deleting %s", child)
return err
}
}
// get version
_, stat, err := c.Get(path)
if err != nil && err != zk.ErrNoNode {
logError("err getting version of %s", path)
return err
}
if err := c.Delete(path, stat.Version); err != nil && err != zk.ErrNoNode {
return err
}
return nil
}
func NewExplorer(d Discoverer, zc *zk.Conn, zp string, location ExplorerLocation) (*Explorer, error) {
state := State{}
ss, _, err := zc.Get(zp)
if err != nil {
if err != zk.ErrNoNode {
return nil, err
}
state.Versions = map[string]Versions{}
} else {
err := json.Unmarshal(ss, &state)
if err != nil {
return nil, err
}
}
return &Explorer{
discoverer: d,
zookeeper: zc,
zp: zp,
state: state,
location: location,
mutex: sync.Mutex{},
}, nil
}
func All(conn *zk.Conn, zkConf conf.Zookeeper) (map[string]Service, error) {
err := ensurePathExists(conn, zkConf.Path)
if err != nil {
return nil, err
}
services := map[string]Service{}
keys, _, err2 := conn.Children(zkConf.Path)
if err2 != nil {
return nil, err2
}
for _, childPath := range keys {
bite, _, e := conn.Get(zkConf.Path + "/" + childPath)
if e != nil {
return nil, e
break
}
appId, _ := unescapeSlashes(childPath)
services[appId] = Service{Id: appId, Acl: string(bite)}
}
return services, nil
}
// watchMessageRoot watch the message root path.
func watchMessageRoot(conn *zk.Conn, fpath string, ch chan *MessageNodeEvent) error {
for {
nodes, watch, err := myzk.GetNodesW(conn, fpath)
if err == myzk.ErrNodeNotExist {
glog.Warningf("zk don't have node \"%s\", retry in %d second", fpath, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
} else if err == myzk.ErrNoChild {
glog.Warningf("zk don't have any children in \"%s\", retry in %d second", fpath, waitNodeDelay)
// all child died, kick all the nodes
for _, client := range MessageRPC.Clients {
glog.V(1).Infof("node: \"%s\" send del node event", client.Addr)
ch <- &MessageNodeEvent{Event: eventNodeDel, Key: &RPCClient{Addr: client.Addr, Weight: client.Weight}}
}
time.Sleep(waitNodeDelaySecond)
continue
} else if err != nil {
glog.Errorf("getNodes error(%v), retry in %d second", err, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
}
nodesMap := map[string]bool{}
// handle new add nodes
for _, node := range nodes {
data, _, err := conn.Get(path.Join(fpath, node))
if err != nil {
glog.Errorf("zk.Get(\"%s\") error(%v)", path.Join(fpath, node), err)
continue
}
// may contains many addrs split by ,
//addrs := strings.Split(string(data), ",")
addrInfos, err := parseMessageAddr(string(data))
if err != nil {
glog.Errorf("parseMessageAddr(\"%s\") error(%v)", string(data), err)
continue
}
for _, addInfo := range addrInfos {
// if not exists in old map then trigger a add event
if _, ok := MessageRPC.Clients[addInfo.Addr]; !ok {
ch <- &MessageNodeEvent{Event: eventNodeAdd, Key: addInfo}
}
nodesMap[addInfo.Addr] = true
}
}
// handle delete nodes
for _, client := range MessageRPC.Clients {
if _, ok := nodesMap[client.Addr]; !ok {
ch <- &MessageNodeEvent{Event: eventNodeDel, Key: client}
}
}
// blocking wait node changed
event := <-watch
glog.Infof("zk path: \"%s\" receive a event %v", fpath, event)
}
}
// watchMessageRoot watch the message root path.
func watchMessageRoot(conn *zk.Conn, fpath string, ch chan *MessageNodeEvent) error {
for {
nodes, watch, err := myzk.GetNodesW(conn, fpath)
if err == myzk.ErrNodeNotExist {
log.Warn("zk don't have node \"%s\", retry in %d second", fpath, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
} else if err == myzk.ErrNoChild {
log.Warn("zk don't have any children in \"%s\", retry in %d second", fpath, waitNodeDelay)
// all child died, kick all the nodes
for _, client := range MessageRPC.Clients {
log.Debug("node: \"%s\" send del node event", client.Addr)
ch <- &MessageNodeEvent{Event: eventNodeDel, Key: &WeightRpc{Addr: client.Addr, Weight: client.Weight}}
}
time.Sleep(waitNodeDelaySecond)
continue
} else if err != nil {
log.Error("getNodes error(%v), retry in %d second", err, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
}
nodesMap := map[string]bool{}
// handle new add nodes
for _, node := range nodes {
data, _, err := conn.Get(path.Join(fpath, node))
if err != nil {
log.Error("zk.Get(\"%s\") error(%v)", path.Join(fpath, node), err)
continue
}
// parse message node info
nodeInfo := &MessageNodeInfo{}
if err := json.Unmarshal(data, nodeInfo); err != nil {
log.Error("json.Unmarshal(\"%s\", nodeInfo) error(%v)", string(data), err)
continue
}
for _, addr := range nodeInfo.Rpc {
// if not exists in old map then trigger a add event
if _, ok := MessageRPC.Clients[addr]; !ok {
ch <- &MessageNodeEvent{Event: eventNodeAdd, Key: &WeightRpc{Addr: addr, Weight: nodeInfo.Weight}}
}
nodesMap[addr] = true
}
}
// handle delete nodes
for _, client := range MessageRPC.Clients {
if _, ok := nodesMap[client.Addr]; !ok {
ch <- &MessageNodeEvent{Event: eventNodeDel, Key: client}
}
}
// blocking wait node changed
event := <-watch
log.Info("zk path: \"%s\" receive a event %v", fpath, event)
}
}
func deleteIfExists(path string, conn *zk.Conn) error {
_, stat, err := conn.Get(path)
if err != nil && err != zk.ErrNoNode {
return err
}
if err != zk.ErrNoNode {
if err := conn.Delete(path, stat.Version); err != nil {
return err
}
}
return nil
}
func bench(conn *zk.Conn, value []byte, ratio float64) {
choice := rand.Float64()
atomic.AddInt32(&reqCnt, 1)
if choice >= ratio {
// write
conn.Delete("/_test_load_obliterator", 0)
conn.Create("/_test_load_obliterator", value, 1, zk.WorldACL(zk.PermAll))
} else {
// read
conn.Get("/_test_load_obliterator")
}
}
// registerCometNode get infomation of comet node
func registerCometNode(conn *zk.Conn, node, fpath string, retry, ping time.Duration, startPing bool) (info *CometNodeInfo, err error) {
// get current node info from zookeeper
fpath = path.Join(fpath, node)
data, _, err := conn.Get(fpath)
if err != nil {
log.Error("zk.Get(\"%s\") error(%v)", fpath, err)
return
}
info = &CometNodeInfo{}
if err = json.Unmarshal(data, info); err != nil {
log.Error("json.Unmarshal(\"%s\", nodeData) error(%v)", string(data), err)
return
}
if len(info.RpcAddr) == 0 {
log.Error("zk nodes: \"%s\" don't have rpc addr", fpath)
err = ErrCometRPC
return
}
// get old node info for finding the old rpc connection
oldInfo := cometNodeInfoMap[node]
// init comet rpc
clients := make(map[string]*WeightRpc, len(info.RpcAddr))
for _, addr := range info.RpcAddr {
var (
r *rpc.Client
)
if oldInfo != nil && oldInfo.Rpc != nil {
if wr, ok := oldInfo.Rpc.Clients[addr]; ok && wr.Client != nil {
// reuse the rpc connection must let old client = nil, avoid reclose rpc.
oldInfo.Rpc.Clients[addr].Client = nil
r = wr.Client
}
}
if r == nil {
if r, err = rpc.Dial("tcp", addr); err != nil {
log.Error("rpc.Dial(\"%s\") error(%v)", addr, err)
return
}
log.Debug("node:%s addr:%s rpc reconnect", node, addr)
}
clients[addr] = &WeightRpc{Weight: 1, Addr: addr, Client: r}
}
// comet rpc use rand load balance
lb, err := NewRandLB(clients, cometService, retry, ping, startPing)
if err != nil {
log.Error("NewRandLR() error(%v)", err)
return
}
info.Rpc = lb
log.Info("zk path: \"%s\" register nodes: \"%s\"", fpath, node)
return
}
// zkData create zookeeper path, if path exists ignore error, and set node data.
func zkData(conn *zk.Conn, redisMaster string) error {
node := path.Join(conf.ZKPath, conf.Node)
tpath := ""
for _, str := range strings.Split(conf.ZKPath, "/")[1:] {
tpath = path.Join(tpath, "/", str)
log.Info("create zookeeper path: \"%s\"", tpath)
_, err := conn.Create(tpath, []byte(""), 0, zk.WorldACL(zk.PermAll))
if err != nil {
if err == zk.ErrNodeExists {
log.Warn("zk.create(\"%s\") exists", tpath)
} else {
log.Error("zk.create(\"%s\") error(%v)", tpath, err)
return err
}
}
}
if _, err := conn.Create(node, []byte{}, 0, zk.WorldACL(zk.PermAll)); err != nil {
if err == zk.ErrNodeExists {
oData, stat, err := conn.Get(node)
if err != nil {
log.Error("zk.Get(\"%s\") error(%v)", node, err)
return err
}
ni := unmarshal(oData)
if ni != nil || len(ni.Servers) == 0 {
log.Warn("node have not data")
return nil
}
data := marshal(ni, redisMaster)
if len(data) == 0 {
log.Warn("marshal error")
return nil
}
if bytes.Equal(oData, data) {
log.Warn("zk data same, no change")
return nil
}
if _, err = conn.Set(node, data, stat.Version); err != nil {
log.Error("zk.Set(\"%s\", data, 0) error(%v)", node, err)
return err
}
log.Info("zk update data: \"%s\"", node)
} else {
log.Error("zk.create(\"%s\") error(%v)", tpath, err)
return err
}
}
return nil
}
func readMember(conn *zk.Conn, path string) (Member, error) {
data, _, err := conn.Get(path)
if err != nil {
if err == zk.ErrNoNode {
return Member{}, fmt.Errorf("Node %s does not exist.", path)
} else {
log.Fatalf("Get operation failed: %v", err)
}
}
var member Member
if err := json.Unmarshal(data, &member); err != nil {
log.Printf("Failed to unmarshal member %s: %v\n", path, err)
}
return member, nil
}
func zkDelete(conn *zk.Conn, zpath string) error {
if zpath != "" {
_, stat, _ := conn.Get(zpath)
err := conn.Delete(zpath, stat.Version)
if err != nil {
return err
}
index := strings.LastIndex(zpath, "/")
if index != -1 {
return zkDelete(conn, zpath[:index])
} else {
return nil
}
} else {
return nil
}
}
func (this *Get) showChildrenRecursively(conn *zk.Conn, path string) {
children, _, err := conn.Children(path)
if err != nil {
return
}
sort.Strings(children)
for _, child := range children {
if path == "/" {
path = ""
}
znode := fmt.Sprintf("%s/%s", path, child)
// display znode content
data, stat, err := conn.Get(znode)
must(err)
if stat.EphemeralOwner > 0 {
if patternMatched(znode, this.likePattern) {
this.Ui.Output(color.Yellow(znode))
}
} else {
if patternMatched(znode, this.likePattern) {
this.Ui.Output(color.Green(znode))
}
}
if len(data) > 0 && patternMatched(znode, this.likePattern) {
if this.verbose {
this.Ui.Output(fmt.Sprintf("%s %#v",
strings.Repeat(" ", 3), stat))
this.Ui.Output(fmt.Sprintf("%s %v",
strings.Repeat(" ", 3), data))
}
this.Ui.Output(fmt.Sprintf("%s %s",
strings.Repeat(" ", 3), string(data)))
}
this.showChildrenRecursively(conn, znode)
}
}
func (this *Dump) dump(conn *zk.Conn, path string) {
children, _, err := conn.Children(path)
if err != nil {
must(err)
return
}
sort.Strings(children)
var buf [4]byte
for _, child := range children {
if path == "/" {
path = ""
}
znode := fmt.Sprintf("%s/%s", path, child)
// display znode content
data, stat, err := conn.Get(znode)
must(err)
if stat.EphemeralOwner > 0 {
// ignore ephemeral znodes
continue
}
_, err = this.f.Write([]byte(znode))
must(err)
_, err = this.f.Write([]byte{'\n'})
must(err)
v := buf[0:4]
binary.BigEndian.PutUint32(v, uint32(len(data)))
_, err = this.f.Write(v)
if len(data) > 0 {
_, err = this.f.Write(data)
must(err)
}
this.dump(conn, znode)
}
}
func registerCometNode(conn *zk.Conn, node, fpath string, retry, ping time.Duration, vnode int) (*CometNodeInfo, error) {
fpath = path.Join(fpath, node)
data, _, err := conn.Get(fpath)
if err != nil {
glog.Errorf("zk.Get(\"%s\") error(%v)", fpath, err)
return nil, err
}
// fetch and parse comet info
w, addrs, err := parseCometNode(string(data))
if err != nil {
glog.Errorf("parseCometNode(\"%s\") error(%v)", string(data), err)
return nil, err
}
info := &CometNodeInfo{Addr: addrs, weight: w}
rpcAddr, ok := addrs[cometProtocolRPC]
if !ok || len(rpcAddr) == 0 {
glog.Errorf("zk nodes: \"%s\" don't have rpc addr", fpath)
return nil, ErrCometRPC
}
// init comet rpc
clients := make(map[string]*RPCClient, len(rpcAddr))
for _, addr := range rpcAddr {
r, err := rpc.Dial("tcp", addr.Addr)
if err != nil {
glog.Errorf("rpc.Dial(\"%s\") error(%v)", addr.Addr, err)
return nil, err
}
addr.Client = r
clients[addr.Addr] = addr
}
lb, err := NewRandLB(clients, cometService, retry, ping, vnode, true)
if err != nil {
glog.Errorf("NewRandLR() error(%v)", err)
panic(err)
}
info.CometRPC = lb
glog.Infof("zk path: \"%s\" register nodes: \"%s\"", fpath, node)
return info, nil
}
func markTerminated(conn *zk.Conn, hss *zzk.HostServiceState) {
ssPath := zzk.ServiceStatePath(hss.ServiceId, hss.ServiceStateId)
_, stats, err := conn.Get(ssPath)
if err != nil {
glog.V(0).Infof("Unable to get service state %s for delete because: %v", ssPath, err)
return
}
err = conn.Delete(ssPath, stats.Version)
if err != nil {
glog.V(0).Infof("Unable to delete service state %s because: %v", ssPath, err)
return
}
hssPath := zzk.HostServiceStatePath(hss.HostId, hss.ServiceStateId)
_, stats, err = conn.Get(hssPath)
if err != nil {
glog.V(0).Infof("Unable to get host service state %s for delete becaus: %v", hssPath, err)
return
}
err = conn.Delete(hssPath, stats.Version)
if err != nil {
glog.V(0).Infof("Unable to delete host service state %s", hssPath)
}
}
// 得到节点的数据
func GetNodeData(conn *zk.Conn, path string) (string, error) {
data, _, err := conn.Get(path)
return string(data), err
}
})
Describe("Set", func() {
Context("when setting a shallow key", func() {
BeforeEach(func() {
nodeArr[0] = StoreNode{
Key: "/foo",
Value: []byte("bar"),
TTL: 0,
}
err := adapter.Set(nodeArr)
Ω(err).ShouldNot(HaveOccured())
})
It("should be able to set the key", func() {
data, stat, err := client.Get("/foo")
Ω(string(data)).Should(Equal("0,bar"))
Ω(stat.NumChildren).Should(BeNumerically("==", 0))
Ω(stat.Version).Should(BeNumerically("==", 0))
Ω(err).ShouldNot(HaveOccured())
acl, _, err := client.GetACL("/foo")
Ω(acl).Should(Equal(zk.WorldACL(zk.PermAll)))
Ω(err).ShouldNot(HaveOccured())
})
Context("setting the key again", func() {
BeforeEach(func() {
nodeArr[0].Value = []byte("baz")
nodeArr[0].TTL = 20
err := adapter.Set(nodeArr)