func recursiveDelete(c *zk.Conn, path string) error {
children, _, err := c.Children(path)
if err != nil && err != zk.ErrNoNode {
logError("err finding children of %s", path)
return err
}
for _, child := range children {
err := recursiveDelete(c, path+"/"+child)
if err != nil && err != zk.ErrNoNode {
logError("err deleting %s", child)
return err
}
}
// get version
_, stat, err := c.Get(path)
if err != nil && err != zk.ErrNoNode {
logError("err getting version of %s", path)
return err
}
if err := c.Delete(path, stat.Version); err != nil && err != zk.ErrNoNode {
return err
}
return nil
}
func All(conn *zk.Conn, zkConf conf.Zookeeper) (map[string]Service, error) {
err := ensurePathExists(conn, zkConf.Path)
if err != nil {
return nil, err
}
services := map[string]Service{}
keys, _, err2 := conn.Children(zkConf.Path)
if err2 != nil {
return nil, err2
}
for _, childPath := range keys {
bite, _, e := conn.Get(zkConf.Path + "/" + childPath)
if e != nil {
return nil, e
break
}
appId, _ := unescapeSlashes(childPath)
services[appId] = Service{Id: appId, Acl: string(bite)}
}
return services, nil
}
func DeleteNodebyData(path string, conn *zk.Conn, data []byte) error {
children, _, err := conn.Children(path)
if err != nil {
glog.Warning("Could not list children")
return err
}
for _, child := range children {
child_path := path + "/" + child
child_data, _, err := conn.Get(child_path)
if err != nil {
glog.Warning("Could not get data for %s", child_path)
continue
}
if bytes.Compare(data, child_data) == 0 {
for i := 0; i < 5; i++ {
_, stats, _ := conn.Get(child_path)
err = conn.Delete(child_path, stats.Version)
if err == nil || err == zk.ErrNoNode {
return nil
}
}
glog.Error("Could not delete matched node %s", child_path)
}
}
return nil
}
func DeleteRecursive(zconn zookeeper.Conn, zkPath string, version int) error {
// version: -1 delete any version of the node at path - only applies to the top node
err := zconn.Delete(zkPath, int32(version))
if err == nil {
return nil
}
if !ZkErrorEqual(err, zookeeper.ErrNotEmpty) {
return err
}
// Remove the ability for other nodes to get created while we are trying to delete.
// Otherwise, you can enter a race condition, or get starved out from deleting.
_, err = zconn.SetACL(zkPath, zookeeper.WorldACL(zookeeper.PermAdmin|zookeeper.PermDelete|zookeeper.PermRead), int32(version))
if err != nil {
return err
}
children, _, err := zconn.Children(zkPath)
if err != nil {
return err
}
for _, child := range children {
err := DeleteRecursive(zconn, path.Join(zkPath, child), -1)
if err != nil && !ZkErrorEqual(err, zookeeper.ErrNoNode) {
return fmt.Errorf("zkutil: recursive delete failed: %v", err)
}
}
err = zconn.Delete(zkPath, int32(version))
if err != nil && !ZkErrorEqual(err, zookeeper.ErrNotEmpty) {
err = fmt.Errorf("zkutil: nodes getting recreated underneath delete (app race condition): %v", zkPath)
}
return err
}
func mirrorNode(conn *zk.Conn, node *NodeTree) error {
var err error
var children []string
nodePath := appendPath(node.PathPrefix, node.Name)
if children, _, err = conn.Children(nodePath); err != nil {
return err
}
for _, child := range children {
var data []byte
childPath := appendPath(nodePath, child)
if data, _, err = conn.Get(childPath); err != nil {
return err
}
childNode := NewNodeTree(child, nodePath, data)
node.AddChild(childNode)
if err := mirrorNode(conn, childNode); err != nil {
return err
}
}
return nil
}
// The lexically lowest node is the lock holder - verify that this
// path holds the lock. Call this queue-lock because the semantics are
// a hybrid. Normal zookeeper locks make assumptions about sequential
// numbering that don't hold when the data in a lock is modified.
// if the provided 'interrupted' chan is closed, we'll just stop waiting
// and return an interruption error
func ObtainQueueLock(zconn zookeeper.Conn, zkPath string, wait time.Duration, interrupted chan struct{}) error {
queueNode := path.Dir(zkPath)
lockNode := path.Base(zkPath)
timer := time.NewTimer(wait)
trylock:
children, _, err := zconn.Children(queueNode)
if err != nil {
return fmt.Errorf("zkutil: trylock failed %v", err)
}
sort.Strings(children)
if len(children) > 0 {
if children[0] == lockNode {
return nil
}
if wait > 0 {
prevLock := ""
for i := 1; i < len(children); i++ {
if children[i] == lockNode {
prevLock = children[i-1]
break
}
}
if prevLock == "" {
return fmt.Errorf("zkutil: no previous queue node found: %v", zkPath)
}
zkPrevLock := path.Join(queueNode, prevLock)
_, stat, watch, err := zconn.ExistsW(zkPrevLock)
if err != nil {
return fmt.Errorf("zkutil: unable to watch queued node %v %v", zkPrevLock, err)
}
if stat == nil {
goto trylock
}
select {
case <-timer.C:
break
case <-interrupted:
return ErrInterrupted
case <-watch:
// The precise event doesn't matter - try to read again regardless.
goto trylock
}
}
return ErrTimeout
}
return fmt.Errorf("zkutil: empty queue node: %v", queueNode)
}
func walkChildrenRecursively(conn *zk.Conn, path string, walkfunc func(string)) error {
children, _, err := conn.Children(path)
if err != nil {
return err
}
sort.Strings(children)
for _, child := range children {
if path == "/" {
walkfunc(path + child)
err = walkChildrenRecursively(conn, path+child, walkfunc)
}
walkfunc(path + "/" + child)
err = walkChildrenRecursively(conn, path+"/"+child, walkfunc)
}
return err
}
// GetNodes get all child from zk path.
func GetNodes(conn *zk.Conn, path string) ([]string, error) {
nodes, stat, err := conn.Children(path)
if err != nil {
if err == zk.ErrNoNode {
return nil, ErrNodeNotExist
}
glog.Errorf("zk.Children(\"%s\") error(%v)", path, err)
return nil, err
}
if stat == nil {
return nil, ErrNodeNotExist
}
if len(nodes) == 0 {
return nil, ErrNoChild
}
return nodes, nil
}
func childrenRecursive(zconn *zookeeper.Conn, path string, incrementalPath string) ([]string, error) {
children, _, err := zconn.Children(path)
if err != nil {
return children, err
}
sort.Sort(sort.StringSlice(children))
recursiveChildren := []string{}
for _, child := range children {
incrementalChild := gopath.Join(incrementalPath, child)
recursiveChildren = append(recursiveChildren, incrementalChild)
incrementalChildren, err := childrenRecursive(zconn, gopath.Join(path, child), incrementalChild)
if err != nil {
return children, err
}
recursiveChildren = append(recursiveChildren, incrementalChildren...)
}
return recursiveChildren, err
}
func (z *Zookeeper) znodeTree(conn *zk.Conn, keyPath string, nextKeyPath string) ([]string, error) {
children, _, err := conn.Children(keyPath)
if err != nil {
return children, err
}
sort.Sort(sort.StringSlice(children))
nested := []string{}
for _, child := range children {
nextChild := path.Join(nextKeyPath, child)
nested = append(nested, nextChild)
nestedChildren, err := z.znodeTree(conn, path.Join(keyPath, child), nextChild)
if err != nil {
return children, err
}
nested = append(nested, nestedChildren...)
}
return nested, err
}
func ZkGetChildren(zkConn *zk.Conn, root string) []string {
children, stat, err := zkConn.Children(root)
if err != nil {
if err == zk.ErrNoNode {
return nil
}
fmt.Printf("zk.ChildrenW(\"%s\") error(%v)", root, err)
return nil
}
if stat == nil {
return nil
}
if len(children) == 0 {
return nil
}
return children
}
func ChildrenRecursive(zconn zookeeper.Conn, zkPath string) ([]string, error) {
var err error
mutex := sync.Mutex{}
wg := sync.WaitGroup{}
pathList := make([]string, 0, 32)
children, _, err := zconn.Children(zkPath)
if err != nil {
return nil, err
}
for _, child := range children {
wg.Add(1)
go func(child string) {
childPath := path.Join(zkPath, child)
rChildren, zkErr := ChildrenRecursive(zconn, childPath)
if zkErr != nil {
// If other processes are deleting nodes, we need to ignore
// the missing nodes.
if !ZkErrorEqual(zkErr, zookeeper.ErrNoNode) {
mutex.Lock()
err = zkErr
mutex.Unlock()
}
} else {
mutex.Lock()
pathList = append(pathList, child)
for _, rChild := range rChildren {
pathList = append(pathList, path.Join(child, rChild))
}
mutex.Unlock()
}
wg.Done()
}(child)
}
wg.Wait()
mutex.Lock()
defer mutex.Unlock()
if err != nil {
return nil, err
}
return pathList, nil
}
// childrenRecursiveInternal: internal implementation of recursive-children query.
func childrenRecursiveInternal(connection *zk.Conn, path string, incrementalPath string) ([]string, error) {
children, _, err := connection.Children(path)
if err != nil {
return children, err
}
sort.Sort(sort.StringSlice(children))
recursiveChildren := []string{}
for _, child := range children {
incrementalChild := gopath.Join(incrementalPath, child)
recursiveChildren = append(recursiveChildren, incrementalChild)
log.Debugf("incremental child: %+v", incrementalChild)
incrementalChildren, err := childrenRecursiveInternal(connection, gopath.Join(path, child), incrementalChild)
if err != nil {
return children, err
}
recursiveChildren = append(recursiveChildren, incrementalChildren...)
}
return recursiveChildren, err
}
func zkDeleteChildren(conn *zk.Conn, path string) {
children, _, _ := conn.Children(path)
// Leaf
if len(children) == 0 {
fmt.Println("Deleting ", path)
err := conn.Delete(path, -1)
if err != nil {
log.Panic(err)
}
return
}
// Branches
for _, name := range children {
zkDeleteChildren(conn, path+"/"+name)
}
return
}
func (this *Get) showChildrenRecursively(conn *zk.Conn, path string) {
children, _, err := conn.Children(path)
if err != nil {
return
}
sort.Strings(children)
for _, child := range children {
if path == "/" {
path = ""
}
znode := fmt.Sprintf("%s/%s", path, child)
// display znode content
data, stat, err := conn.Get(znode)
must(err)
if stat.EphemeralOwner > 0 {
if patternMatched(znode, this.likePattern) {
this.Ui.Output(color.Yellow(znode))
}
} else {
if patternMatched(znode, this.likePattern) {
this.Ui.Output(color.Green(znode))
}
}
if len(data) > 0 && patternMatched(znode, this.likePattern) {
if this.verbose {
this.Ui.Output(fmt.Sprintf("%s %#v",
strings.Repeat(" ", 3), stat))
this.Ui.Output(fmt.Sprintf("%s %v",
strings.Repeat(" ", 3), data))
}
this.Ui.Output(fmt.Sprintf("%s %s",
strings.Repeat(" ", 3), string(data)))
}
this.showChildrenRecursively(conn, znode)
}
}
func (this *Ls) showChildrenRecursively(conn *zk.Conn, path string) {
children, _, err := conn.Children(path)
if err != nil {
return
}
sort.Strings(children)
for _, child := range children {
if path == "/" {
path = ""
}
znode := path + "/" + child
if patternMatched(znode, this.likePattern) {
this.Ui.Output(znode)
}
this.showChildrenRecursively(conn, znode)
}
}
func (this *Dump) dump(conn *zk.Conn, path string) {
children, _, err := conn.Children(path)
if err != nil {
must(err)
return
}
sort.Strings(children)
var buf [4]byte
for _, child := range children {
if path == "/" {
path = ""
}
znode := fmt.Sprintf("%s/%s", path, child)
// display znode content
data, stat, err := conn.Get(znode)
must(err)
if stat.EphemeralOwner > 0 {
// ignore ephemeral znodes
continue
}
_, err = this.f.Write([]byte(znode))
must(err)
_, err = this.f.Write([]byte{'\n'})
must(err)
v := buf[0:4]
binary.BigEndian.PutUint32(v, uint32(len(data)))
_, err = this.f.Write(v)
if len(data) > 0 {
_, err = this.f.Write(data)
must(err)
}
this.dump(conn, znode)
}
}
func (zk *ZookeeperClusterRunner) deleteRecursively(client *zkClient.Conn, key string) {
if key != "/zookeeper" {
children, _, err := client.Children(key)
Ω(err).ShouldNot(HaveOccured(), "Failed to fetch children for '%s'", key)
for _, child := range children {
childPath := key + "/" + child
if key == "/" {
childPath = key + child
}
zk.deleteRecursively(client, childPath)
}
if key != "/" {
err = client.Delete(key, -1)
}
Ω(err).ShouldNot(HaveOccured(), "Failed to delete key '%s'", key)
}
return
}
func pollZooKeeper(conn *zk.Conn, path string, evts chan zk.Event, quit chan bool) {
children, _, err := conn.Children(path)
if err != nil {
fmt.Printf("%+v", err)
panic(err)
}
watcherControl := make([]chan<- bool, len(children)+2)
watcherControl[0] = sinkSelfEvents(conn, path, evts)
watcherControl[1] = sinkChildEvents(conn, path, evts)
for i, child := range children {
p := path + "/" + child
watcherControl[i+2] = sinkSelfEvents(conn, p, evts)
}
<-quit
for _, ch := range watcherControl {
ch <- true
}
}
func resolveRecursive(zconn zookeeper.Conn, parts []string, toplevel bool) ([]string, error) {
for i, part := range parts {
if HasWildcard(part) {
var children []string
zkParentPath := strings.Join(parts[:i], "/")
var err error
children, _, err = zconn.Children(zkParentPath)
if err != nil {
// we asked for something like
// /zk/cell/aaa/* and
// /zk/cell/aaa doesn't exist
// -> return empty list, no error
// (note we check both a regular zk
// error and the error the test
// produces)
if ZkErrorEqual(err, zookeeper.ErrNoNode) {
return nil, nil
}
// otherwise we return the error
return nil, err
}
sort.Strings(children)
results := make([][]string, len(children))
wg := &sync.WaitGroup{}
mu := &sync.Mutex{}
var firstError error
for j, child := range children {
matched, err := path.Match(part, child)
if err != nil {
return nil, err
}
if matched {
// we have a match!
wg.Add(1)
newParts := make([]string, len(parts))
copy(newParts, parts)
newParts[i] = child
go func(j int) {
defer wg.Done()
subResult, err := resolveRecursive(zconn, newParts, false)
if err != nil {
mu.Lock()
if firstError != nil {
log.Infof("Multiple error: %v", err)
} else {
firstError = err
}
mu.Unlock()
} else {
results[j] = subResult
}
}(j)
}
}
wg.Wait()
if firstError != nil {
return nil, firstError
}
result := make([]string, 0, 32)
for j := 0; j < len(children); j++ {
subResult := results[j]
if subResult != nil {
result = append(result, subResult...)
}
}
// we found a part that is a wildcard, we
// added the children already, we're done
return result, nil
}
}
// no part contains a wildcard, add the path if it exists, and done
path := strings.Join(parts, "/")
if toplevel {
// for whatever the user typed at the toplevel, we don't
// check it exists or not, we just return it
return []string{path}, nil
}
// this is an expanded path, we need to check if it exists
_, stat, err := zconn.Exists(path)
if err != nil {
return nil, err
}
if stat != nil {
return []string{path}, nil
}
return nil, nil
}
func GetServerList(conn *zk.Conn) (list []string, err error) {
list, _, err = conn.Children("/go_servers")
return
}
})
})
Context("setting a sibling key", func() {
BeforeEach(func() {
nodeArr[0] = StoreNode{
Key: "/restaurant/menu/lunch",
Value: []byte("fried chicken"),
TTL: 0,
}
err := adapter.Set(nodeArr)
Ω(err).ShouldNot(HaveOccured())
})
It("should be able to overwrite the key", func() {
kiddos, _, err := client.Children("/restaurant/menu")
Ω(kiddos).Should(HaveLen(2))
Ω(kiddos).Should(ContainElement("breakfast"))
Ω(kiddos).Should(ContainElement("lunch"))
Ω(err).ShouldNot(HaveOccured())
})
})
})
Context("when the store is down", func() {
BeforeEach(func() {
log.SetOutput(ioutil.Discard)
zookeeperRunner.Stop()
})
AfterEach(func() {
// GetConsumsers 获取consumer的列表
func GetConsumsers(zknode string, zkc *zk.Conn) ([]string, error) {
consumers, _, err := zkc.Children(zknode + "/consumers")
return consumers, err
}
// GetTopicsWithConsumser 获取topic对应的consumer
func GetTopicsWithConsumser(zknode string, zkc *zk.Conn, consumer string) ([]string, error) {
getzknode := zknode + "/consumers/" + consumer + "/offsets"
topics, _, err := zkc.Children(getzknode)
return topics, err
}