// Close the release channel when you want to clean up nicely.
func CreatePidNode(zconn zookeeper.Conn, zkPath string, contents string, done chan struct{}) error {
// On the first try, assume the cluster is up and running, that will
// help hunt down any config issues present at startup
if _, err := zconn.Create(zkPath, []byte(contents), zookeeper.FlagEphemeral, zookeeper.WorldACL(PERM_FILE)); err != nil {
if ZkErrorEqual(err, zookeeper.ErrNodeExists) {
err = zconn.Delete(zkPath, -1)
}
if err != nil {
return fmt.Errorf("zkutil: failed deleting pid node: %v: %v", zkPath, err)
}
_, err = zconn.Create(zkPath, []byte(contents), zookeeper.FlagEphemeral, zookeeper.WorldACL(PERM_FILE))
if err != nil {
return fmt.Errorf("zkutil: failed creating pid node: %v: %v", zkPath, err)
}
}
go func() {
for {
_, _, watch, err := zconn.GetW(zkPath)
if err != nil {
if ZkErrorEqual(err, zookeeper.ErrNoNode) {
_, err = zconn.Create(zkPath, []byte(contents), zookeeper.FlagEphemeral, zookeeper.WorldACL(zookeeper.PermAll))
if err != nil {
log.Warningf("failed recreating pid node: %v: %v", zkPath, err)
} else {
log.Infof("recreated pid node: %v", zkPath)
continue
}
} else {
log.Warningf("failed reading pid node: %v", err)
}
} else {
select {
case event := <-watch:
if ZkEventOk(event) && event.Type == zookeeper.EventNodeDeleted {
// Most likely another process has started up. However,
// there is a chance that an ephemeral node is deleted by
// the session expiring, yet that same session gets a watch
// notification. This seems like buggy behavior, but rather
// than race too hard on the node, just wait a bit and see
// if the situation resolves itself.
log.Warningf("pid deleted: %v", zkPath)
} else {
log.Infof("pid node event: %v", event)
}
// break here and wait for a bit before attempting
case <-done:
log.Infof("pid watcher stopped on done: %v", zkPath)
return
}
}
select {
// No one likes a thundering herd, least of all zookeeper.
case <-time.After(5*time.Second + time.Duration(rand.Int63n(55e9))):
case <-done:
log.Infof("pid watcher stopped on done: %v", zkPath)
return
}
}
}()
return nil
}
func (zs *ZKStorage) createZPath(conn *zk.Conn, zpath string) error {
_, err := conn.Create(zpath, nil, 0, zk.WorldACL(zk.PermAll))
if err != nil {
if zk.ErrNodeExists == err {
return nil
} else {
parent, _ := path.Split(zpath)
if len(parent) == 0 {
return errors.New("Specified blank path")
}
err = zs.createZPath(conn, parent[:len(parent)-1])
if err != nil {
return err
}
_, err = conn.Create(zpath, nil, 0, zk.WorldACL(zk.PermAll))
if err == zk.ErrNodeExists {
err = nil
}
}
}
if zk.ErrNodeExists == err {
return nil
} else {
return err
}
}
// RegisterTmp create a ephemeral node, and watch it, if node droped then send a SIGQUIT to self.
func RegisterTemp(conn *zk.Conn, fpath, data string) error {
tpath, err := conn.Create(path.Join(fpath)+"/", []byte(data), zk.FlagEphemeral|zk.FlagSequence, zk.WorldACL(zk.PermAll))
if err != nil {
glog.Errorf("conn.Create(\"%s\", \"%s\", zk.FlagEphemeral|zk.FlagSequence) error(%v)", fpath, data, err)
return err
}
glog.V(1).Infof("create a zookeeper node:%s", tpath)
// watch self
go func() {
for {
glog.Infof("zk path: \"%s\" set a watch", tpath)
exist, _, watch, err := conn.ExistsW(tpath)
if err != nil {
glog.Errorf("zk.ExistsW(\"%s\") error(%v)", tpath, err)
glog.Warningf("zk path: \"%s\" set watch failed, kill itself", tpath)
killSelf()
return
}
if !exist {
glog.Warningf("zk path: \"%s\" not exist, kill itself", tpath)
killSelf()
return
}
event := <-watch
glog.Infof("zk path: \"%s\" receive a event %v", tpath, event)
}
}()
return nil
}
// 创建一个临时节点，并监听自己
func CreateTempW(conn *zk.Conn, fpath, data string, watchFunc func(bool, zk.Event)) error {
tpath, err := conn.Create(path.Join(fpath)+"/", []byte(data), zk.FlagEphemeral|zk.FlagSequence, zk.WorldACL(zk.PermAll))
if err != nil {
glog.Errorf("conn.Create(\"%s\", \"%s\", zk.FlagEphemeral|zk.FlagSequence) error(%v)", fpath, data, err)
return err
}
glog.V(1).Infof("create a zookeeper node:%s", tpath)
// watch self
if watchFunc != nil {
go func() {
for {
glog.Infof("zk path: \"%s\" set a watch", tpath)
exist, _, watch, err := conn.ExistsW(tpath)
if err != nil {
glog.Errorf("zk.ExistsW(%s) error(%v)", tpath, err)
return
}
event := <-watch
watchFunc(exist, event)
glog.Infof("zk path: \"%s\" receive a event %v", tpath, event)
}
}()
}
return nil
}
func createPath(path string, data []byte, client *zk.Conn) error {
exists, _, err := client.Exists(path)
if err != nil {
return err
}
if exists {
return nil
}
name := "/"
p := strings.Split(path, "/")
for _, v := range p[1 : len(p)-1] {
name += v
e, _, _ := client.Exists(name)
if !e {
_, err = client.Create(name, []byte{}, int32(0), zk.WorldACL(zk.PermAll))
if err != nil {
return err
}
}
name += "/"
}
_, err = client.Create(path, data, int32(0), zk.WorldACL(zk.PermAll))
return err
}
/*
Read ZK ACL:
http://zookeeper.apache.org/doc/trunk/zookeeperProgrammers.html#sc_ACLPermissions
*/
func Create(conn *zk.Conn, zkConf conf.Zookeeper, appId string, domainValue string) (string, error) {
path := concatPath(zkConf.Path, appId)
resPath, err := conn.Create(path, []byte(domainValue), 0, defaultACL())
if err != nil {
return "", err
}
return resPath, nil
}
// RegisterTmp create a ephemeral node, and watch it, if node droped then send a SIGQUIT to self.
func RegisterTemp(conn *zk.Conn, fpath string, data []byte) error {
tpath, err := conn.Create(path.Join(fpath), data, zk.FlagEphemeral, zk.WorldACL(zk.PermAll))
if err != nil {
log.Error("conn.Create(\"%s\", \"%s\", zk.FlagEphemeral) error(%v)", fpath, string(data), err)
return err
}
log.Debug("create a zookeeper node:%s", tpath)
return nil
}
// notify every Comet node to migrate
func notifyMigrate(conn *zk.Conn, migrateLockPath, znode, key string, update bool, nodeWeightMap map[string]int) (err error) {
// try lock
if _, err = conn.Create(migrateLockPath, []byte("1"), zk.FlagEphemeral, zk.WorldACL(zk.PermAll)); err != nil {
log.Error("conn.Create(\"/gopush-migrate-lock\", \"1\", zk.FlagEphemeral) error(%v)", err)
return
}
// call comet migrate rpc
wg := &sync.WaitGroup{}
wg.Add(len(cometNodeInfoMap))
for node, nodeInfo := range cometNodeInfoMap {
go func(n string, info *CometNodeInfo) {
if info.Rpc == nil {
log.Error("notify migrate failed, no rpc found, node:%s", n)
wg.Done()
return
}
r := info.Rpc.Get()
if r == nil {
log.Error("notify migrate failed, no rpc found, node:%s", n)
wg.Done()
return
}
reply := 0
args := &CometMigrateArgs{Nodes: nodeWeightMap}
if err = r.Call(CometServiceMigrate, args, &reply); err != nil {
log.Error("rpc.Call(\"%s\") error(%v)", CometServiceMigrate, err)
wg.Done()
return
}
log.Debug("notify node:%s migrate succeed", n)
wg.Done()
}(node, nodeInfo)
}
wg.Wait()
// update znode info
if update {
var data []byte
data, err = json.Marshal(cometNodeInfoMap[key])
if err != nil {
log.Error("json.Marshal() node:%s error(%v)", key, err)
return
}
if _, err = conn.Set(znode, data, -1); err != nil {
log.Error("conn.Set(\"%s\",\"%s\",\"-1\") error(%v)", znode, string(data), err)
return
}
}
// release lock
if err = conn.Delete(migrateLockPath, -1); err != nil {
log.Error("conn.Delete(\"%s\") error(%v)", migrateLockPath, err)
}
return
}
func CreateNode(path string, conn *zk.Conn) error {
var err error
for i := 0; i < 5; i++ {
_, err = conn.Create(path, []byte{}, 0, zk.WorldACL(zk.PermAll))
if err == zk.ErrNodeExists || err == nil {
return nil
}
glog.Warningf("Could not create node:%s, %v", path, err)
}
return err
}
// create the parent znodes up to path. does not error if any of the
// parent znodes exist.
func createParentPath(path string, conn *zk.Conn, acl []zk.ACL) error {
parts := strings.Split(path, "/")
prePath := ""
for _, p := range parts[1 : len(parts)-1] {
prePath += "/" + p
_, err := conn.Create(prePath, []byte{}, 0, acl)
if err != nil && err != zk.ErrNodeExists {
return err
}
}
return nil
}
func CreateOrUpdate(zconn zookeeper.Conn, zkPath, value string, flags int, aclv []zookeeper.ACL, recursive bool) (pathCreated string, err error) {
if recursive {
pathCreated, err = CreateRecursive(zconn, zkPath, value, 0, zookeeper.WorldACL(zookeeper.PermAll))
} else {
pathCreated, err = zconn.Create(zkPath, []byte(value), 0, zookeeper.WorldACL(zookeeper.PermAll))
}
if err != nil && ZkErrorEqual(err, zookeeper.ErrNodeExists) {
pathCreated = ""
_, err = zconn.Set(zkPath, []byte(value), -1)
}
return
}
func bench(conn *zk.Conn, value []byte, ratio float64) {
choice := rand.Float64()
atomic.AddInt32(&reqCnt, 1)
if choice >= ratio {
// write
conn.Delete("/_test_load_obliterator", 0)
conn.Create("/_test_load_obliterator", value, 1, zk.WorldACL(zk.PermAll))
} else {
// read
conn.Get("/_test_load_obliterator")
}
}
func ZkCreateTempNode(zkConn *zk.Conn, path string) {
flags := int32(zk.FlagEphemeral)
acl := zk.WorldACL(zk.PermAll)
fmt.Printf("%s before create \n", path)
p, err := zkConn.Create(path, []byte(""), flags, acl)
if err != nil {
panic(err.Error())
}
fmt.Printf("%s after create \n", p)
}
func zk_write(k string, v string, c *zk.Conn) {
e, stat, err := c.Exists(k)
check(err)
if e {
stat, err = c.Set(k, []byte(v), stat.Version)
check(err)
} else {
string, err := c.Create(k, []byte(v), int32(0), zk.WorldACL(zk.PermAll))
if string == "" {
check(err)
}
}
}
func ensurePathExists(conn *zk.Conn, path string) error {
pathExists, _, _ := conn.Exists(path)
if pathExists {
return nil
}
_, err := conn.Create(path, []byte{}, 0, defaultACL())
if err != nil {
return err
}
return nil
}
func zk_reginster(path string, c *zk.Conn) (string, error) {
mgo_path := path + "/" + *server_addr
//tPath, err := c.Create(mgo_path, []byte{}, 0, zk.WorldACL(zk.PermAll))
tPath, err := c.Create(path, []byte{}, 0, zk.WorldACL(zk.PermAll))
log.Infof("zk_reginster | path :%+v", mgo_path)
tPath, err = c.Create(mgo_path, []byte{}, zk.FlagEphemeral|zk.FlagSequence, zk.WorldACL(zk.PermAll))
if err != nil {
log.Warnf("zk_reginster | Create returned: %+v", err)
}
log.Infof("zk_reginster | create :%+v", tPath)
return tPath, err
}
// zkData create zookeeper path, if path exists ignore error, and set node data.
func zkData(conn *zk.Conn, redisMaster string) error {
node := path.Join(conf.ZKPath, conf.Node)
tpath := ""
for _, str := range strings.Split(conf.ZKPath, "/")[1:] {
tpath = path.Join(tpath, "/", str)
log.Info("create zookeeper path: \"%s\"", tpath)
_, err := conn.Create(tpath, []byte(""), 0, zk.WorldACL(zk.PermAll))
if err != nil {
if err == zk.ErrNodeExists {
log.Warn("zk.create(\"%s\") exists", tpath)
} else {
log.Error("zk.create(\"%s\") error(%v)", tpath, err)
return err
}
}
}
if _, err := conn.Create(node, []byte{}, 0, zk.WorldACL(zk.PermAll)); err != nil {
if err == zk.ErrNodeExists {
oData, stat, err := conn.Get(node)
if err != nil {
log.Error("zk.Get(\"%s\") error(%v)", node, err)
return err
}
ni := unmarshal(oData)
if ni != nil || len(ni.Servers) == 0 {
log.Warn("node have not data")
return nil
}
data := marshal(ni, redisMaster)
if len(data) == 0 {
log.Warn("marshal error")
return nil
}
if bytes.Equal(oData, data) {
log.Warn("zk data same, no change")
return nil
}
if _, err = conn.Set(node, data, stat.Version); err != nil {
log.Error("zk.Set(\"%s\", data, 0) error(%v)", node, err)
return err
}
log.Info("zk update data: \"%s\"", node)
} else {
log.Error("zk.create(\"%s\") error(%v)", tpath, err)
return err
}
}
return nil
}
func CreateRecursive(conn *zk.Conn, zkPath, value string, flags int, acls []zk.ACL) (createdPath string, err error) {
createdPath, err = conn.Create(zkPath, []byte(value), int32(flags), acls)
if ErrorEqual(err, zk.ErrNoNode) {
dirAcls := make([]zk.ACL, len(acls))
for i, acl := range acls {
dirAcls[i] = acl
dirAcls[i].Perms = PERM_DIRECTORY
}
_, err = CreateRecursive(conn, path.Dir(zkPath), "", flags, dirAcls)
if err != nil && !ErrorEqual(err, zk.ErrNodeExists) {
return "", err
}
createdPath, err = conn.Create(zkPath, []byte(value), int32(flags), acls)
}
return
}
func CreateRecursive(zconn *zookeeper.Conn, zkPath, value string, flags int32, aclv []zookeeper.ACL) (pathCreated string, err error) {
pathCreated, err = zconn.Create(zkPath, []byte(value), flags, aclv)
if err == zookeeper.ErrNoNode {
dirAclv := make([]zookeeper.ACL, len(aclv))
for i, acl := range aclv {
dirAclv[i] = acl
dirAclv[i].Perms = PERM_DIRECTORY
}
_, err = CreateRecursive(zconn, path.Dir(zkPath), "", flags, dirAclv)
if err != nil && err != zookeeper.ErrNodeExists {
return "", err
}
pathCreated, err = zconn.Create(zkPath, []byte(value), flags, aclv)
}
return
}
func Create(conn *zk.Conn, fpath string) error {
tpath := ""
for _, str := range strings.Split(fpath, "/")[1:] {
tpath = path.Join(tpath, "/", str)
glog.V(1).Infof("create zookeeper path: \"%s\"", tpath)
_, err := conn.Create(tpath, []byte(""), 0, zk.WorldACL(zk.PermAll))
if err != nil {
if err == zk.ErrNodeExists {
glog.Warningf("zk.create(\"%s\") exists", tpath)
} else {
glog.Errorf("zk.create(\"%s\") error(%v)", tpath, err)
return err
}
}
}
return nil
}
// createInternalWithACL: create a new path with acl
func createInternalWithACL(connection *zk.Conn, path string, data []byte, force bool, perms []zk.ACL) (string, error) {
if path == "/" {
return "/", nil
}
log.Debugf("creating: %s with acl ", path)
attempts := 0
for {
attempts += 1
returnValue, err := connection.Create(path, data, flags, perms)
log.Debugf("create status for %s: %s, %+v", path, returnValue, err)
if err != nil && force && attempts < 2 {
returnValue, err = createInternalWithACL(connection, gopath.Dir(path), []byte("zookeepercli auto-generated"), force, perms)
} else {
return returnValue, err
}
}
return "", nil
}
func ZkCreateRoot(zkConn *zk.Conn, root string) {
flags := int32(0)
acl := zk.WorldACL(zk.PermAll)
found, _, _, err := zkConn.ExistsW(root)
if err != nil {
panic(err.Error())
}
if !found {
fmt.Printf("create %s\n", root)
path, err := zkConn.Create(root, []byte(""), flags, acl)
if err != nil {
panic(err.Error())
}
fmt.Printf("created %s\n", path)
}
}
// NewElection initializes a new instance of a Election that can later be used to request
// leadership for a specific resource.
//
// It accepts:
// zkConn - a connection to a running Zookeeper instance
// resource - the resource for which leadership is being requested
//
// It will return either a non-nil Election instance and a nil error, or a nil
// Election and a non-nil error.
//
func NewElection(zkConn *zk.Conn, electionNode string) (Election, error) {
//TODO: what should flags and acl be set to?
flags := int32(0)
acl := zk.WorldACL(zk.PermAll)
exists, _, _ := zkConn.Exists(electionNode)
var (
path string
err error
)
if !exists {
path, err = zkConn.Create(electionNode, []byte("data"), flags, acl)
must(err)
fmt.Printf("created: %+v\n", path)
}
return Election{electionNode, Candidate{}, false, zkConn, nil}, nil
}
func checkPath(conn *zk.Conn, path string) {
exists, _, err := conn.Exists(path)
if exists {
if path == "/" {
path = os.Getenv("ZK_BASE")
if path == "" {
path = zkBase
}
_, err := conn.Create(path, []byte{}, flags, acl)
check(err)
}
return
}
checkPath(conn, filepath.Dir(path))
_, err = conn.Create(path, []byte{}, flags, acl)
check(err)
return
}
func mkdirs(conn *zk.Conn, path string) (err error) {
if path == "" {
return errors.New("path should not been empty")
}
if path == "/" {
return nil
}
if path[0] != '/' {
return errors.New("path must start with /")
}
//check whether this path exists
exist, _, err := conn.Exists(path)
if exist {
return nil
}
flags := int32(0)
acl := zk.WorldACL(zk.PermAll)
_, err = conn.Create(path, []byte(""), flags, acl)
if err == nil { //created successfully
return
}
//create parent
paths := strings.Split(path[1:], "/")
createdPath := ""
for _, p := range paths {
createdPath = createdPath + "/" + p
exist, _, _ = conn.Exists(createdPath)
if !exist {
_, err = conn.Create(createdPath, []byte(""), flags, acl)
if err != nil {
return
}
}
}
return nil
}
// Create a path and any pieces required, think mkdir -p.
// Intermediate znodes are always created empty.
func CreateRecursive(zconn zookeeper.Conn, zkPath, value string, flags int, aclv []zookeeper.ACL) (pathCreated string, err error) {
parts := strings.Split(zkPath, "/")
if parts[1] != MagicPrefix {
return "", fmt.Errorf("zkutil: non /%v path: %v", MagicPrefix, zkPath)
}
pathCreated, err = zconn.Create(zkPath, []byte(value), int32(flags), aclv)
if ZkErrorEqual(err, zookeeper.ErrNoNode) {
// Make sure that nodes are either "file" or "directory" to mirror file system
// semantics.
dirAclv := make([]zookeeper.ACL, len(aclv))
for i, acl := range aclv {
dirAclv[i] = acl
dirAclv[i].Perms = PERM_DIRECTORY
}
_, err = CreateRecursive(zconn, path.Dir(zkPath), "", flags, dirAclv)
if err != nil && !ZkErrorEqual(err, zookeeper.ErrNodeExists) {
return "", err
}
pathCreated, err = zconn.Create(zkPath, []byte(value), int32(flags), aclv)
}
return
}
// Tries to create the configuration path, if it doesn't exist
// It tries multiple times in case there's a race with another quotaservice node coming up
func createPath(conn *zk.Conn, path string) (err error) {
for i := 0; i < createRetries; i++ {
exists, _, err := conn.Exists(path)
if exists && err == nil {
return nil
}
_, err = conn.Create(path, []byte{}, 0, zk.WorldACL(zk.PermAll))
if err == nil {
return nil
}
logging.Printf("Could not create zk path, sleeping for 100ms")
time.Sleep(100 * time.Millisecond)
}
if err == nil {
err = errors.New("could not create and get path " + path)
}
return err
}
func createRecursively(zkConn *zk.Conn, path string, data []byte, flags int32, acl []zk.ACL) error {
pieces := strings.Split(path, "/")
acc := ""
for _, piece := range pieces {
fullpath := acc + "/" + piece
// skip trying to create the root node.
if fullpath == "/" {
continue
}
// just try to create it
_, err := zkConn.Create(fullpath, data, flags, acl)
// if its *NOT* an existing node, then return the err.
if err != nil && err != zk.ErrNodeExists {
return err
}
// continue on.
acc = acc + "/" + piece
}
return nil
}
func zkCreateNodes(path string, nodes []string, c *zk.Conn) error {
if len(nodes) > 0 {
// strings.Split will return empty-strings for leading split chars, lets skip over these.
if len(nodes[0]) == 0 {
return zkCreateNodes(path, nodes[1:], c)
}
fqPath := path + "/" + nodes[0]
log.Printf("Creating path: %v", fqPath)
exists, _, err := c.Exists(fqPath)
if err != nil {
return err
}
if !exists {
_, err := c.Create(fqPath, []byte{}, 0, zk.WorldACL(zk.PermAll))
if err != nil {
return err
}
}
return zkCreateNodes(fqPath, nodes[1:], c)
}
return nil
}
Ω(string(node.Value)).Should(Equal("waffle"))
Ω(int(node.TTL)).Should(Equal(0))
})
})
Context("when the node does not exist", func() {
It("returns the StoreErrorKeyNotFound error", func() {
node, err := adapter.Get("/no/node/for/you")
Ω(err).Should(Equal(ErrorKeyNotFound))
Ω(node).Should(BeZero())
})
})
Context("when the node has an invalid format", func() {
BeforeEach(func() {
client.Create("/missingTTL", []byte("waffle"), 0, zk.WorldACL(zk.PermAll))
client.Create("/invalidTTL", []byte("a,waffle"), 0, zk.WorldACL(zk.PermAll))
})
It("returns the StoreErrorInvalidFormat error", func() {
node, err := adapter.Get("/missingTTL")
Ω(err).Should(Equal(ErrorInvalidFormat), "Expected the error to be an IsInvalidFormatError error")
Ω(node).Should(BeZero())
node, err = adapter.Get("/invalidTTL")
Ω(err).Should(Equal(ErrorInvalidFormat), "Expected the error to be an IsInvalidFormatError error")
Ω(node).Should(BeZero())
})
})
})