func registerConfigNode() error {
zkPath := fmt.Sprintf("/zk/codis/db_%s/living-codis-config", productName)
hostname, err := os.Hostname()
if err != nil {
return errors.Trace(err)
}
pid := os.Getpid()
content := fmt.Sprintf(`{"hostname": "%v", "pid": %v}`, hostname, pid)
nodeName := fmt.Sprintf("%v-%v", hostname, pid)
zkhelper.CreateRecursive(zkConn, zkPath, "", 0, zkhelper.DefaultDirACLs())
pathCreated, err := zkConn.Create(path.Join(zkPath, nodeName), []byte(content),
zk.FlagEphemeral, zkhelper.DefaultDirACLs())
log.Info("living node created:", pathCreated)
if err != nil {
return errors.Trace(err)
}
livingNode = pathCreated
return nil
}
func CreateProxyInfo(zkConn zkhelper.Conn, productName string, pi *ProxyInfo) (string, error) {
data, err := json.Marshal(pi)
if err != nil {
return "", errors.Trace(err)
}
zkhelper.CreateRecursive(zkConn, GetProxyPath(productName), string(data),
0, zkhelper.DefaultDirACLs())
return zkConn.Create(path.Join(GetProxyPath(productName), pi.Id), data,
zk.FlagEphemeral, zkhelper.DefaultDirACLs())
}
func CreateProxyInfo(coordConn zkhelper.Conn, productName string, pi *ProxyInfo) (string, error) {
data, err := json.Marshal(pi)
if err != nil {
return "", errors.Trace(err)
}
dir := GetProxyPath(productName)
zkhelper.CreateRecursive(coordConn, dir, "", 0, zkhelper.DefaultDirACLs())
return coordConn.Create(path.Join(dir, pi.ID), data, zk.FlagEphemeral, zkhelper.DefaultFileACLs())
}
func addAgent(a *agentInfo) error {
basePath := agentPath()
zkhelper.CreateRecursive(globalConn, basePath, "", 0, zkhelper.DefaultDirACLs())
contents, err := json.Marshal(a)
if err != nil {
return errors.Trace(err)
}
_, err = globalConn.Create(path.Join(basePath, a.ID), contents, zk.FlagEphemeral, zkhelper.DefaultFileACLs())
return errors.Trace(err)
}
func CreateActionRootPath(zkConn zkhelper.Conn, path string) error {
// if action dir not exists, create it first
exists, err := zkhelper.NodeExists(zkConn, path)
if err != nil {
return errors.Trace(err)
}
if !exists {
_, err := zkhelper.CreateOrUpdate(zkConn, path, "", 0, zkhelper.DefaultDirACLs(), true)
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func NewAction(zkConn zkhelper.Conn, productName string, actionType ActionType, target interface{}, desc string, needConfirm bool) error {
ts := strconv.FormatInt(time.Now().Unix(), 10)
action := &Action{
Type: actionType,
Desc: desc,
Target: target,
Ts: ts,
}
// set action receivers
proxies, err := ProxyList(zkConn, productName, func(p *ProxyInfo) bool {
return p.State == PROXY_STATE_ONLINE
})
if err != nil {
return errors.Trace(err)
}
for _, p := range proxies {
action.Receivers = append(action.Receivers, p.Id)
}
b, _ := json.Marshal(action)
prefix := GetWatchActionPath(productName)
err = CreateActionRootPath(zkConn, prefix)
if err != nil {
return errors.Trace(err)
}
// create action node
actionCreated, err := zkConn.Create(prefix+"/action_", b, int32(zk.FlagSequence), zkhelper.DefaultDirACLs())
if err != nil {
log.Error(err, prefix)
return errors.Trace(err)
}
if needConfirm {
if err := WaitForReceiver(zkConn, productName, actionCreated, proxies); err != nil {
return errors.Trace(err)
}
}
return nil
}
func (self *ServerGroup) Create(zkConn zkhelper.Conn) error {
if self.Id < 0 {
return errors.NotSupportedf("invalid server group id %d", self.Id)
}
zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
_, err := zkhelper.CreateOrUpdate(zkConn, zkPath, "", 0, zkhelper.DefaultDirACLs(), true)
if err != nil {
return errors.Trace(err)
}
err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
if err != nil {
return errors.Trace(err)
}
// set no server slots' group id to this server group, no need to return error
slots, err := NoGroupSlots(zkConn, self.ProductName)
if err == nil && len(slots) > 0 {
SetSlots(zkConn, self.ProductName, slots, self.Id, SLOT_STATUS_ONLINE)
}
return nil
}
func createDashboardNode() error {
conn := CreateZkConn()
defer conn.Close()
// make sure root dir is exists
rootDir := fmt.Sprintf("/zk/codis/db_%s", globalEnv.ProductName())
zkhelper.CreateRecursive(conn, rootDir, "", 0, zkhelper.DefaultDirACLs())
zkPath := fmt.Sprintf("%s/dashboard", rootDir)
// make sure we're the only one dashboard
if exists, _, _ := conn.Exists(zkPath); exists {
data, _, _ := conn.Get(zkPath)
return errors.New("dashboard already exists: " + string(data))
}
content := fmt.Sprintf(`{"addr": "%v", "pid": %v}`, globalEnv.DashboardAddr(), os.Getpid())
pathCreated, err := conn.Create(zkPath, []byte(content),
zk.FlagEphemeral, zkhelper.DefaultFileACLs())
log.Info("dashboard node created:", pathCreated, string(content))
return errors.Trace(err)
}
func createDashboardNode(conn zkhelper.Conn) error {
// make sure root dir is exists
rootDir := fmt.Sprintf("/zk/reborn/db_%s", globalEnv.ProductName())
zkhelper.CreateRecursive(conn, rootDir, "", 0, zkhelper.DefaultDirACLs())
coordPath := fmt.Sprintf("%s/dashboard", rootDir)
// make sure we're the only one dashboard
timeoutCh := time.After(60 * time.Second)
for {
if exists, _, ch, _ := conn.ExistsW(coordPath); exists {
data, _, _ := conn.Get(coordPath)
if checkDashboardAlive(data) {
return errors.Errorf("dashboard already exists: %s", string(data))
} else {
log.Warningf("dashboard %s exists in zk, wait it removed", data)
select {
case <-ch:
case <-timeoutCh:
return errors.Errorf("wait existed dashboard %s removed timeout", string(data))
}
}
} else {
break
}
}
content := fmt.Sprintf(`{"addr": "%v", "pid": %v}`, globalEnv.DashboardAddr(), os.Getpid())
pathCreated, err := conn.Create(coordPath, []byte(content),
zk.FlagEphemeral, zkhelper.DefaultFileACLs())
log.Infof("dashboard node %s created, data %s, err %v", pathCreated, string(content), err)
return errors.Trace(err)
}
func (m *MigrateManager) createNode() error {
zkhelper.CreateRecursive(m.zkConn, fmt.Sprintf("/zk/codis/db_%s/migrate_tasks", m.productName), "", 0, zkhelper.DefaultDirACLs())
_, err := m.zkConn.Create(getManagerPath(m.productName),
[]byte(""), zk.FlagEphemeral, zkhelper.DefaultFileACLs())
if err != nil {
log.Error("dashboard already exists! err: ", err)
}
return nil
}
func NewActionWithTimeout(zkConn zkhelper.Conn, productName string, actionType ActionType, target interface{}, desc string, needConfirm bool, timeoutInMs int) error {
ts := strconv.FormatInt(time.Now().Unix(), 10)
action := &Action{
Type: actionType,
Desc: desc,
Target: target,
Ts: ts,
}
// set action receivers
proxies, err := ProxyList(zkConn, productName, func(p *ProxyInfo) bool {
return p.State == PROXY_STATE_ONLINE
})
if err != nil {
return errors.Trace(err)
}
if needConfirm {
// do fencing here, make sure 'offline' proxies are really offline
// now we only check whether the proxy lists are match
fenceProxies, err := GetFenceProxyMap(zkConn, productName)
if err != nil {
return errors.Trace(err)
}
for _, proxy := range proxies {
delete(fenceProxies, proxy.Addr)
}
if len(fenceProxies) > 0 {
errMsg := bytes.NewBufferString("Some proxies may not stop cleanly:")
for k, _ := range fenceProxies {
errMsg.WriteString(" ")
errMsg.WriteString(k)
}
return errors.New(errMsg.String())
}
}
for _, p := range proxies {
buf, err := json.Marshal(p)
if err != nil {
return errors.Trace(err)
}
action.Receivers = append(action.Receivers, string(buf))
}
b, _ := json.Marshal(action)
prefix := GetWatchActionPath(productName)
//action root path
err = CreateActionRootPath(zkConn, prefix)
if err != nil {
return errors.Trace(err)
}
//response path
respPath := path.Join(path.Dir(prefix), "ActionResponse")
err = CreateActionRootPath(zkConn, respPath)
if err != nil {
return errors.Trace(err)
}
//create response node, etcd do not support create in order directory
//get path first
actionRespPath, err := zkConn.Create(respPath+"/", b, int32(zk.FlagSequence), zkhelper.DefaultFileACLs())
if err != nil {
log.Error(err, respPath)
return errors.Trace(err)
}
//remove file then create directory
zkConn.Delete(actionRespPath, -1)
actionRespPath, err = zkConn.Create(actionRespPath, b, 0, zkhelper.DefaultDirACLs())
if err != nil {
log.Error(err, respPath)
return errors.Trace(err)
}
suffix := path.Base(actionRespPath)
// create action node
actionPath := path.Join(prefix, suffix)
_, err = zkConn.Create(actionPath, b, 0, zkhelper.DefaultFileACLs())
if err != nil {
log.Error(err, actionPath)
return errors.Trace(err)
}
if needConfirm {
if err := WaitForReceiverWithTimeout(zkConn, productName, actionRespPath, proxies, timeoutInMs); err != nil {
return errors.Trace(err)
}
}
return nil
}