func CreateProxyInfo(zkConn zkhelper.Conn, productName string, pi *ProxyInfo) (string, error) {
data, err := json.Marshal(pi)
if err != nil {
return "", errors.Trace(err)
}
dir := GetProxyPath(productName)
zkhelper.CreateRecursive(zkConn, dir, "", 0, zkhelper.DefaultDirACLs())
return zkConn.Create(path.Join(dir, pi.Id), data, zk.FlagEphemeral, zkhelper.DefaultFileACLs())
}
// 创建指定的Path
func (top *Topology) CreateDir(path string) (string, error) {
dir := top.FullPath(path)
if ok, _ := top.Exist(dir); ok {
log.Println("Path Exists")
return dir, nil
} else {
return zkhelper.CreateRecursive(top.zkConn, dir, "", 0, zkhelper.DefaultDirACLs())
}
}
func NewMigrateManager(zkConn zkhelper.Conn, pn string) *MigrateManager {
m := &MigrateManager{
zkConn: zkConn,
productName: pn,
}
zkhelper.CreateRecursive(m.zkConn, getMigrateTasksPath(m.productName), "", 0, zkhelper.DefaultDirACLs())
m.mayRecover()
go m.loop()
return m
}
//
// 设置RPC Proxy的数据:
// 绑定的前端的ip/port, 例如: {"rpc_front": "tcp://127.0.0.1:5550"}
//
func (top *Topology) SetRpcProxyData(proxyInfo map[string]interface{}) error {
path := top.FullPath("/rpc_proxy")
data, err := json.Marshal(proxyInfo)
if err != nil {
return err
}
// topo.FlagEphemeral 这里的ProxyInfo是手动配置的，需要持久化
path, err = CreateOrUpdate(top.zkConn, path, string(data), 0, zkhelper.DefaultDirACLs(), true)
log.Println(green("SetRpcProxyData"), "Path: ", path, ", Error: ", err, ", Data: ", string(data))
return err
}
func CreateActionRootPath(zkConn zkhelper.Conn, path string) error {
// if action dir not exists, create it first
exists, err := zkhelper.NodeExists(zkConn, path)
if err != nil {
return errors.Trace(err)
}
if !exists {
_, err := zkhelper.CreateOrUpdate(zkConn, path, "", 0, zkhelper.DefaultDirACLs(), true)
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (self *ServerGroup) Create(zkConn zkhelper.Conn) error {
if self.Id < 0 {
return errors.Errorf("invalid server group id %d", self.Id)
}
zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
_, err := zkhelper.CreateOrUpdate(zkConn, zkPath, "", 0, zkhelper.DefaultDirACLs(), true)
if err != nil {
return errors.Trace(err)
}
err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
if err != nil {
return errors.Trace(err)
}
return nil
}
func createDashboardNode() error {
// make sure root dir is exists
rootDir := fmt.Sprintf("/zk/codis/db_%s", globalEnv.ProductName())
zkhelper.CreateRecursive(safeZkConn, rootDir, "", 0, zkhelper.DefaultDirACLs())
zkPath := fmt.Sprintf("%s/dashboard", rootDir)
// make sure we're the only one dashboard
if exists, _, _ := safeZkConn.Exists(zkPath); exists {
data, _, _ := safeZkConn.Get(zkPath)
return errors.New("dashboard already exists: " + string(data))
}
content := fmt.Sprintf(`{"addr": "%v", "pid": %v}`, globalEnv.DashboardAddr(), os.Getpid())
pathCreated, err := safeZkConn.Create(zkPath, []byte(content), 0, zkhelper.DefaultFileACLs())
createdDashboardNode = true
log.Info("dashboard node created:", pathCreated, string(content))
return errors.Trace(err)
}
func (self *ServerGroup) Create(zkConn zkhelper.Conn) error {
if self.Id < 0 {
return errors.Errorf("invalid server group id %d", self.Id)
}
zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
_, err := zkhelper.CreateOrUpdate(zkConn, zkPath, "", 0, zkhelper.DefaultDirACLs(), true)
if err != nil {
return errors.Trace(err)
}
err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
if err != nil {
return errors.Trace(err)
}
// set no server slots' group id to this server group, no need to return error
slots, err := NoGroupSlots(zkConn, self.ProductName)
if err == nil && len(slots) > 0 {
SetSlots(zkConn, self.ProductName, slots, self.Id, SLOT_STATUS_ONLINE)
}
return nil
}
//
// 注册一个服务的Endpoints
//
func (top *Topology) AddServiceEndPoint(service string, endpoint string, endpointInfo map[string]interface{}) error {
path := top.ProductServiceEndPointPath(service, endpoint)
data, err := json.Marshal(endpointInfo)
if err != nil {
return err
}
// 创建Service(XXX: Service本身不包含数据)
CreateRecursive(top.zkConn, os_path.Dir(path), "", 0, zkhelper.DefaultDirACLs())
// 当前的Session挂了，服务就下线
// topo.FlagEphemeral
// 参考： https://www.box.com/blog/a-gotcha-when-using-zookeeper-ephemeral-nodes/
// 如果之前的Session信息还存在，则先删除；然后再添加
top.zkConn.Delete(path, -1)
var pathCreated string
pathCreated, err = top.zkConn.Create(path, []byte(data), int32(topo.FlagEphemeral), zkhelper.DefaultFileACLs())
log.Println(green("SetRpcProxyData"), "Path: ", pathCreated, ", Error: ", err)
return err
}
func createDashboardNode() error {
// make sure root dir is exists
rootDir := fmt.Sprintf("/zk/codis/db_%s", globalEnv.ProductName())
zkhelper.CreateRecursive(safeZkConn, rootDir, "", 0, zkhelper.DefaultDirACLs())
zkPath := fmt.Sprintf("%s/dashboard", rootDir)
// make sure we're the only one dashboard
if exists, _, _ := safeZkConn.Exists(zkPath); exists {
data, _, _ := safeZkConn.Get(zkPath)
return errors.New("dashboard already exists: " + string(data))
}
content := fmt.Sprintf(`{"addr": "%v", "pid": %v}`, globalEnv.DashboardAddr(), os.Getpid())
pathCreated, err := safeZkConn.Create(zkPath, []byte(content), 0, zkhelper.DefaultFileACLs())
createdDashboardNode = true
log.Infof("dashboard node created: %v, %s", pathCreated, string(content))
log.Warn("********** Attention **********")
log.Warn("You should use `kill {pid}` rather than `kill -9 {pid}` to stop me,")
log.Warn("or the node resisted on zk will not be cleaned when I'm quiting and you must remove it manually")
log.Warn("*******************************")
return errors.Trace(err)
}
func NewActionWithTimeout(zkConn zkhelper.Conn, productName string, actionType ActionType, target interface{}, desc string, needConfirm bool, timeoutInMs int) error {
ts := strconv.FormatInt(time.Now().Unix(), 10)
action := &Action{
Type: actionType,
Desc: desc,
Target: target,
Ts: ts,
}
// set action receivers
proxies, err := ProxyList(zkConn, productName, func(p *ProxyInfo) bool {
return p.State == PROXY_STATE_ONLINE
})
if err != nil {
return errors.Trace(err)
}
if needConfirm {
// do fencing here, make sure 'offline' proxies are really offline
// now we only check whether the proxy lists are match
fenceProxies, err := GetFenceProxyMap(zkConn, productName)
if err != nil {
return errors.Trace(err)
}
for _, proxy := range proxies {
delete(fenceProxies, proxy.Addr)
}
if len(fenceProxies) > 0 {
errMsg := bytes.NewBufferString("Some proxies may not stop cleanly:")
for k, _ := range fenceProxies {
errMsg.WriteString(" ")
errMsg.WriteString(k)
}
return errors.Errorf("%s", errMsg)
}
}
for _, p := range proxies {
buf, err := json.Marshal(p)
if err != nil {
return errors.Trace(err)
}
action.Receivers = append(action.Receivers, string(buf))
}
b, _ := json.Marshal(action)
prefix := GetWatchActionPath(productName)
//action root path
err = CreateActionRootPath(zkConn, prefix)
if err != nil {
return errors.Trace(err)
}
//response path
respPath := path.Join(path.Dir(prefix), "ActionResponse")
err = CreateActionRootPath(zkConn, respPath)
if err != nil {
return errors.Trace(err)
}
//create response node, etcd do not support create in order directory
//get path first
actionRespPath, err := zkConn.Create(respPath+"/", b, int32(zk.FlagSequence), zkhelper.DefaultFileACLs())
if err != nil {
log.ErrorErrorf(err, "zk create resp node = %s", respPath)
return errors.Trace(err)
}
//remove file then create directory
zkConn.Delete(actionRespPath, -1)
actionRespPath, err = zkConn.Create(actionRespPath, b, 0, zkhelper.DefaultDirACLs())
if err != nil {
log.ErrorErrorf(err, "zk create resp node = %s", respPath)
return errors.Trace(err)
}
suffix := path.Base(actionRespPath)
// create action node
actionPath := path.Join(prefix, suffix)
_, err = zkConn.Create(actionPath, b, 0, zkhelper.DefaultFileACLs())
if err != nil {
log.ErrorErrorf(err, "zk create action path = %s", actionPath)
return errors.Trace(err)
}
if needConfirm {
if err := WaitForReceiverWithTimeout(zkConn, productName, actionRespPath, proxies, timeoutInMs); err != nil {
return errors.Trace(err)
}
}
return nil
}
func TestWaitForReceiverTimeout(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
proxies := []ProxyInfo{}
for i := 0; i < 5; i++ {
proxies = append(proxies, ProxyInfo{
Id: fmt.Sprintf("proxy_%d", i),
Addr: fmt.Sprintf("localhost:%d", i+1234),
State: PROXY_STATE_ONLINE,
})
CreateProxyInfo(fakeZkConn, productName, &proxies[i])
}
zkhelper.CreateRecursive(fakeZkConn, GetActionResponsePath(productName)+"/1", "", 0, zkhelper.DefaultDirACLs())
go func() {
time.Sleep(time.Second * 2)
doResponseForTest(fakeZkConn, "1", &proxies[0])
doResponseForTest(fakeZkConn, "1", &proxies[2])
doResponseForTest(fakeZkConn, "1", &proxies[4])
for {
for i := 0; i < 5; i++ {
pname := fmt.Sprintf("proxy_%d", i)
p, _ := GetProxyInfo(fakeZkConn, productName, pname)
if p != nil && p.State == PROXY_STATE_MARK_OFFLINE {
zkhelper.DeleteRecursive(fakeZkConn, path.Join(GetProxyPath(productName), pname), -1)
}
}
}
}()
err := WaitForReceiverWithTimeout(fakeZkConn, productName, GetActionResponsePath(productName)+"/1", proxies, 1000*10)
if err != ErrReceiverTimeout {
t.Error("there is no timeout as expected")
}
p, _ := GetProxyInfo(fakeZkConn, productName, "proxy_0")
if p == nil || p.State != PROXY_STATE_ONLINE {
t.Error("proxy_0 status is not as expected")
}
p, _ = GetProxyInfo(fakeZkConn, productName, "proxy_1")
if p != nil {
t.Error("proxy_1 status is not as expected")
}
p, _ = GetProxyInfo(fakeZkConn, productName, "proxy_2")
if p == nil || p.State != PROXY_STATE_ONLINE {
t.Error("proxy_2 status is not as expected")
}
p, _ = GetProxyInfo(fakeZkConn, productName, "proxy_3")
if p != nil {
t.Error("proxy_3 status is not as expected")
}
p, _ = GetProxyInfo(fakeZkConn, productName, "proxy_4")
if p == nil || p.State != PROXY_STATE_ONLINE {
t.Error("proxy_4 status is not as expected")
}
}