func (s *Slot) Update(zkConn zkhelper.Conn) error {
// status validation
switch s.State.Status {
case SLOT_STATUS_MIGRATE, SLOT_STATUS_OFFLINE,
SLOT_STATUS_ONLINE, SLOT_STATUS_PRE_MIGRATE:
{
// valid status, OK
}
default:
{
return errors.Trace(ErrUnknownSlotStatus)
}
}
data, err := json.Marshal(s)
if err != nil {
return errors.Trace(err)
}
zkPath := GetSlotPath(s.ProductName, s.Id)
_, err = zkhelper.CreateOrUpdate(zkConn, zkPath, string(data), 0, zkhelper.DefaultFileACLs(), true)
if err != nil {
return errors.Trace(err)
}
if s.State.Status == SLOT_STATUS_MIGRATE {
err = NewAction(zkConn, s.ProductName, ACTION_TYPE_SLOT_MIGRATE, s, "", true)
} else {
err = NewAction(zkConn, s.ProductName, ACTION_TYPE_SLOT_CHANGED, s, "", true)
}
return errors.Trace(err)
}
func TestProxyOfflineInWaitActionReceiver(t *testing.T) {
log.Infof("test proxy offline when waiting action response")
fakeZkConn := zkhelper.NewConn()
for i := 1; i <= 4; i++ {
CreateProxyInfo(fakeZkConn, productName, &ProxyInfo{
Id: strconv.Itoa(i),
State: PROXY_STATE_ONLINE,
})
go waitForProxyMarkOffline(fakeZkConn, strconv.Itoa(i))
}
lst, _ := ProxyList(fakeZkConn, productName, nil)
assert.Must(len(lst) == 4)
go func() {
time.Sleep(500 * time.Millisecond)
actionPath := path.Join(GetActionResponsePath(productName), fakeZkConn.Seq2Str(1))
//create test response for proxy 4, means proxy 1,2,3 are timeout
fakeZkConn.Create(path.Join(actionPath, "4"), nil,
0, zkhelper.DefaultFileACLs())
}()
err := NewActionWithTimeout(fakeZkConn, productName, ACTION_TYPE_SLOT_CHANGED, nil, "desc", true, 3*1000)
if err != nil {
assert.Must(err.Error() == ErrReceiverTimeout.Error())
}
for i := 1; i <= 3; i++ {
info, _ := GetProxyInfo(fakeZkConn, productName, strconv.Itoa(i))
assert.Must(info.State == PROXY_STATE_OFFLINE)
}
}
func CreateProxyInfo(zkConn zkhelper.Conn, productName string, pi *ProxyInfo) (string, error) {
data, err := json.Marshal(pi)
if err != nil {
return "", errors.Trace(err)
}
dir := GetProxyPath(productName)
zkhelper.CreateRecursive(zkConn, dir, "", 0, zkhelper.DefaultDirACLs())
return zkConn.Create(path.Join(dir, pi.Id), data, zk.FlagEphemeral, zkhelper.DefaultFileACLs())
}
func (self *ServerGroup) AddServer(zkConn zkhelper.Conn, s *Server, passwd string) error {
s.GroupId = self.Id
servers, err := self.GetServers(zkConn)
if err != nil {
return errors.Trace(err)
}
var masterAddr string
for _, server := range servers {
if server.Type == SERVER_TYPE_MASTER {
masterAddr = server.Addr
}
}
// make sure there is only one master
if s.Type == SERVER_TYPE_MASTER && len(masterAddr) > 0 {
return errors.Trace(ErrNodeExists)
}
// if this group has no server. auto promote this server to master
if len(servers) == 0 {
s.Type = SERVER_TYPE_MASTER
}
val, err := json.Marshal(s)
if err != nil {
return errors.Trace(err)
}
zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, s.Addr)
_, err = zkhelper.CreateOrUpdate(zkConn, zkPath, string(val), 0, zkhelper.DefaultFileACLs(), true)
if err != nil {
return errors.Trace(err)
}
// update servers
servers, err = self.GetServers(zkConn)
if err != nil {
return errors.Trace(err)
}
self.Servers = servers
if s.Type == SERVER_TYPE_MASTER {
err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", true)
if err != nil {
return errors.Trace(err)
}
} else if s.Type == SERVER_TYPE_SLAVE && len(masterAddr) > 0 {
// send command slaveof to slave
err := utils.SlaveOf(s.Addr, passwd, masterAddr)
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func doResponseForTest(conn zkhelper.Conn, seq string, pi *ProxyInfo) error {
actionPath := GetActionResponsePath(productName) + "/" + seq
data, err := json.Marshal(pi)
if err != nil {
return errors.Trace(err)
}
_, err = conn.Create(path.Join(actionPath, pi.Id), data,
0, zkhelper.DefaultFileACLs())
return err
}
func (top *Topology) DoResponse(seq int, pi *models.ProxyInfo) error {
//create response node
actionPath := top.GetActionResponsePath(seq)
//log.Debug("actionPath:", actionPath)
data, err := json.Marshal(pi)
if err != nil {
return errors.Trace(err)
}
_, err = top.zkConn.Create(path.Join(actionPath, pi.Id), data,
0, zkhelper.DefaultFileACLs())
return err
}
func SetSlotRange(zkConn zkhelper.Conn, productName string, fromSlot, toSlot, groupId int, status SlotStatus) error {
if status != SLOT_STATUS_OFFLINE && status != SLOT_STATUS_ONLINE {
return errors.Errorf("invalid status")
}
ok, err := GroupExists(zkConn, productName, groupId)
if err != nil {
return errors.Trace(err)
}
if !ok {
return errors.Errorf("group %d is not found", groupId)
}
for i := fromSlot; i <= toSlot; i++ {
s, err := GetSlot(zkConn, productName, i)
if err != nil {
return errors.Trace(err)
}
if s.State.Status != SLOT_STATUS_OFFLINE {
return errors.New(fmt.Sprintf("slot %d is not offline, if you want to change the group for a slot, use migrate", s.Id))
}
s.GroupId = groupId
s.State.Status = status
data, err := json.Marshal(s)
if err != nil {
return errors.Trace(err)
}
zkPath := GetSlotPath(productName, i)
_, err = zkhelper.CreateOrUpdate(zkConn, zkPath, string(data), 0, zkhelper.DefaultFileACLs(), true)
if err != nil {
return errors.Trace(err)
}
}
param := SlotMultiSetParam{
From: fromSlot,
To: toSlot,
GroupId: groupId,
Status: status,
}
err = NewAction(zkConn, productName, ACTION_TYPE_MULTI_SLOT_CHANGED, param, "", true)
return errors.Trace(err)
}
func createDashboardNode() error {
// make sure root dir is exists
rootDir := fmt.Sprintf("/zk/codis/db_%s", globalEnv.ProductName())
zkhelper.CreateRecursive(safeZkConn, rootDir, "", 0, zkhelper.DefaultDirACLs())
zkPath := fmt.Sprintf("%s/dashboard", rootDir)
// make sure we're the only one dashboard
if exists, _, _ := safeZkConn.Exists(zkPath); exists {
data, _, _ := safeZkConn.Get(zkPath)
return errors.New("dashboard already exists: " + string(data))
}
content := fmt.Sprintf(`{"addr": "%v", "pid": %v}`, globalEnv.DashboardAddr(), os.Getpid())
pathCreated, err := safeZkConn.Create(zkPath, []byte(content), 0, zkhelper.DefaultFileACLs())
createdDashboardNode = true
log.Info("dashboard node created:", pathCreated, string(content))
return errors.Trace(err)
}
func SetSlots(zkConn zkhelper.Conn, productName string, slots []*Slot, groupId int, status SlotStatus) error {
if status != SLOT_STATUS_OFFLINE && status != SLOT_STATUS_ONLINE {
return errors.Errorf("invalid status")
}
ok, err := GroupExists(zkConn, productName, groupId)
if err != nil {
return errors.Trace(err)
}
if !ok {
return errors.Errorf("group %d is not found", groupId)
}
for _, s := range slots {
s.GroupId = groupId
s.State.Status = status
data, err := json.Marshal(s)
if err != nil {
return errors.Trace(err)
}
zkPath := GetSlotPath(productName, s.Id)
_, err = zkhelper.CreateOrUpdate(zkConn, zkPath, string(data), 0, zkhelper.DefaultFileACLs(), true)
if err != nil {
return errors.Trace(err)
}
}
param := SlotMultiSetParam{
From: -1,
To: -1,
GroupId: groupId,
Status: status,
}
err = NewAction(zkConn, productName, ACTION_TYPE_MULTI_SLOT_CHANGED, param, "", true)
return errors.Trace(err)
}
//
// 注册一个服务的Endpoints
//
func (top *Topology) AddServiceEndPoint(service string, endpoint string, endpointInfo map[string]interface{}) error {
path := top.ProductServiceEndPointPath(service, endpoint)
data, err := json.Marshal(endpointInfo)
if err != nil {
return err
}
// 创建Service(XXX: Service本身不包含数据)
CreateRecursive(top.zkConn, os_path.Dir(path), "", 0, zkhelper.DefaultDirACLs())
// 当前的Session挂了，服务就下线
// topo.FlagEphemeral
// 参考： https://www.box.com/blog/a-gotcha-when-using-zookeeper-ephemeral-nodes/
// 如果之前的Session信息还存在，则先删除；然后再添加
top.zkConn.Delete(path, -1)
var pathCreated string
pathCreated, err = top.zkConn.Create(path, []byte(data), int32(topo.FlagEphemeral), zkhelper.DefaultFileACLs())
log.Println(green("SetRpcProxyData"), "Path: ", pathCreated, ", Error: ", err)
return err
}
func createDashboardNode() error {
// make sure root dir is exists
rootDir := fmt.Sprintf("/zk/codis/db_%s", globalEnv.ProductName())
zkhelper.CreateRecursive(safeZkConn, rootDir, "", 0, zkhelper.DefaultDirACLs())
zkPath := fmt.Sprintf("%s/dashboard", rootDir)
// make sure we're the only one dashboard
if exists, _, _ := safeZkConn.Exists(zkPath); exists {
data, _, _ := safeZkConn.Get(zkPath)
return errors.New("dashboard already exists: " + string(data))
}
content := fmt.Sprintf(`{"addr": "%v", "pid": %v}`, globalEnv.DashboardAddr(), os.Getpid())
pathCreated, err := safeZkConn.Create(zkPath, []byte(content), 0, zkhelper.DefaultFileACLs())
createdDashboardNode = true
log.Infof("dashboard node created: %v, %s", pathCreated, string(content))
log.Warn("********** Attention **********")
log.Warn("You should use `kill {pid}` rather than `kill -9 {pid}` to stop me,")
log.Warn("or the node resisted on zk will not be cleaned when I'm quiting and you must remove it manually")
log.Warn("*******************************")
return errors.Trace(err)
}
func CreateProxyFenceNode(zkConn zkhelper.Conn, productName string, pi *ProxyInfo) (string, error) {
return zkhelper.CreateRecursive(zkConn, path.Join(GetProxyFencePath(productName), pi.Addr), "",
0, zkhelper.DefaultFileACLs())
}
//add a new task to zk
func (m *MigrateManager) PostTask(info *MigrateTaskInfo) {
b, _ := json.Marshal(info)
p, _ := safeZkConn.Create(getMigrateTasksPath(m.productName)+"/", b, zk.FlagSequence, zkhelper.DefaultFileACLs())
_, info.Id = path.Split(p)
}
func NewActionWithTimeout(zkConn zkhelper.Conn, productName string, actionType ActionType, target interface{}, desc string, needConfirm bool, timeoutInMs int) error {
ts := strconv.FormatInt(time.Now().Unix(), 10)
action := &Action{
Type: actionType,
Desc: desc,
Target: target,
Ts: ts,
}
// set action receivers
proxies, err := ProxyList(zkConn, productName, func(p *ProxyInfo) bool {
return p.State == PROXY_STATE_ONLINE
})
if err != nil {
return errors.Trace(err)
}
if needConfirm {
// do fencing here, make sure 'offline' proxies are really offline
// now we only check whether the proxy lists are match
fenceProxies, err := GetFenceProxyMap(zkConn, productName)
if err != nil {
return errors.Trace(err)
}
for _, proxy := range proxies {
delete(fenceProxies, proxy.Addr)
}
if len(fenceProxies) > 0 {
errMsg := bytes.NewBufferString("Some proxies may not stop cleanly:")
for k, _ := range fenceProxies {
errMsg.WriteString(" ")
errMsg.WriteString(k)
}
return errors.Errorf("%s", errMsg)
}
}
for _, p := range proxies {
buf, err := json.Marshal(p)
if err != nil {
return errors.Trace(err)
}
action.Receivers = append(action.Receivers, string(buf))
}
b, _ := json.Marshal(action)
prefix := GetWatchActionPath(productName)
//action root path
err = CreateActionRootPath(zkConn, prefix)
if err != nil {
return errors.Trace(err)
}
//response path
respPath := path.Join(path.Dir(prefix), "ActionResponse")
err = CreateActionRootPath(zkConn, respPath)
if err != nil {
return errors.Trace(err)
}
//create response node, etcd do not support create in order directory
//get path first
actionRespPath, err := zkConn.Create(respPath+"/", b, int32(zk.FlagSequence), zkhelper.DefaultFileACLs())
if err != nil {
log.ErrorErrorf(err, "zk create resp node = %s", respPath)
return errors.Trace(err)
}
//remove file then create directory
zkConn.Delete(actionRespPath, -1)
actionRespPath, err = zkConn.Create(actionRespPath, b, 0, zkhelper.DefaultDirACLs())
if err != nil {
log.ErrorErrorf(err, "zk create resp node = %s", respPath)
return errors.Trace(err)
}
suffix := path.Base(actionRespPath)
// create action node
actionPath := path.Join(prefix, suffix)
_, err = zkConn.Create(actionPath, b, 0, zkhelper.DefaultFileACLs())
if err != nil {
log.ErrorErrorf(err, "zk create action path = %s", actionPath)
return errors.Trace(err)
}
if needConfirm {
if err := WaitForReceiverWithTimeout(zkConn, productName, actionRespPath, proxies, timeoutInMs); err != nil {
return errors.Trace(err)
}
}
return nil
}