// create new server group
func apiAddServerGroup(newGroup models.ServerGroup) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("add group %+v", newGroup)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
newGroup.ProductName = globalEnv.ProductName()
exists, err := newGroup.Exists(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "check group exits failed")
return 500, err.Error()
}
if exists {
return 500, "group already exists"
}
err = newGroup.Create(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "create node for new group failed")
return 500, err.Error()
}
return jsonRetSucc()
}
/*
* return err should be reconnect to zk
*
*/
func (s *Server) fillSlot(i int) error {
slotInfo, slotGroup, err := s.topo.GetSlotByIndex(i)
if err != nil {
log.ErrorErrorf(err, "get slot by index failed", i)
return err
}
var from string
var addr = groupMaster(*slotGroup)
if slotInfo.State.Status == models.SLOT_STATUS_MIGRATE {
fromGroup, err := s.topo.GetGroup(slotInfo.State.MigrateStatus.From)
if err != nil {
log.ErrorErrorf(err, "get migrate from failed")
return err
}
from = groupMaster(*fromGroup)
if from == addr {
log.Errorf("set slot %04d migrate from %s to %s", i, from, addr)
return nil
}
if "" == addr {
log.Errorf("set slot %04d addr nil", i)
return nil
}
}
s.groups[i] = slotInfo.GroupId
s.router.FillSlot(i, addr, from,
slotInfo.State.Status == models.SLOT_STATUS_PRE_MIGRATE)
return err
}
func (s *Server) register() {
var err error
for {
if _, err = s.topo.CreateProxyInfo(&s.info); err != nil {
log.ErrorErrorf(err, "create proxy node failed")
if s.topo.IsErrSessionExpired(err) {
s.topo.RefreshZkConn()
}
time.Sleep(ZK_RECONNECT_INTERVAL * time.Second)
} else {
break
}
}
for {
if _, err = s.topo.CreateProxyFenceNode(&s.info); err != nil {
log.ErrorErrorf(err, "create fence node failed")
if s.topo.IsErrSessionExpired(err) {
s.topo.RefreshZkConn()
} else if s.topo.IsErrNodeExist(err) {
break
}
time.Sleep(ZK_RECONNECT_INTERVAL * time.Second)
} else {
break
}
}
}
func apiPromoteServer(server models.Server, param martini.Params) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("promote server %+v", server)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
group, err := models.GetGroup(safeZkConn, globalEnv.ProductName(), server.GroupId)
if err != nil {
log.ErrorErrorf(err, "get group %d failed", server.GroupId)
return 500, err.Error()
}
err = group.Promote(safeZkConn, server.Addr, globalEnv.Password())
if err != nil {
log.ErrorErrorf(err, "promote group %d failed", server.GroupId)
return 500, err.Error()
}
return jsonRetSucc()
}
func apiSlotRangeSet(task RangeSetTask) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("set slot range, %+v", task)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
// default set online
if len(task.Status) == 0 {
task.Status = string(models.SLOT_STATUS_ONLINE)
}
err := models.SetSlotRange(safeZkConn, globalEnv.ProductName(), task.FromSlot, task.ToSlot, task.NewGroupId, models.SlotStatus(task.Status))
if err != nil {
log.ErrorErrorf(err, "set slot range [%d,%d] failed", task.FromSlot, task.ToSlot)
return 500, err.Error()
}
return jsonRetSucc()
}
func (top *Topology) GetSlotByIndex(i int) (*models.Slot, *models.ServerGroup, error) {
var slot *models.Slot
var groupServer *models.ServerGroup
var err error
for {
slot, err = models.GetSlot(top.zkConn, top.ProductName, i)
if err == nil || top.IsFatalErr(err) {
break
} else {
time.Sleep(ZK_RECONNECT_INTERVAL * time.Second)
}
if err != nil {
log.ErrorErrorf(err, " GetSlot ")
}
}
if err != nil {
return nil, nil, errors.Trace(err)
}
for {
groupServer, err = models.GetGroup(top.zkConn, top.ProductName, slot.GroupId)
if err == nil || top.IsFatalErr(err) {
break
} else {
time.Sleep(ZK_RECONNECT_INTERVAL * time.Second)
}
if err != nil {
log.ErrorErrorf(err, " GetGroup ")
}
}
if err != nil {
return nil, nil, errors.Trace(err)
}
return slot, groupServer, nil
}
// add redis server to exist server group
func apiAddServerToGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("add server to group, %+v", server)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
// check group exists first
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
exists, err := serverGroup.Exists(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "check group exits failed")
return 500, err.Error()
}
// create new group if not exists
if !exists {
if err := serverGroup.Create(safeZkConn); err != nil {
return 500, err.Error()
}
}
if err := serverGroup.AddServer(safeZkConn, &server, globalEnv.Password()); err != nil {
log.ErrorErrorf(err, "add server to group failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func apiGetServerGroup(param martini.Params) (int, string) {
id := param["id"]
groupId, err := strconv.Atoi(id)
if err != nil {
log.ErrorErrorf(err, "parse groupid failed")
return 500, err.Error()
}
group, err := models.GetGroup(safeZkConn, globalEnv.ProductName(), groupId)
if err != nil {
log.ErrorErrorf(err, "get group %d failed", groupId)
return 500, err.Error()
}
b, err := json.MarshalIndent(group, " ", " ")
return 200, string(b)
}
func (s *Server) waitOnline() bool {
for {
info, err := s.topo.GetProxyInfo(s.info.Id)
if err != nil {
log.ErrorErrorf(err, "get proxy info failed: %s", s.info.Id)
if s.topo.IsFatalErr(err) {
s.reRegister(models.PROXY_STATE_MARK_OFFLINE, true)
}
continue
}
switch info.State {
case models.PROXY_STATE_MARK_OFFLINE:
log.Infof("mark offline, proxy got offline event: %s", s.info.Id)
s.markOffline()
return false
case models.PROXY_STATE_ONLINE:
s.info.State = info.State
log.Infof("we are online: %s", s.info.Id)
s.rewatchProxy(false)
return true
}
select {
case <-s.kill:
log.Infof("mark offline, proxy is killed: %s", s.info.Id)
s.markOffline()
return false
default:
}
log.Infof("wait to be online: %s", s.info.Id)
time.Sleep(3 * time.Second)
}
}
func (s *Server) loopEvents() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
var tick int = 0
for s.info.State == models.PROXY_STATE_ONLINE {
select {
case <-s.kill:
log.Infof("mark offline, proxy is killed: %s", s.info.Id)
s.markOffline()
case e := <-s.evtbus:
evtPath := getEventPath(e)
log.Infof("got event %s, %v, lastActionSeq %d", s.info.Id, e, s.lastActionSeq)
if strings.Index(evtPath, models.GetActionResponsePath(s.conf.productName)) == 0 {
seq, err := strconv.Atoi(path.Base(evtPath))
if err != nil {
log.ErrorErrorf(err, "parse action seq failed")
} else {
if seq < s.lastActionSeq {
log.Infof("ignore seq = %d", seq)
continue
}
}
}
s.processAction(e)
case <-ticker.C:
if maxTick := s.conf.pingPeriod; maxTick != 0 {
if tick++; tick >= maxTick {
s.router.KeepAlive()
tick = 0
}
}
}
}
}
func cmdProxy(argv []string) (err error) {
usage := `usage:
codis-config proxy list
codis-config proxy offline <proxy_name>
codis-config proxy online <proxy_name>
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.ErrorErrorf(err, "parse args failed")
return err
}
log.Debugf("parse args = {%+v}", args)
if args["list"].(bool) {
return runProxyList()
}
proxyName := args["<proxy_name>"].(string)
if args["online"].(bool) {
return runSetProxyStatus(proxyName, models.PROXY_STATE_ONLINE)
}
if args["offline"].(bool) {
return runSetProxyStatus(proxyName, models.PROXY_STATE_MARK_OFFLINE)
}
return nil
}
func cmdDashboard(argv []string) (err error) {
usage := `usage: codis-config dashboard [--addr=<address>] [--http-log=<log_file>]
options:
--addr listen ip:port, e.g. localhost:12345, :8086, [default: :8086]
--http-log http request log [default: request.log ]
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.ErrorErrorf(err, "parse args failed")
return err
}
log.Debugf("parse args = {%+v}", args)
logFileName := "request.log"
if args["--http-log"] != nil {
logFileName = args["--http-log"].(string)
}
addr := ":8086"
if args["--addr"] != nil {
addr = args["--addr"].(string)
}
runDashboard(addr, logFileName)
return nil
}
func getAllProxySlowop() map[string][]*router.SlowOpInfo {
proxies, err := models.ProxyList(unsafeZkConn, globalEnv.ProductName(), nil)
if err != nil {
log.ErrorErrorf(err, "get proxy list failed")
return nil
}
ret := make(map[string][]*router.SlowOpInfo)
for _, p := range proxies {
m, err := p.DebugVars()
if err != nil {
log.WarnErrorf(err, "get proxy debug varsfailed")
}
if m["router"] != nil {
route := m["router"].(map[string]interface{})
if route["slowop"] != nil {
ops := make([]*router.SlowOpInfo, 0, 10)
for _, elem := range route["slowop"].([]interface{}) {
op := elem.(map[string]interface{})
s := &router.SlowOpInfo{
Time: op["time"].(string),
Key: op["key"].(string),
Duration: int64(op["duration"].(float64)),
Reqs: int64(op["req"].(float64)),
}
ops = append(ops, s)
}
ret[p.Id] = ops
}
}
}
return ret
}
// actions
func apiActionGC(r *http.Request) (int, string) {
r.ParseForm()
keep, _ := strconv.Atoi(r.FormValue("keep"))
secs, _ := strconv.Atoi(r.FormValue("secs"))
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("action gc")); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
var err error
if keep >= 0 {
err = models.ActionGC(safeZkConn, globalEnv.ProductName(), models.GC_TYPE_N, keep)
} else if secs > 0 {
err = models.ActionGC(safeZkConn, globalEnv.ProductName(), models.GC_TYPE_SEC, secs)
}
if err != nil {
return 500, err.Error()
}
return jsonRetSucc()
}
func cmdServer(argv []string) (err error) {
usage := `usage:
codis-config server list
codis-config server add <group_id> <redis_addr> <role>
codis-config server remove <group_id> <redis_addr>
codis-config server promote <group_id> <redis_addr>
codis-config server add-group <group_id>
codis-config server remove-group <group_id>
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.ErrorErrorf(err, "parse args failed")
return err
}
log.Debugf("parse args = {%+v}", args)
if args["list"].(bool) {
return runListServerGroup()
}
groupId, err := strconv.Atoi(args["<group_id>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <group_id> failed")
return err
}
if args["remove-group"].(bool) {
return runRemoveServerGroup(groupId)
}
if args["add-group"].(bool) {
return runAddServerGroup(groupId)
}
serverAddr := args["<redis_addr>"].(string)
if args["add"].(bool) {
role := args["<role>"].(string)
return runAddServerToGroup(groupId, serverAddr, role)
}
if args["remove"].(bool) {
return runRemoveServerFromGroup(groupId, serverAddr)
}
if args["promote"].(bool) {
return runPromoteServerToMaster(groupId, serverAddr)
}
return nil
}
func apiForceRemoveLocks() (int, string) {
err := models.ForceRemoveLock(safeZkConn, globalEnv.ProductName())
if err != nil {
log.ErrorErrorf(err, "force remove lock failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func cmdAction(argv []string) (err error) {
usage := `usage: codis-config action (gc [-n <num> | -s <seconds>] | remove-lock | remove-fence)
options:
gc:
gc -n N keep last N actions;
gc -s Sec keep last Sec seconds actions;
remove-lock force remove zookeeper lock;
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.ErrorErrorf(err, "parse args failed")
return errors.Trace(err)
}
log.Debugf("parse args = {%+v}", args)
if args["remove-lock"].(bool) {
return errors.Trace(runRemoveLock())
}
if args["remove-fence"].(bool) {
return errors.Trace(runRemoveFence())
}
if args["gc"].(bool) {
if args["-n"].(bool) {
n, err := strconv.Atoi(args["<num>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <num> failed")
return err
}
return runGCKeepN(n)
} else if args["-s"].(bool) {
sec, err := strconv.Atoi(args["<seconds>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <seconds> failed")
return errors.Trace(err)
}
return runGCKeepNSec(sec)
}
}
return nil
}
func apiGetServerGroupList() (int, string) {
groups, err := models.ServerGroups(safeZkConn, globalEnv.ProductName())
if err != nil {
log.ErrorErrorf(err, "get server groups failed")
return 500, err.Error()
}
b, err := json.MarshalIndent(groups, " ", " ")
return 200, string(b)
}
func apiGetSlots() (int, string) {
slots, err := models.Slots(safeZkConn, globalEnv.ProductName())
if err != nil {
log.ErrorErrorf(err, "Error getting slot info, try init slots first?")
return 500, err.Error()
}
b, err := json.MarshalIndent(slots, " ", " ")
return 200, string(b)
}
func apiGetProxyList(param martini.Params) (int, string) {
proxies, err := models.ProxyList(safeZkConn, globalEnv.ProductName(), nil)
if err != nil {
log.ErrorErrorf(err, "get proxy list failed")
return 500, err.Error()
}
b, err := json.MarshalIndent(proxies, " ", " ")
return 200, string(b)
}
func (m *MigrateManager) loop() error {
for {
time.Sleep(time.Second)
info := m.NextTask()
if info == nil {
continue
}
t := GetMigrateTask(*info)
err := t.preMigrateCheck()
if err != nil {
log.ErrorErrorf(err, "pre migrate check failed")
}
err = t.run()
if err != nil {
log.ErrorErrorf(err, "migrate failed")
}
}
}
func apiGetRedisSlotInfo(param martini.Params) (int, string) {
addr := param["addr"]
slotId, err := strconv.Atoi(param["id"])
if err != nil {
log.ErrorErrorf(err, "parse slotid failed")
return 500, err.Error()
}
slotInfo, err := utils.SlotsInfo(addr, globalEnv.Password(), slotId, slotId)
if err != nil {
log.ErrorErrorf(err, "get slot info %d failed", slotId)
return 500, err.Error()
}
out, _ := json.MarshalIndent(map[string]interface{}{
"keys": slotInfo[slotId],
"slot_id": slotId,
}, " ", " ")
return 200, string(out)
}
func apiRebalance(param martini.Params) (int, string) {
if len(globalMigrateManager.Tasks()) > 0 {
return 500, "there are migration tasks running, you should wait them done"
}
if err := Rebalance(); err != nil {
log.ErrorErrorf(err, "rebalance failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func apiRemoveServerFromGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("removing server from group, %+v", server))
defer func() {
err := lock.Unlock()
if err != nil {
log.ErrorErrorf(err, "unlock node failed")
}
}()
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
err := serverGroup.RemoveServer(safeZkConn, server.Addr)
if err != nil {
log.ErrorErrorf(err, "remove group %d failed", groupId)
return 500, err.Error()
}
return jsonRetSucc()
}
func apiGetRedisSlotInfoFromGroupId(param martini.Params) (int, string) {
groupId, err := strconv.Atoi(param["group_id"])
if err != nil {
log.ErrorErrorf(err, "parse groupid failed")
return 500, err.Error()
}
slotId, err := strconv.Atoi(param["slot_id"])
if err != nil {
log.ErrorErrorf(err, "parse slotid failed")
return 500, err.Error()
}
g, err := models.GetGroup(safeZkConn, globalEnv.ProductName(), groupId)
if err != nil {
log.ErrorErrorf(err, "get group %d failed", groupId)
return 500, err.Error()
}
s, err := g.Master(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "get master of group %d failed", groupId)
return 500, err.Error()
}
if s == nil {
return 500, "master not found"
}
slotInfo, err := utils.SlotsInfo(s.Addr, globalEnv.Password(), slotId, slotId)
if err != nil {
log.ErrorErrorf(err, "get slot info %d failed", slotId)
return 500, err.Error()
}
out, _ := json.MarshalIndent(map[string]interface{}{
"keys": slotInfo[slotId],
"slot_id": slotId,
"group_id": groupId,
"addr": s.Addr,
}, " ", " ")
return 200, string(out)
}
func apiRemoveServerGroup(param martini.Params) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("removing group %s", param["id"]))
defer func() {
err := lock.Unlock()
if err != nil {
log.ErrorErrorf(err, "unlock node failed")
}
}()
groupId, _ := strconv.Atoi(param["id"])
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
if err := serverGroup.Remove(safeZkConn); err != nil {
log.ErrorErrorf(err, "remove server group failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func apiSetProxyStatus(proxy models.ProxyInfo, param martini.Params) (int, string) {
err := models.SetProxyStatus(safeZkConn, globalEnv.ProductName(), proxy.Id, proxy.State)
if err != nil {
// if this proxy is not online, just return success
if proxy.State == models.PROXY_STATE_MARK_OFFLINE && zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return jsonRetSucc()
}
log.ErrorErrorf(err, "set proxy states failed: %+v", proxy)
return 500, err.Error()
}
return jsonRetSucc()
}
func (c *safeConn) SetACL(path string, aclv []zk.ACL, version int32) (stat zk.Stat, err error) {
for i := 0; i <= retryMaxOnOps; i++ {
c.builder.lock.RLock()
stat, err = c.Conn.SetACL(path, aclv, version)
c.builder.lock.RUnlock()
if err == nil || !isConnectionError(err) {
return
}
c.builder.resetConnection()
}
log.ErrorErrorf(err, "zk error after retries")
return
}
func (c *safeConn) Get(path string) (data []byte, stat zk.Stat, err error) {
for i := 0; i <= retryMaxOnOps; i++ {
c.builder.lock.RLock()
data, stat, err = c.Conn.Get(path)
c.builder.lock.RUnlock()
if err == nil || !isConnectionError(err) {
return
}
c.builder.resetConnection()
}
log.ErrorErrorf(err, "zk error after retries")
return
}
func (top *Topology) InitZkConn() {
var err error
for {
connConf := topo.ConnConf{time.Duration(top.readTimeout) * time.Second, time.Duration(top.connTimeout) * time.Second, int32(top.zkSessionTimeout * 1000)}
top.zkConn, err = top.fact.ConnectWithConf(top.zkAddr, connConf)
if err != nil {
log.ErrorErrorf(err, "init failed")
time.Sleep(5 * time.Second)
} else {
break
}
}
}