// add redis server to exist server group
func apiAddServerToGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("add server to group, %+v", server)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
// check group exists first
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
exists, err := serverGroup.Exists(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "check group exits failed")
return 500, err.Error()
}
// create new group if not exists
if !exists {
if err := serverGroup.Create(safeZkConn); err != nil {
return 500, err.Error()
}
}
if err := serverGroup.AddServer(safeZkConn, &server, globalEnv.Password()); err != nil {
log.ErrorErrorf(err, "add server to group failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func apiPromoteServer(server models.Server, param martini.Params) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("promote server %+v", server)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
group, err := models.GetGroup(safeZkConn, globalEnv.ProductName(), server.GroupId)
if err != nil {
log.ErrorErrorf(err, "get group %d failed", server.GroupId)
return 500, err.Error()
}
err = group.Promote(safeZkConn, server.Addr, globalEnv.Password())
if err != nil {
log.ErrorErrorf(err, "promote group %d failed", server.GroupId)
return 500, err.Error()
}
return jsonRetSucc()
}
func apiSlotRangeSet(task RangeSetTask) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("set slot range, %+v", task)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
// default set online
if len(task.Status) == 0 {
task.Status = string(models.SLOT_STATUS_ONLINE)
}
err := models.SetSlotRange(safeZkConn, globalEnv.ProductName(), task.FromSlot, task.ToSlot, task.NewGroupId, models.SlotStatus(task.Status))
if err != nil {
log.ErrorErrorf(err, "set slot range [%d,%d] failed", task.FromSlot, task.ToSlot)
return 500, err.Error()
}
return jsonRetSucc()
}
// create new server group
func apiAddServerGroup(newGroup models.ServerGroup) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("add group %+v", newGroup)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
newGroup.ProductName = globalEnv.ProductName()
exists, err := newGroup.Exists(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "check group exits failed")
return 500, err.Error()
}
if exists {
return 500, "group already exists"
}
err = newGroup.Create(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "create node for new group failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func apiGetServerGroup(param martini.Params) (int, string) {
id := param["id"]
groupId, err := strconv.Atoi(id)
if err != nil {
log.ErrorErrorf(err, "parse groupid failed")
return 500, err.Error()
}
group, err := models.GetGroup(safeZkConn, globalEnv.ProductName(), groupId)
if err != nil {
log.ErrorErrorf(err, "get group %d failed", groupId)
return 500, err.Error()
}
b, err := json.MarshalIndent(group, " ", " ")
return 200, string(b)
}
// actions
func apiActionGC(r *http.Request) (int, string) {
r.ParseForm()
keep, _ := strconv.Atoi(r.FormValue("keep"))
secs, _ := strconv.Atoi(r.FormValue("secs"))
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("action gc")); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
var err error
if keep >= 0 {
err = models.ActionGC(safeZkConn, globalEnv.ProductName(), models.GC_TYPE_N, keep)
} else if secs > 0 {
err = models.ActionGC(safeZkConn, globalEnv.ProductName(), models.GC_TYPE_SEC, secs)
}
if err != nil {
return 500, err.Error()
}
return jsonRetSucc()
}
func (s *Server) loopEvents() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
var tick int = 0
for s.info.State == models.PROXY_STATE_ONLINE {
select {
case <-s.kill:
log.Infof("mark offline, proxy is killed: %s", s.info.Id)
s.markOffline()
case e := <-s.evtbus:
evtPath := getEventPath(e)
log.Infof("got event %s, %v, lastActionSeq %d", s.info.Id, e, s.lastActionSeq)
if strings.Index(evtPath, models.GetActionResponsePath(s.conf.productName)) == 0 {
seq, err := strconv.Atoi(path.Base(evtPath))
if err != nil {
log.ErrorErrorf(err, "parse action seq failed")
} else {
if seq < s.lastActionSeq {
log.Infof("ignore seq = %d", seq)
continue
}
}
}
s.processAction(e)
case <-ticker.C:
if maxTick := s.conf.pingPeriod; maxTick != 0 {
if tick++; tick >= maxTick {
s.router.KeepAlive()
tick = 0
}
}
}
}
}
func cmdProxy(argv []string) (err error) {
usage := `usage:
codis-config proxy list
codis-config proxy offline <proxy_name>
codis-config proxy online <proxy_name>
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.ErrorErrorf(err, "parse args failed")
return err
}
log.Debugf("parse args = {%+v}", args)
if args["list"].(bool) {
return runProxyList()
}
proxyName := args["<proxy_name>"].(string)
if args["online"].(bool) {
return runSetProxyStatus(proxyName, models.PROXY_STATE_ONLINE)
}
if args["offline"].(bool) {
return runSetProxyStatus(proxyName, models.PROXY_STATE_MARK_OFFLINE)
}
return nil
}
func cmdDashboard(argv []string) (err error) {
usage := `usage: codis-config dashboard [--addr=<address>] [--http-log=<log_file>]
options:
--addr listen ip:port, e.g. localhost:18087, :18087, [default: :18087]
--http-log http request log [default: request.log ]
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.ErrorErrorf(err, "parse args failed")
return err
}
log.Debugf("parse args = {%+v}", args)
logFileName := "request.log"
if args["--http-log"] != nil {
logFileName = args["--http-log"].(string)
}
addr := ":18087"
if args["--addr"] != nil {
addr = args["--addr"].(string)
}
runDashboard(addr, logFileName)
return nil
}
func cmdServer(argv []string) (err error) {
usage := `usage:
codis-config server list
codis-config server add <group_id> <redis_addr> <role>
codis-config server remove <group_id> <redis_addr>
codis-config server promote <group_id> <redis_addr>
codis-config server add-group <group_id>
codis-config server remove-group <group_id>
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.ErrorErrorf(err, "parse args failed")
return err
}
log.Debugf("parse args = {%+v}", args)
if args["list"].(bool) {
return runListServerGroup()
}
groupId, err := strconv.Atoi(args["<group_id>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <group_id> failed")
return err
}
if args["remove-group"].(bool) {
return runRemoveServerGroup(groupId)
}
if args["add-group"].(bool) {
return runAddServerGroup(groupId)
}
serverAddr := args["<redis_addr>"].(string)
if args["add"].(bool) {
role := args["<role>"].(string)
return runAddServerToGroup(groupId, serverAddr, role)
}
if args["remove"].(bool) {
return runRemoveServerFromGroup(groupId, serverAddr)
}
if args["promote"].(bool) {
return runPromoteServerToMaster(groupId, serverAddr)
}
return nil
}
func apiForceRemoveLocks() (int, string) {
err := models.ForceRemoveLock(safeZkConn, globalEnv.ProductName())
if err != nil {
log.ErrorErrorf(err, "force remove lock failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func cmdAction(argv []string) (err error) {
usage := `usage: codis-config action (gc [-n <num> | -s <seconds>] | remove-lock | remove-fence)
options:
gc:
gc -n N keep last N actions;
gc -s Sec keep last Sec seconds actions;
remove-lock force remove zookeeper lock;
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.ErrorErrorf(err, "parse args failed")
return errors.Trace(err)
}
log.Debugf("parse args = {%+v}", args)
if args["remove-lock"].(bool) {
return errors.Trace(runRemoveLock())
}
if args["remove-fence"].(bool) {
return errors.Trace(runRemoveFence())
}
if args["gc"].(bool) {
if args["-n"].(bool) {
n, err := strconv.Atoi(args["<num>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <num> failed")
return err
}
return runGCKeepN(n)
} else if args["-s"].(bool) {
sec, err := strconv.Atoi(args["<seconds>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <seconds> failed")
return errors.Trace(err)
}
return runGCKeepNSec(sec)
}
}
return nil
}
func apiGetRedisSlotInfo(param martini.Params) (int, string) {
addr := param["addr"]
slotId, err := strconv.Atoi(param["id"])
if err != nil {
log.ErrorErrorf(err, "parse slotid failed")
return 500, err.Error()
}
slotInfo, err := utils.SlotsInfo(addr, globalEnv.Password(), slotId, slotId)
if err != nil {
log.ErrorErrorf(err, "get slot info %d failed", slotId)
return 500, err.Error()
}
out, _ := json.MarshalIndent(map[string]interface{}{
"keys": slotInfo[slotId],
"slot_id": slotId,
}, " ", " ")
return 200, string(out)
}
func apiGetServerGroupList() (int, string) {
groups, err := models.ServerGroups(safeZkConn, globalEnv.ProductName())
if err != nil {
log.ErrorErrorf(err, "get server groups failed")
return 500, err.Error()
}
b, err := json.MarshalIndent(groups, " ", " ")
return 200, string(b)
}
func (m *MigrateManager) loop() error {
for {
time.Sleep(time.Second)
info := m.NextTask()
if info == nil {
continue
}
t := GetMigrateTask(*info)
err := t.preMigrateCheck()
if err != nil {
log.ErrorErrorf(err, "pre migrate check failed")
}
err = t.run()
if err != nil {
log.ErrorErrorf(err, "migrate failed")
}
}
}
func apiGetSlots() (int, string) {
slots, err := models.Slots(safeZkConn, globalEnv.ProductName())
if err != nil {
log.ErrorErrorf(err, "Error getting slot info, try init slots first?")
return 500, err.Error()
}
b, err := json.MarshalIndent(slots, " ", " ")
return 200, string(b)
}
func apiGetProxyList(param martini.Params) (int, string) {
proxies, err := models.ProxyList(safeZkConn, globalEnv.ProductName(), nil)
if err != nil {
log.ErrorErrorf(err, "get proxy list failed")
return 500, err.Error()
}
b, err := json.MarshalIndent(proxies, " ", " ")
return 200, string(b)
}
func apiRebalance(param martini.Params) (int, string) {
if len(globalMigrateManager.Tasks()) > 0 {
return 500, "there are migration tasks running, you should wait them done"
}
if err := Rebalance(); err != nil {
log.ErrorErrorf(err, "rebalance failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func apiGetRedisSlotInfoFromGroupId(param martini.Params) (int, string) {
groupId, err := strconv.Atoi(param["group_id"])
if err != nil {
log.ErrorErrorf(err, "parse groupid failed")
return 500, err.Error()
}
slotId, err := strconv.Atoi(param["slot_id"])
if err != nil {
log.ErrorErrorf(err, "parse slotid failed")
return 500, err.Error()
}
g, err := models.GetGroup(safeZkConn, globalEnv.ProductName(), groupId)
if err != nil {
log.ErrorErrorf(err, "get group %d failed", groupId)
return 500, err.Error()
}
s, err := g.Master(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "get master of group %d failed", groupId)
return 500, err.Error()
}
if s == nil {
return 500, "master not found"
}
slotInfo, err := utils.SlotsInfo(s.Addr, globalEnv.Password(), slotId, slotId)
if err != nil {
log.ErrorErrorf(err, "get slot info %d failed", slotId)
return 500, err.Error()
}
out, _ := json.MarshalIndent(map[string]interface{}{
"keys": slotInfo[slotId],
"slot_id": slotId,
"group_id": groupId,
"addr": s.Addr,
}, " ", " ")
return 200, string(out)
}
func apiSetProxyStatus(proxy models.ProxyInfo, param martini.Params) (int, string) {
err := models.SetProxyStatus(safeZkConn, globalEnv.ProductName(), proxy.Id, proxy.State)
if err != nil {
// if this proxy is not online, just return success
if proxy.State == models.PROXY_STATE_MARK_OFFLINE && zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return jsonRetSucc()
}
log.ErrorErrorf(err, "set proxy states failed: %+v", proxy)
return 500, err.Error()
}
return jsonRetSucc()
}
func apiRemoveServerFromGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("removing server from group, %+v", server)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
err := serverGroup.RemoveServer(safeZkConn, server.Addr)
if err != nil {
log.ErrorErrorf(err, "remove group %d failed", groupId)
return 500, err.Error()
}
return jsonRetSucc()
}
func apiRemoveServerGroup(param martini.Params) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("removing group %s", param["id"])); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
groupId, _ := strconv.Atoi(param["id"])
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
if err := serverGroup.Remove(safeZkConn); err != nil {
log.ErrorErrorf(err, "remove server group failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func (t *MigrateTask) run() error {
log.Infof("migration start: %+v", t.MigrateTaskInfo)
to := t.NewGroupId
t.UpdateStatus(MIGRATE_TASK_MIGRATING)
err := t.migrateSingleSlot(t.SlotId, to)
if err != nil {
log.ErrorErrorf(err, "migrate single slot failed")
t.UpdateStatus(MIGRATE_TASK_ERR)
t.rollbackPremigrate()
return err
}
t.UpdateFinish()
log.Infof("migration finished: %+v", t.MigrateTaskInfo)
return nil
}
func apiGetSingleSlot(param martini.Params) (int, string) {
id, err := strconv.Atoi(param["id"])
if err != nil {
return 500, err.Error()
}
slot, err := models.GetSlot(safeZkConn, globalEnv.ProductName(), id)
if err != nil {
log.ErrorErrorf(err, "get slot %d failed", id)
return 500, err.Error()
}
b, err := json.MarshalIndent(slot, " ", " ")
return 200, string(b)
}
// for debug
func getAllProxyDebugVars() map[string]map[string]interface{} {
proxies, err := models.ProxyList(unsafeZkConn, globalEnv.ProductName(), nil)
if err != nil {
log.ErrorErrorf(err, "get proxy list failed")
return nil
}
ret := make(map[string]map[string]interface{})
for _, p := range proxies {
m, err := p.DebugVars()
if err != nil {
log.WarnErrorf(err, "get proxy debug varsfailed")
}
ret[p.Id] = m
}
return ret
}
func getAllProxyOps() int64 {
proxies, err := models.ProxyList(unsafeZkConn, globalEnv.ProductName(), nil)
if err != nil {
log.ErrorErrorf(err, "get proxy list failed")
return -1
}
var total int64
for _, p := range proxies {
i, err := p.Ops()
if err != nil {
log.WarnErrorf(err, "get proxy ops failed")
}
total += i
}
return total
}
func apiInitSlots(r *http.Request) (int, string) {
r.ParseForm()
isForce := false
val := r.FormValue("is_force")
if len(val) > 0 && (val == "1" || val == "true") {
isForce = true
}
if !isForce {
s, _ := models.Slots(safeZkConn, globalEnv.ProductName())
if len(s) > 0 {
return 500, "slots already initialized, you may use 'is_force' flag and try again."
}
}
if err := models.InitSlotSet(safeZkConn, globalEnv.ProductName(), models.DEFAULT_SLOT_NUM); err != nil {
log.ErrorErrorf(err, "init slot set failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func apiOverview() (int, string) {
// get all server groups
groups, err := models.ServerGroups(unsafeZkConn, globalEnv.ProductName())
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return 500, err.Error()
}
var instances []string
for _, group := range groups {
for _, srv := range group.Servers {
if srv.Type == "master" {
instances = append(instances, srv.Addr)
}
}
}
info := make(map[string]interface{})
info["product"] = globalEnv.ProductName()
info["ops"] = proxiesSpeed
redisInfos := make([]map[string]string, 0)
if len(instances) > 0 {
for _, instance := range instances {
info, err := utils.GetRedisStat(instance, globalEnv.Password())
if err != nil {
log.ErrorErrorf(err, "get redis stat failed")
}
redisInfos = append(redisInfos, info)
}
}
info["redis_infos"] = redisInfos
b, err := json.MarshalIndent(info, " ", " ")
return 200, string(b)
}
func cmdSlot(argv []string) (err error) {
usage := `usage:
codis-config slot init [-f]
codis-config slot info <slot_id>
codis-config slot set <slot_id> <group_id> <status>
codis-config slot range-set <slot_from> <slot_to> <group_id> <status>
codis-config slot migrate <slot_from> <slot_to> <group_id> [--delay=<delay_time_in_ms>]
codis-config slot rebalance [--delay=<delay_time_in_ms>]
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.ErrorErrorf(err, "parse args failed")
return errors.Trace(err)
}
log.Debugf("parse args = {%+v}", args)
// no need to lock here
// locked in runmigratetask
if args["migrate"].(bool) {
delay := 0
groupId, err := strconv.Atoi(args["<group_id>"].(string))
if args["--delay"] != nil {
delay, err = strconv.Atoi(args["--delay"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <group_id> failed")
return errors.Trace(err)
}
}
slotFrom, err := strconv.Atoi(args["<slot_from>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <slot_from> failed")
return errors.Trace(err)
}
slotTo, err := strconv.Atoi(args["<slot_to>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <slot_to> failed")
return errors.Trace(err)
}
return runSlotMigrate(slotFrom, slotTo, groupId, delay)
}
if args["rebalance"].(bool) {
delay := 0
if args["--delay"] != nil {
delay, err = strconv.Atoi(args["--delay"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <delay> failed")
return errors.Trace(err)
}
}
return runRebalance(delay)
}
if args["init"].(bool) {
force := args["-f"].(bool)
return runSlotInit(force)
}
if args["info"].(bool) {
slotId, err := strconv.Atoi(args["<slot_id>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <slot_id> failed")
return errors.Trace(err)
}
return runSlotInfo(slotId)
}
groupId, err := strconv.Atoi(args["<group_id>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <group_id> failed")
return errors.Trace(err)
}
if args["set"].(bool) {
slotId, err := strconv.Atoi(args["<slot_id>"].(string))
status := args["<status>"].(string)
if err != nil {
log.ErrorErrorf(err, "parse <slot_id> failed")
return errors.Trace(err)
}
return runSlotSet(slotId, groupId, status)
}
if args["range-set"].(bool) {
status := args["<status>"].(string)
slotFrom, err := strconv.Atoi(args["<slot_from>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <slot_from> failed")
return errors.Trace(err)
}
slotTo, err := strconv.Atoi(args["<slot_to>"].(string))
if err != nil {
log.ErrorErrorf(err, "parse <slot_to> failed")
return errors.Trace(err)
}
return errors.Trace(runSlotRangeSet(slotFrom, slotTo, groupId, status))
}
return nil
}
func (t *MigrateTask) migrateSingleSlot(slotId int, to int) error {
// set slot status
s, err := models.GetSlot(t.zkConn, t.productName, slotId)
if err != nil {
log.ErrorErrorf(err, "get slot info failed")
return err
}
if s.State.Status == models.SLOT_STATUS_OFFLINE {
log.Warnf("status is offline: %+v", s)
return nil
}
from := s.GroupId
if s.State.Status == models.SLOT_STATUS_MIGRATE {
from = s.State.MigrateStatus.From
}
// make sure from group & target group exists
exists, err := models.GroupExists(t.zkConn, t.productName, from)
if err != nil {
return errors.Trace(err)
}
if !exists {
log.Errorf("src group %d not exist when migrate from %d to %d", from, from, to)
return errors.Errorf("group %d not found", from)
}
exists, err = models.GroupExists(t.zkConn, t.productName, to)
if err != nil {
return errors.Trace(err)
}
if !exists {
return errors.Errorf("group %d not found", to)
}
// cannot migrate to itself, just ignore
if from == to {
log.Warnf("from == to, ignore: %+v", s)
return nil
}
// modify slot status
if err := s.SetMigrateStatus(t.zkConn, from, to); err != nil {
log.ErrorErrorf(err, "set migrate status failed")
return err
}
err = t.Migrate(s, from, to, func(p SlotMigrateProgress) {
// on migrate slot progress
if p.Remain%5000 == 0 {
log.Infof("%+v", p)
}
})
if err != nil {
log.ErrorErrorf(err, "migrate slot failed")
return err
}
// migrate done, change slot status back
s.State.Status = models.SLOT_STATUS_ONLINE
s.State.MigrateStatus.From = models.INVALID_ID
s.State.MigrateStatus.To = models.INVALID_ID
if err := s.Update(t.zkConn); err != nil {
log.ErrorErrorf(err, "update zk status failed, should be: %+v", s)
return err
}
return nil
}