// add redis server to exist server group
func apiAddServerToGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("add server to group, %+v", server)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
// check group exists first
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
exists, err := serverGroup.Exists(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "check group exits failed")
return 500, err.Error()
}
// create new group if not exists
if !exists {
if err := serverGroup.Create(safeZkConn); err != nil {
return 500, err.Error()
}
}
if err := serverGroup.AddServer(safeZkConn, &server, globalEnv.Password()); err != nil {
log.ErrorErrorf(err, "add server to group failed")
return 500, err.Error()
}
return jsonRetSucc()
}
// create new server group
func apiAddServerGroup(newGroup models.ServerGroup) (int, string) {
conn := CreateZkConn()
defer conn.Close()
lock := utils.GetZkLock(conn, productName)
lock.Lock(fmt.Sprintf("add group %+v", newGroup))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
newGroup.ProductName = productName
exists, err := newGroup.Exists(conn)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
if exists {
return jsonRet(map[string]interface{}{
"ret": 0,
"msg": "group already exists",
})
}
err = newGroup.Create(conn)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
return jsonRetSucc()
}
func TestForceRemoveLock(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
zkLock := utils.GetZkLock(fakeZkConn, productName)
if zkLock == nil {
t.Error("create lock error")
}
zkLock.Lock("force remove lock")
zkPath := fmt.Sprintf("/zk/codis/db_%s/LOCK", productName)
children, _, err := fakeZkConn.Children(zkPath)
if err != nil {
t.Error(err)
}
if len(children) == 0 {
t.Error("create lock error")
}
ForceRemoveLock(fakeZkConn, productName)
children, _, err = fakeZkConn.Children(zkPath)
if err != nil {
t.Error(err)
}
if len(children) != 0 {
t.Error("remove lock error")
}
}
// add redis server to exist server group
func apiAddServerToGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
conn := CreateZkConn()
defer conn.Close()
lock := utils.GetZkLock(conn, productName)
lock.Lock(fmt.Sprintf("add server to group, %+v", server))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
// check group exists first
serverGroup := models.NewServerGroup(productName, groupId)
exists, err := serverGroup.Exists(conn)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
if !exists {
return jsonRetFail(-1, "group not exists")
}
if err := serverGroup.AddServer(conn, &server); err != nil {
log.Warning(err)
return 500, err.Error()
}
return jsonRetSucc()
}
// create new server group
func apiAddServerGroup(newGroup models.ServerGroup) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("add group %+v", newGroup))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
newGroup.ProductName = globalEnv.ProductName()
exists, err := newGroup.Exists(safeZkConn)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
if exists {
return 500, "group already exists"
}
err = newGroup.Create(safeZkConn)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
return jsonRetSucc()
}
// create new server group
func apiAddServerGroup(newGroup models.ServerGroup) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("add group %+v", newGroup)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
newGroup.ProductName = globalEnv.ProductName()
exists, err := newGroup.Exists(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "check group exits failed")
return 500, err.Error()
}
if exists {
return 500, "group already exists"
}
err = newGroup.Create(safeZkConn)
if err != nil {
log.ErrorErrorf(err, "create node for new group failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func apiSlotRangeSet(task RangeSetTask) (int, string) {
conn := CreateZkConn()
defer conn.Close()
lock := utils.GetZkLock(conn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("set slot range, %+v", task))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
// default set online
if len(task.Status) == 0 {
task.Status = string(models.SLOT_STATUS_ONLINE)
}
err := models.SetSlotRange(conn, globalEnv.ProductName(), task.FromSlot, task.ToSlot, task.NewGroupId, models.SlotStatus(task.Status))
if err != nil {
log.Warning(err)
return 500, err.Error()
}
return jsonRetSucc()
}
// actions
func apiActionGC(r *http.Request) (int, string) {
r.ParseForm()
keep, _ := strconv.Atoi(r.FormValue("keep"))
secs, _ := strconv.Atoi(r.FormValue("secs"))
conn := CreateZkConn()
defer conn.Close()
lock := utils.GetZkLock(conn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("action gc"))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
var err error
if keep >= 0 {
err = models.ActionGC(conn, globalEnv.ProductName(), models.GC_TYPE_N, keep)
} else if secs > 0 {
err = models.ActionGC(conn, globalEnv.ProductName(), models.GC_TYPE_SEC, secs)
}
if err != nil {
return 500, err.Error()
}
return jsonRetSucc()
}
func apiPromoteServer(server models.Server, param martini.Params) (int, string) {
conn := CreateZkConn()
defer conn.Close()
lock := utils.GetZkLock(conn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("promote server %+v", server))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
group, err := models.GetGroup(conn, globalEnv.ProductName(), server.GroupId)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
err = group.Promote(conn, server.Addr)
if err != nil {
log.Warning(errors.ErrorStack(err))
log.Warning(err)
return 500, err.Error()
}
return jsonRetSucc()
}
// add redis server to exist server group
func apiAddServerToGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("add server to group, %+v", server))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
// check group exists first
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
exists, err := serverGroup.Exists(safeZkConn)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
// create new group if not exists
if !exists {
if err := serverGroup.Create(safeZkConn); err != nil {
return 500, err.Error()
}
}
if err := serverGroup.AddServer(safeZkConn, &server); err != nil {
log.Warning(errors.ErrorStack(err))
return 500, err.Error()
}
return jsonRetSucc()
}
// actions
func apiActionGC(r *http.Request) (int, string) {
r.ParseForm()
keep, _ := strconv.Atoi(r.FormValue("keep"))
secs, _ := strconv.Atoi(r.FormValue("secs"))
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("action gc")); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
var err error
if keep >= 0 {
err = models.ActionGC(safeZkConn, globalEnv.ProductName(), models.GC_TYPE_N, keep)
} else if secs > 0 {
err = models.ActionGC(safeZkConn, globalEnv.ProductName(), models.GC_TYPE_SEC, secs)
}
if err != nil {
return 500, err.Error()
}
return jsonRetSucc()
}
func apiSlotRangeSet(task RangeSetTask) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("set slot range, %+v", task)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
// default set online
if len(task.Status) == 0 {
task.Status = string(models.SLOT_STATUS_ONLINE)
}
err := models.SetSlotRange(safeZkConn, globalEnv.ProductName(), task.FromSlot, task.ToSlot, task.NewGroupId, models.SlotStatus(task.Status))
if err != nil {
log.ErrorErrorf(err, "set slot range [%d,%d] failed", task.FromSlot, task.ToSlot)
return 500, err.Error()
}
return jsonRetSucc()
}
func apiPromoteServer(server models.Server, param martini.Params) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("promote server %+v", server)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
group, err := models.GetGroup(safeZkConn, globalEnv.ProductName(), server.GroupId)
if err != nil {
log.ErrorErrorf(err, "get group %d failed", server.GroupId)
return 500, err.Error()
}
err = group.Promote(safeZkConn, server.Addr, globalEnv.Password())
if err != nil {
log.ErrorErrorf(err, "promote group %d failed", server.GroupId)
return 500, err.Error()
}
return jsonRetSucc()
}
func TestForceRemoveLock(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
zkLock := utils.GetZkLock(fakeZkConn, productName)
assert.Must(zkLock != nil)
zkLock.Lock("force remove lock")
zkPath := fmt.Sprintf("/zk/codis/db_%s/LOCK", productName)
children, _, err := fakeZkConn.Children(zkPath)
assert.MustNoError(err)
assert.Must(len(children) != 0)
ForceRemoveLock(fakeZkConn, productName)
children, _, err = fakeZkConn.Children(zkPath)
assert.MustNoError(err)
assert.Must(len(children) == 0)
}
func apiRemoveServerFromGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("removing server from group, %+v", server))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
err := serverGroup.RemoveServer(safeZkConn, server.Addr)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
return jsonRetSucc()
}
func apiRemoveServerGroup(param martini.Params) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("removing group %s", param["id"]))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
groupId, _ := strconv.Atoi(param["id"])
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
if err := serverGroup.Remove(safeZkConn); err != nil {
log.Error(errors.ErrorStack(err))
return 500, err.Error()
}
return jsonRetSucc()
}
func apiSlotRangeSet(task RangeSetTask) (int, string) {
conn := CreateZkConn()
defer conn.Close()
lock := utils.GetZkLock(conn, productName)
lock.Lock(fmt.Sprintf("set slot range, %+v", task))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
err := models.SetSlotRange(conn, productName, task.FromSlot, task.ToSlot, task.NewGroupId, models.SLOT_STATUS_ONLINE)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
return jsonRetSucc()
}
func apiRemoveServerFromGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("removing server from group, %+v", server)); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
err := serverGroup.RemoveServer(safeZkConn, server.Addr)
if err != nil {
log.ErrorErrorf(err, "remove group %d failed", groupId)
return 500, err.Error()
}
return jsonRetSucc()
}
func apiRemoveServerGroup(param martini.Params) (int, string) {
lock := utils.GetZkLock(safeZkConn, globalEnv.ProductName())
if err := lock.LockWithTimeout(0, fmt.Sprintf("removing group %s", param["id"])); err != nil {
return 500, err.Error()
}
defer func() {
err := lock.Unlock()
if err != nil && err != zk.ErrNoNode {
log.ErrorErrorf(err, "unlock node failed")
}
}()
groupId, _ := strconv.Atoi(param["id"])
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
if err := serverGroup.Remove(safeZkConn); err != nil {
log.ErrorErrorf(err, "remove server group failed")
return 500, err.Error()
}
return jsonRetSucc()
}
func apiRemoveServerFromGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
conn := CreateZkConn()
defer conn.Close()
lock := utils.GetZkLock(conn, productName)
lock.Lock(fmt.Sprintf("remove server from group, %+v", server))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
serverGroup := models.NewServerGroup(productName, groupId)
err := serverGroup.RemoveServer(conn, server)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
return jsonRetSucc()
}
func main() {
log.SetLevelByString("info")
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
signal.Notify(c, syscall.SIGTERM)
go func() {
<-c
Fatal("ctrl-c or SIGTERM found, exit")
}()
// productName, _ = config.ReadString("product", "test")
args, err := docopt.Parse(usage, nil, true, "codis config v0.1", true)
if err != nil {
log.Error(err)
}
// set config file
if args["-c"] != nil {
configFile = args["-c"].(string)
config, err = utils.InitConfigFromFile(configFile)
if err != nil {
Fatal(err)
}
} else {
config, err = utils.InitConfig()
if err != nil {
Fatal(err)
}
}
// set output log file
if args["-L"] != nil {
log.SetOutputByName(args["-L"].(string))
}
// set log level
if args["--log-level"] != nil {
log.SetLevelByString(args["--log-level"].(string))
}
productName, _ = config.ReadString("product", "test")
zkAddr, _ = config.ReadString("zk", "localhost:2181")
zkConn, _ = zkhelper.ConnectToZk(zkAddr)
zkLock = utils.GetZkLock(zkConn, productName)
log.Debugf("product: %s", productName)
log.Debugf("zk: %s", zkAddr)
if err := registerConfigNode(); err != nil {
log.Fatal(errors.ErrorStack(err))
}
defer unRegisterConfigNode()
if err := removeOrphanLocks(); err != nil {
log.Fatal(errors.ErrorStack(err))
}
cmd := args["<command>"].(string)
cmdArgs := args["<args>"].([]string)
go http.ListenAndServe(":10086", nil)
err = runCommand(cmd, cmdArgs)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
}
// migrate multi slots
func RunMigrateTask(task *MigrateTask) error {
conn := CreateZkConn()
defer conn.Close()
lock := utils.GetZkLock(conn, productName)
to := task.NewGroupId
task.Status = MIGRATE_TASK_MIGRATING
for slotId := task.FromSlot; slotId <= task.ToSlot; slotId++ {
err := func() error {
log.Info("start migrate slot:", slotId)
lock.Lock(fmt.Sprintf("migrate %d", slotId))
defer func() {
err := lock.Unlock()
if err != nil {
log.Info(err)
}
}()
s, err := models.GetSlot(conn, productName, slotId)
if err != nil {
log.Error(err)
return err
}
if s.State.Status != models.SLOT_STATUS_ONLINE && s.State.Status != models.SLOT_STATUS_MIGRATE {
log.Warning("status is not online && migrate", s)
return nil
}
from := s.GroupId
if s.State.Status == models.SLOT_STATUS_MIGRATE {
from = s.State.MigrateStatus.From
}
if from == to {
log.Warning("from == to, ignore", s)
return nil
}
// modify slot status
if err := s.SetMigrateStatus(conn, from, to); err != nil {
log.Error(err)
return err
}
// do real migrate
err = MigrateSingleSlot(conn, slotId, from, to, task.Delay, task.stopChan)
if err != nil {
log.Error(err)
return err
}
// migrate done, change slot status back
s.State.Status = models.SLOT_STATUS_ONLINE
s.State.MigrateStatus.From = models.INVALID_ID
s.State.MigrateStatus.To = models.INVALID_ID
if err := s.Update(zkConn); err != nil {
log.Error(err)
return err
}
return nil
}()
if err == ErrStopMigrateByUser {
log.Info("stop migration job by user")
break
} else if err != nil {
task.Status = MIGRATE_TASK_ERR
return err
}
task.Percent = (slotId - task.FromSlot + 1) * 100 / (task.ToSlot - task.FromSlot + 1)
log.Info("total percent:", task.Percent)
}
task.Status = MIGRATE_TASK_FINISHED
log.Info("migration finished")
return nil
}