func TestForceRemoveLock(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
zkLock := utils.GetZkLock(fakeZkConn, productName)
if zkLock == nil {
t.Error("create lock error")
}
zkLock.Lock("force remove lock")
zkPath := fmt.Sprintf("/zk/codis/db_%s/LOCK", productName)
children, _, err := fakeZkConn.Children(zkPath)
if err != nil {
t.Error(err)
}
if len(children) == 0 {
t.Error("create lock error")
}
ForceRemoveLock(fakeZkConn, productName)
children, _, err = fakeZkConn.Children(zkPath)
if err != nil {
t.Error(err)
}
if len(children) != 0 {
t.Error("remove lock error")
}
}
func Promote(oldMaster string, newMaster string) error {
conn, _ := zkhelper.ConnectToZk(*zkAddr)
defer conn.Close()
groups, err := models.ServerGroups(conn, *productName)
if err != nil {
return err
}
var groupId int
found := false
for _, group := range groups {
for _, server := range group.Servers {
if server.Addr == oldMaster {
groupId = group.Id
found = true
break
}
}
}
if !found {
return fmt.Errorf("can not find %s in any groups", oldMaster)
}
lock := utils.GetZkLock(conn, *productName)
lock.Lock(fmt.Sprintf("promote server %+v", newMaster))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
group, err := models.GetGroup(conn, *productName, groupId)
if err != nil {
return err
}
err = group.Promote(conn, newMaster)
if err != nil {
return err
}
return nil
}
func main() {
log.SetLevelByString("info")
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
signal.Notify(c, syscall.SIGTERM)
go func() {
<-c
Fatal("ctrl-c or SIGTERM found, exit")
}()
// productName, _ = config.ReadString("product", "test")
args, err := docopt.Parse(usage, nil, true, "codis config v0.1", true)
if err != nil {
log.Error(err)
}
// set config file
if args["-c"] != nil {
configFile = args["-c"].(string)
config, err = utils.InitConfigFromFile(configFile)
if err != nil {
Fatal(err)
}
} else {
config, err = utils.InitConfig()
if err != nil {
Fatal(err)
}
}
// set output log file
if args["-L"] != nil {
log.SetOutputByName(args["-L"].(string))
}
// set log level
if args["--log-level"] != nil {
log.SetLevelByString(args["--log-level"].(string))
}
productName, _ = config.ReadString("product", "test")
zkAddr, _ = config.ReadString("zk", "localhost:2181")
zkConn, _ = zkhelper.ConnectToZk(zkAddr)
zkLock = utils.GetZkLock(zkConn, productName)
broker, _ = config.ReadString("broker", "ledisdb")
slot_num, _ = config.ReadInt("slot_num", 16)
log.Debugf("product: %s", productName)
log.Debugf("zk: %s", zkAddr)
log.Debugf("broker: %s", broker)
if err := registerConfigNode(); err != nil {
log.Fatal(errors.ErrorStack(err))
}
defer unRegisterConfigNode()
if err := removeOrphanLocks(); err != nil {
log.Fatal(errors.ErrorStack(err))
}
cmd := args["<command>"].(string)
cmdArgs := args["<args>"].([]string)
go http.ListenAndServe(":10086", nil)
err = runCommand(cmd, cmdArgs)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
}
// migrate multi slots
func RunMigrateTask(task *MigrateTask) error {
conn := CreateZkConn()
defer conn.Close()
lock := utils.GetZkLock(conn, productName)
to := task.NewGroupId
task.Status = MIGRATE_TASK_MIGRATING
for slotId := task.FromSlot; slotId <= task.ToSlot; slotId++ {
err := func() error {
log.Info("start migrate slot:", slotId)
lock.Lock(fmt.Sprintf("migrate %d", slotId))
defer func() {
err := lock.Unlock()
if err != nil {
log.Info(err)
}
}()
// set slot status
s, err := models.GetSlot(conn, productName, slotId)
if err != nil {
log.Error(err)
return err
}
if s.State.Status != models.SLOT_STATUS_ONLINE && s.State.Status != models.SLOT_STATUS_MIGRATE {
log.Warning("status is not online && migrate", s)
return nil
}
from := s.GroupId
if s.State.Status == models.SLOT_STATUS_MIGRATE {
from = s.State.MigrateStatus.From
}
// make sure from group & target group exists
exists, err := models.GroupExists(conn, productName, from)
if err != nil {
return errors.Trace(err)
}
if !exists {
log.Errorf("src group %d not exist when migrate from %d to %d", from, from, to)
return errors.NotFoundf("group %d", from)
}
exists, err = models.GroupExists(conn, productName, to)
if err != nil {
return errors.Trace(err)
}
if !exists {
return errors.NotFoundf("group %d", to)
}
// cannot migrate to itself
if from == to {
log.Warning("from == to, ignore", s)
return nil
}
// modify slot status
if err := s.SetMigrateStatus(conn, from, to); err != nil {
log.Error(err)
return err
}
// do real migrate
err = MigrateSingleSlot(conn, slotId, from, to, task.Delay, task.stopChan)
if err != nil {
log.Error(err)
return err
}
// migrate done, change slot status back
s.State.Status = models.SLOT_STATUS_ONLINE
s.State.MigrateStatus.From = models.INVALID_ID
s.State.MigrateStatus.To = models.INVALID_ID
if err := s.Update(zkConn); err != nil {
log.Error(err)
return err
}
return nil
}()
if err == ErrStopMigrateByUser {
log.Info("stop migration job by user")
break
} else if err != nil {
log.Error(err)
task.Status = MIGRATE_TASK_ERR
return err
}
task.Percent = (slotId - task.FromSlot + 1) * 100 / (task.ToSlot - task.FromSlot + 1)
log.Info("total percent:", task.Percent)
}
task.Status = MIGRATE_TASK_FINISHED
log.Info("migration finished")
return nil
}