func (top *Topology) GetSlotByIndex(i int) (*models.Slot, *models.ServerGroup, error) {
var slot *models.Slot
var groupServer *models.ServerGroup
var err error
for {
slot, err = models.GetSlot(top.zkConn, top.ProductName, i)
if err == nil || top.IsFatalErr(err) {
break
} else {
time.Sleep(ZK_RECONNECT_INTERVAL * time.Second)
}
if err != nil {
log.ErrorErrorf(err, " GetSlot ")
}
}
if err != nil {
return nil, nil, errors.Trace(err)
}
for {
groupServer, err = models.GetGroup(top.zkConn, top.ProductName, slot.GroupId)
if err == nil || top.IsFatalErr(err) {
break
} else {
time.Sleep(ZK_RECONNECT_INTERVAL * time.Second)
}
if err != nil {
log.ErrorErrorf(err, " GetGroup ")
}
}
if err != nil {
return nil, nil, errors.Trace(err)
}
return slot, groupServer, nil
}
func runSlotInfo(slotId int) error {
s, err := models.GetSlot(zkConn, productName, slotId)
if err != nil {
return errors.Trace(err)
}
b, _ := json.MarshalIndent(s, " ", " ")
fmt.Println(string(b))
return nil
}
func (t *MigrateTask) rollbackPremigrate() {
if s, err := models.GetSlot(t.zkConn, t.productName, t.SlotId); err == nil && (s.State.Status == models.SLOT_STATUS_PRE_MIGRATE || s.State.Status == models.SLOT_STATUS_MIGRATE) {
s.State.Status = models.SLOT_STATUS_ONLINE
err = s.Update(t.zkConn)
if err != nil {
log.Warn("rollback premigrate failed", err)
} else {
log.Infof("rollback slot %d from premigrate to online\n", s.Id)
}
}
}
func (top *Topology) GetSlotByIndex(i int) (*models.Slot, *models.ServerGroup, error) {
slot, err := models.GetSlot(top.zkConn, top.ProductName, i)
if err != nil {
return nil, nil, errors.Trace(err)
}
groupServer, err := models.GetGroup(top.zkConn, top.ProductName, slot.GroupId)
if err != nil {
return nil, nil, errors.Trace(err)
}
return slot, groupServer, nil
}
func apiGetSingleSlot(param martini.Params) (int, string) {
id, err := strconv.Atoi(param["id"])
if err != nil {
return 500, err.Error()
}
slot, err := models.GetSlot(safeZkConn, globalEnv.ProductName(), id)
if err != nil {
log.Warning(errors.Trace(err))
return 500, err.Error()
}
b, err := json.MarshalIndent(slot, " ", " ")
return 200, string(b)
}
func (top *Topology) GetSlotByIndex(i int) (*models.Slot, *models.ServerGroup, error) {
slot, err := models.GetSlot(top.zkConn, top.ProductName, i)
if err != nil {
return nil, nil, errors.Trace(err)
}
log.Debugf("get slot %d : %+v", i, slot)
if slot.State.Status != models.SLOT_STATUS_ONLINE && slot.State.Status != models.SLOT_STATUS_MIGRATE {
log.Errorf("slot not online, %+v", slot)
}
groupServer, err := models.GetGroup(top.zkConn, top.ProductName, slot.GroupId)
if err != nil {
return nil, nil, errors.Trace(err)
}
return slot, groupServer, nil
}
func (t *MigrateTask) migrateSingleSlot(slotId int, to int) error {
// set slot status
s, err := models.GetSlot(t.zkConn, t.productName, slotId)
if err != nil {
log.ErrorErrorf(err, "get slot info failed")
return err
}
if s.State.Status == models.SLOT_STATUS_OFFLINE {
log.Warnf("status is offline: %+v", s)
return nil
}
from := s.GroupId
if s.State.Status == models.SLOT_STATUS_MIGRATE {
from = s.State.MigrateStatus.From
}
// make sure from group & target group exists
exists, err := models.GroupExists(t.zkConn, t.productName, from)
if err != nil {
return errors.Trace(err)
}
if !exists {
log.Errorf("src group %d not exist when migrate from %d to %d", from, from, to)
return errors.Errorf("group %d not found", from)
}
exists, err = models.GroupExists(t.zkConn, t.productName, to)
if err != nil {
return errors.Trace(err)
}
if !exists {
return errors.Errorf("group %d not found", to)
}
// cannot migrate to itself, just ignore
if from == to {
log.Warnf("from == to, ignore: %+v", s)
return nil
}
// modify slot status
if err := s.SetMigrateStatus(t.zkConn, from, to); err != nil {
log.ErrorErrorf(err, "set migrate status failed")
return err
}
err = t.Migrate(s, from, to, func(p SlotMigrateProgress) {
// on migrate slot progress
if p.Remain%5000 == 0 {
log.Infof("%+v", p)
}
})
if err != nil {
log.ErrorErrorf(err, "migrate slot failed")
return err
}
// migrate done, change slot status back
s.State.Status = models.SLOT_STATUS_ONLINE
s.State.MigrateStatus.From = models.INVALID_ID
s.State.MigrateStatus.To = models.INVALID_ID
if err := s.Update(t.zkConn); err != nil {
log.ErrorErrorf(err, "update zk status failed, should be: %+v", s)
return err
}
return nil
}
func (t *MigrateTask) migrateSingleSlot(slotId int, to int) error {
// set slot status
s, err := models.GetSlot(t.zkConn, t.productName, slotId)
if err != nil {
log.Error(err)
return err
}
if s.State.Status != models.SLOT_STATUS_ONLINE && s.State.Status != models.SLOT_STATUS_MIGRATE {
log.Warning("status is not online && migrate", s)
return nil
}
from := s.GroupId
if s.State.Status == models.SLOT_STATUS_MIGRATE {
from = s.State.MigrateStatus.From
}
// make sure from group & target group exists
exists, err := models.GroupExists(t.zkConn, t.productName, from)
if err != nil {
return errors.Trace(err)
}
if !exists {
log.Errorf("src group %d not exist when migrate from %d to %d", from, from, to)
return errors.NotFoundf("group %d", from)
}
exists, err = models.GroupExists(t.zkConn, t.productName, to)
if err != nil {
return errors.Trace(err)
}
if !exists {
return errors.NotFoundf("group %d", to)
}
// cannot migrate to itself, just ignore
if from == to {
log.Warning("from == to, ignore", s)
return nil
}
// modify slot status
if err := s.SetMigrateStatus(t.zkConn, from, to); err != nil {
log.Error(err)
return err
}
err = t.slotMigrator.Migrate(s, from, to, t, func(p SlotMigrateProgress) {
// on migrate slot progress
if p.Remain%500 == 0 {
log.Info(p)
}
})
if err != nil {
log.Error(err)
return err
}
// migrate done, change slot status back
s.State.Status = models.SLOT_STATUS_ONLINE
s.State.MigrateStatus.From = models.INVALID_ID
s.State.MigrateStatus.To = models.INVALID_ID
if err := s.Update(t.zkConn); err != nil {
log.Error(err)
return err
}
return nil
}
// migrate multi slots
func RunMigrateTask(task *MigrateTask) error {
conn := CreateZkConn()
defer conn.Close()
lock := utils.GetZkLock(conn, productName)
to := task.NewGroupId
task.Status = MIGRATE_TASK_MIGRATING
for slotId := task.FromSlot; slotId <= task.ToSlot; slotId++ {
err := func() error {
log.Info("start migrate slot:", slotId)
lock.Lock(fmt.Sprintf("migrate %d", slotId))
defer func() {
err := lock.Unlock()
if err != nil {
log.Info(err)
}
}()
s, err := models.GetSlot(conn, productName, slotId)
if err != nil {
log.Error(err)
return err
}
if s.State.Status != models.SLOT_STATUS_ONLINE && s.State.Status != models.SLOT_STATUS_MIGRATE {
log.Warning("status is not online && migrate", s)
return nil
}
from := s.GroupId
if s.State.Status == models.SLOT_STATUS_MIGRATE {
from = s.State.MigrateStatus.From
}
if from == to {
log.Warning("from == to, ignore", s)
return nil
}
// modify slot status
if err := s.SetMigrateStatus(conn, from, to); err != nil {
log.Error(err)
return err
}
// do real migrate
err = MigrateSingleSlot(conn, slotId, from, to, task.Delay, task.stopChan)
if err != nil {
log.Error(err)
return err
}
// migrate done, change slot status back
s.State.Status = models.SLOT_STATUS_ONLINE
s.State.MigrateStatus.From = models.INVALID_ID
s.State.MigrateStatus.To = models.INVALID_ID
if err := s.Update(zkConn); err != nil {
log.Error(err)
return err
}
return nil
}()
if err == ErrStopMigrateByUser {
log.Info("stop migration job by user")
break
} else if err != nil {
task.Status = MIGRATE_TASK_ERR
return err
}
task.Percent = (slotId - task.FromSlot + 1) * 100 / (task.ToSlot - task.FromSlot + 1)
log.Info("total percent:", task.Percent)
}
task.Status = MIGRATE_TASK_FINISHED
log.Info("migration finished")
return nil
}