func (top *Topology) GetSlotByIndex(i int) (*models.Slot, *models.ServerGroup, error) { var slot *models.Slot var groupServer *models.ServerGroup var err error for { slot, err = models.GetSlot(top.zkConn, top.ProductName, i) if err == nil || top.IsFatalErr(err) { break } else { time.Sleep(ZK_RECONNECT_INTERVAL * time.Second) } if err != nil { log.ErrorErrorf(err, " GetSlot ") } } if err != nil { return nil, nil, errors.Trace(err) } for { groupServer, err = models.GetGroup(top.zkConn, top.ProductName, slot.GroupId) if err == nil || top.IsFatalErr(err) { break } else { time.Sleep(ZK_RECONNECT_INTERVAL * time.Second) } if err != nil { log.ErrorErrorf(err, " GetGroup ") } } if err != nil { return nil, nil, errors.Trace(err) } return slot, groupServer, nil }
func runSlotInfo(slotId int) error { s, err := models.GetSlot(zkConn, productName, slotId) if err != nil { return errors.Trace(err) } b, _ := json.MarshalIndent(s, " ", " ") fmt.Println(string(b)) return nil }
func (t *MigrateTask) rollbackPremigrate() { if s, err := models.GetSlot(t.zkConn, t.productName, t.SlotId); err == nil && (s.State.Status == models.SLOT_STATUS_PRE_MIGRATE || s.State.Status == models.SLOT_STATUS_MIGRATE) { s.State.Status = models.SLOT_STATUS_ONLINE err = s.Update(t.zkConn) if err != nil { log.Warn("rollback premigrate failed", err) } else { log.Infof("rollback slot %d from premigrate to online\n", s.Id) } } }
func (top *Topology) GetSlotByIndex(i int) (*models.Slot, *models.ServerGroup, error) { slot, err := models.GetSlot(top.zkConn, top.ProductName, i) if err != nil { return nil, nil, errors.Trace(err) } groupServer, err := models.GetGroup(top.zkConn, top.ProductName, slot.GroupId) if err != nil { return nil, nil, errors.Trace(err) } return slot, groupServer, nil }
func apiGetSingleSlot(param martini.Params) (int, string) { id, err := strconv.Atoi(param["id"]) if err != nil { return 500, err.Error() } slot, err := models.GetSlot(safeZkConn, globalEnv.ProductName(), id) if err != nil { log.Warning(errors.Trace(err)) return 500, err.Error() } b, err := json.MarshalIndent(slot, " ", " ") return 200, string(b) }
func (top *Topology) GetSlotByIndex(i int) (*models.Slot, *models.ServerGroup, error) { slot, err := models.GetSlot(top.zkConn, top.ProductName, i) if err != nil { return nil, nil, errors.Trace(err) } log.Debugf("get slot %d : %+v", i, slot) if slot.State.Status != models.SLOT_STATUS_ONLINE && slot.State.Status != models.SLOT_STATUS_MIGRATE { log.Errorf("slot not online, %+v", slot) } groupServer, err := models.GetGroup(top.zkConn, top.ProductName, slot.GroupId) if err != nil { return nil, nil, errors.Trace(err) } return slot, groupServer, nil }
func (t *MigrateTask) migrateSingleSlot(slotId int, to int) error { // set slot status s, err := models.GetSlot(t.zkConn, t.productName, slotId) if err != nil { log.ErrorErrorf(err, "get slot info failed") return err } if s.State.Status == models.SLOT_STATUS_OFFLINE { log.Warnf("status is offline: %+v", s) return nil } from := s.GroupId if s.State.Status == models.SLOT_STATUS_MIGRATE { from = s.State.MigrateStatus.From } // make sure from group & target group exists exists, err := models.GroupExists(t.zkConn, t.productName, from) if err != nil { return errors.Trace(err) } if !exists { log.Errorf("src group %d not exist when migrate from %d to %d", from, from, to) return errors.Errorf("group %d not found", from) } exists, err = models.GroupExists(t.zkConn, t.productName, to) if err != nil { return errors.Trace(err) } if !exists { return errors.Errorf("group %d not found", to) } // cannot migrate to itself, just ignore if from == to { log.Warnf("from == to, ignore: %+v", s) return nil } // modify slot status if err := s.SetMigrateStatus(t.zkConn, from, to); err != nil { log.ErrorErrorf(err, "set migrate status failed") return err } err = t.Migrate(s, from, to, func(p SlotMigrateProgress) { // on migrate slot progress if p.Remain%5000 == 0 { log.Infof("%+v", p) } }) if err != nil { log.ErrorErrorf(err, "migrate slot failed") return err } // migrate done, change slot status back s.State.Status = models.SLOT_STATUS_ONLINE s.State.MigrateStatus.From = models.INVALID_ID s.State.MigrateStatus.To = models.INVALID_ID if err := s.Update(t.zkConn); err != nil { log.ErrorErrorf(err, "update zk status failed, should be: %+v", s) return err } return nil }
func (t *MigrateTask) migrateSingleSlot(slotId int, to int) error { // set slot status s, err := models.GetSlot(t.zkConn, t.productName, slotId) if err != nil { log.Error(err) return err } if s.State.Status != models.SLOT_STATUS_ONLINE && s.State.Status != models.SLOT_STATUS_MIGRATE { log.Warning("status is not online && migrate", s) return nil } from := s.GroupId if s.State.Status == models.SLOT_STATUS_MIGRATE { from = s.State.MigrateStatus.From } // make sure from group & target group exists exists, err := models.GroupExists(t.zkConn, t.productName, from) if err != nil { return errors.Trace(err) } if !exists { log.Errorf("src group %d not exist when migrate from %d to %d", from, from, to) return errors.NotFoundf("group %d", from) } exists, err = models.GroupExists(t.zkConn, t.productName, to) if err != nil { return errors.Trace(err) } if !exists { return errors.NotFoundf("group %d", to) } // cannot migrate to itself, just ignore if from == to { log.Warning("from == to, ignore", s) return nil } // modify slot status if err := s.SetMigrateStatus(t.zkConn, from, to); err != nil { log.Error(err) return err } err = t.slotMigrator.Migrate(s, from, to, t, func(p SlotMigrateProgress) { // on migrate slot progress if p.Remain%500 == 0 { log.Info(p) } }) if err != nil { log.Error(err) return err } // migrate done, change slot status back s.State.Status = models.SLOT_STATUS_ONLINE s.State.MigrateStatus.From = models.INVALID_ID s.State.MigrateStatus.To = models.INVALID_ID if err := s.Update(t.zkConn); err != nil { log.Error(err) return err } return nil }
// migrate multi slots func RunMigrateTask(task *MigrateTask) error { conn := CreateZkConn() defer conn.Close() lock := utils.GetZkLock(conn, productName) to := task.NewGroupId task.Status = MIGRATE_TASK_MIGRATING for slotId := task.FromSlot; slotId <= task.ToSlot; slotId++ { err := func() error { log.Info("start migrate slot:", slotId) lock.Lock(fmt.Sprintf("migrate %d", slotId)) defer func() { err := lock.Unlock() if err != nil { log.Info(err) } }() s, err := models.GetSlot(conn, productName, slotId) if err != nil { log.Error(err) return err } if s.State.Status != models.SLOT_STATUS_ONLINE && s.State.Status != models.SLOT_STATUS_MIGRATE { log.Warning("status is not online && migrate", s) return nil } from := s.GroupId if s.State.Status == models.SLOT_STATUS_MIGRATE { from = s.State.MigrateStatus.From } if from == to { log.Warning("from == to, ignore", s) return nil } // modify slot status if err := s.SetMigrateStatus(conn, from, to); err != nil { log.Error(err) return err } // do real migrate err = MigrateSingleSlot(conn, slotId, from, to, task.Delay, task.stopChan) if err != nil { log.Error(err) return err } // migrate done, change slot status back s.State.Status = models.SLOT_STATUS_ONLINE s.State.MigrateStatus.From = models.INVALID_ID s.State.MigrateStatus.To = models.INVALID_ID if err := s.Update(zkConn); err != nil { log.Error(err) return err } return nil }() if err == ErrStopMigrateByUser { log.Info("stop migration job by user") break } else if err != nil { task.Status = MIGRATE_TASK_ERR return err } task.Percent = (slotId - task.FromSlot + 1) * 100 / (task.ToSlot - task.FromSlot + 1) log.Info("total percent:", task.Percent) } task.Status = MIGRATE_TASK_FINISHED log.Info("migration finished") return nil }