func (t *MigrateTask) Run() { if t.CurrentState() == StateNew { t.SetState(StateRunning) } if t.CurrentState() == StateCancelling { t.SetState(StateCancelled) return } prev_key := "" timeout_cnt := 0 for i, r := range t.ranges { if r.Left < 0 { r.Left = 0 } if r.Right > 16383 { r.Right = 16383 } t.currRangeIndex = i t.currSlot = r.Left t.totalKeysInSlot = 0 for t.currSlot <= r.Right { t.streamPub(true) // 尽量在迁移完一个完整Slot或遇到错误时,再进行状态的转换 if t.CurrentState() == StateCancelling { t.SetState(StateCancelled) t.streamPub(false) return } // 暂停,sleep一会继续检查 if t.CurrentState() == StatePausing { t.SetState(StatePaused) } if t.CurrentState() == StatePaused { time.Sleep(100 * time.Millisecond) continue } // 正常运行 app := meta.GetAppConfig() nkeys, err, key := t.migrateSlot(t.currSlot, app.MigrateKeysEachTime) t.totalKeysInSlot += nkeys // Check remains again seed := t.SourceNode() remains, err2 := redis.CountKeysInSlot(seed.Addr(), t.currSlot) if err2 != nil { remains = -1 } if err != nil || remains > 0 { log.Warningf(t.TaskName(), "Migrate slot %d error, %d keys done, total %d keys, remains %d keys, %v", t.currSlot, nkeys, t.totalKeysInSlot, remains, err) if err != nil && strings.HasPrefix(err.Error(), "READONLY") { log.Warningf(t.TaskName(), "Migrating across slaves nodes. "+ "Maybe a manual failover just happened, "+ "if cluster marks down after this point, "+ "we need recover it by ourself using cli commands.") t.SetState(StateCancelled) goto quit } else if err != nil && strings.HasPrefix(err.Error(), "CLUSTERDOWN") { log.Warningf(t.TaskName(), "The cluster is down, please check it yourself, migrating task cancelled.") t.SetState(StateCancelled) goto quit } else if err != nil && strings.HasPrefix(err.Error(), "IOERR") { log.Warningf(t.TaskName(), "Migrating key:%s timeout", key) if timeout_cnt > 10 { log.Warningf(t.TaskName(), "Migrating key:%s timeout too frequently, task cancelled") t.SetState(StateCancelled) goto quit } if prev_key == key { timeout_cnt++ } else { timeout_cnt = 0 prev_key = key } } time.Sleep(500 * time.Millisecond) } else { log.Infof(t.TaskName(), "Migrate slot %d done, %d keys done, total %d keys, remains %d keys", t.currSlot, nkeys, t.totalKeysInSlot, remains) t.currSlot++ t.totalKeysInSlot = 0 } } } t.currSlot-- t.SetState(StateDone) quit: t.streamPub(false) }
/// 迁移slot过程: /// 1. 标记Target分片Master为IMPORTING /// 2. 标记所有Source分片节点为MIGRATING /// 3. 从Source分片Master取keys迁移,直到空,数据迁移完成 /// 4. 设置Target的Slave的slot归属到Target /// 5. 设置Target的Master的slot归属到Target /// 6. 设置Source所有节点的slot归属到Target /// 命令: /// 1. <Target Master> setslot $slot IMPORTING $sourceId /// 2. <Source Slaves> setslot $slot MIGRATING $targetId /// 3. <Source Master> setslot $slot MIGRATING $targetId /// ... migrating all keys /// 4. <Target Slaves> setslot $slot node $targetId /// 5. <Target Master> setslot $slot node $targetId /// 6. <Source Slaves> setslot $slot node $targetId /// 7. <Source Master> setslot $slot node $targetId func (t *MigrateTask) migrateSlot(slot int, keysPer int) (int, error, string) { rs := t.SourceReplicaSet() sourceNode := t.SourceNode() targetNode := t.TargetNode() err := redis.SetSlot(targetNode.Addr(), slot, redis.SLOT_IMPORTING, sourceNode.Id) if err != nil { if strings.HasPrefix(err.Error(), "ERR I'm already the owner of hash slot") { log.Warningf(t.TaskName(), "%s already the owner of hash slot %d", targetNode.Id[:6], slot) // 逻辑到此,说明Target已经包含该slot,但是Source处于Migrating状态 // 迁移实际已经完成,需要清理Source的Migrating状态 srs := t.SourceReplicaSet() err = SetSlotToNode(srs, slot, targetNode.Id) if err != nil { return 0, err, "" } err = SetSlotStable(srs, slot) if err != nil { return 0, err, "" } trs := t.TargetReplicaSet() err = SetSlotToNode(trs, slot, targetNode.Id) if err != nil { return 0, err, "" } err = SetSlotStable(trs, slot) return 0, err, "" } return 0, err, "" } // 需要将Source分片的所有节点标记为MIGRATING,最大限度避免从地域的读造成的数据不一致 for _, node := range rs.AllNodes() { err := redis.SetSlot(node.Addr(), slot, redis.SLOT_MIGRATING, targetNode.Id) if err != nil { if strings.HasPrefix(err.Error(), "ERR I'm not the owner of hash slot") { log.Warningf(t.TaskName(), "%s is not the owner of hash slot %d", sourceNode.Id, slot) srs := t.SourceReplicaSet() err = SetSlotStable(srs, slot) if err != nil { log.Warningf(t.TaskName(), "Failed to clean MIGRATING state of source server.") return 0, err, "" } trs := t.TargetReplicaSet() err = SetSlotStable(trs, slot) if err != nil { log.Warningf(t.TaskName(), "Failed to clean MIGRATING state of target server.") return 0, err, "" } return 0, fmt.Errorf("mig: %s is not the owner of hash slot %d", sourceNode.Id, slot), "" } return 0, err, "" } } nkeys := 0 app := meta.GetAppConfig() for { keys, err := redis.GetKeysInSlot(sourceNode.Addr(), slot, keysPer) if err != nil { return nkeys, err, "" } for _, key := range keys { _, err := redis.Migrate(sourceNode.Addr(), targetNode.Ip, targetNode.Port, key, app.MigrateTimeout) if err != nil { return nkeys, err, key } nkeys++ } if len(keys) == 0 { // 迁移完成,需要等SourceSlaves同步(DEL)完成,即SourceSlaves节点中该slot内已无key slaveSyncDone := true srs := t.SourceReplicaSet() for _, node := range srs.AllNodes() { nkeys, err := redis.CountKeysInSlot(node.Addr(), slot) if err != nil { return nkeys, err, "" } if nkeys > 0 { slaveSyncDone = false } } if !slaveSyncDone { // FIXME // master migrate done, slave still have some keys in slot, setslot will ensure slave clear the data log.Info(t.TaskName(), "source node not empty, setslot will clear") //return nkeys, fmt.Errorf("mig: source nodes not all empty, will retry."), "" } // 设置slot归属到新节点,该操作自动清理IMPORTING和MIGRATING状态 // 如果设置的是Source节点,设置slot归属时,Redis会确保该slot中已无剩余的key trs := t.TargetReplicaSet() // 优先设置从节点,保证当主的数据分布还未广播到从节点时主挂掉,slot信息也不会丢失 for _, node := range trs.Slaves { if node.Fail { continue } err = redis.SetSlot(node.Addr(), slot, redis.SLOT_NODE, targetNode.Id) if err != nil { return nkeys, err, "" } } // 该操作增加Epoch并广播出去 err = redis.SetSlot(trs.Master.Addr(), slot, redis.SLOT_NODE, targetNode.Id) if err != nil { return nkeys, err, "" } // 更新节点上slot的归属 for _, rs := range t.cluster.ReplicaSets() { if rs.Master.IsStandbyMaster() { continue } err = SetSlotToNode(rs, slot, targetNode.Id) if err != nil { return nkeys, err, "" } } break } } return nkeys, nil, "" }