func (m *MigrateManager) CreateTask(sourceId, targetId string, ranges []topo.Range, cluster *topo.Cluster) (*MigrateTask, error) { sourceRS := cluster.FindReplicaSetByNode(sourceId) targetRS := cluster.FindReplicaSetByNode(targetId) if sourceRS == nil || targetRS == nil { return nil, ErrReplicatSetNotFound } task := NewMigrateTask(cluster, sourceRS, targetRS, ranges) err := m.AddTask(task) if err != nil { return nil, err } return task, nil }
func (m *MigrateManager) HandleNodeStateChange(cluster *topo.Cluster) { // 如果存在迁移任务,先跳过,等结束后再处理 if len(m.tasks) > 0 { goto done } // 处理主节点的迁移任务重建 for _, node := range cluster.AllNodes() { if node.Fail { continue } // Wait a while if time.Now().Sub(m.lastTaskEndTime) < 5*time.Second { continue } for id, slots := range node.Migrating { // 根据slot生成ranges ranges := []topo.Range{} for _, slot := range slots { // 如果是自己 if id == node.Id { redis.SetSlot(node.Addr(), slot, redis.SLOT_STABLE, "") } else { ranges = append(ranges, topo.Range{Left: slot, Right: slot}) } } // Source source := node if !node.IsMaster() { srs := cluster.FindReplicaSetByNode(node.Id) if srs != nil { source = srs.Master } } // Target rs := cluster.FindReplicaSetByNode(id) if source.Fail || rs.Master.Fail { continue } _, err := m.CreateTask(source.Id, rs.Master.Id, ranges, cluster) if err != nil { log.Warningf(node.Addr(), "Can not recover migrate task, %v", err) } else { log.Warningf(node.Addr(), "Will recover migrating task for node %s(%s) with MIGRATING info"+ ", Task(Source:%s, Target:%s).", node.Id, node.Addr(), source.Addr(), rs.Master.Addr()) goto done } } for id, slots := range node.Importing { // 根据slot生成ranges ranges := []topo.Range{} for _, slot := range slots { // 如果是自己 if id == node.Id { redis.SetSlot(node.Addr(), slot, redis.SLOT_STABLE, "") } else { ranges = append(ranges, topo.Range{Left: slot, Right: slot}) } } // Target target := node if !node.IsMaster() { trs := cluster.FindReplicaSetByNode(node.Id) if trs != nil { target = trs.Master } } if target.IsStandbyMaster() { s := cluster.FindNodeBySlot(ranges[0].Left) if s != nil { log.Warningf(node.Addr(), "Reset migrate task target to %s(%s)", s.Id, s.Addr()) target = s } } // Source rs := cluster.FindReplicaSetByNode(id) if target.Fail || rs.Master.Fail { continue } _, err := m.CreateTask(rs.Master.Id, target.Id, ranges, cluster) if err != nil { log.Warningf(node.Addr(), "Can not recover migrate task, %v", err) } else { log.Warningf(node.Addr(), "Will recover migrating task for node %s(%s) with IMPORTING info"+ ", Task(Source:%s,Target:%s).", node.Id, node.Addr(), rs.Master.Addr(), target.Addr()) goto done } } } done: for _, task := range m.tasks { if task.CurrentState() != StateDone { m.handleTaskChange(task, cluster) } } }
// 更新任务状态机 func (m *MigrateManager) handleTaskChange(task *MigrateTask, cluster *topo.Cluster) error { fromNode := cluster.FindNode(task.SourceNode().Id) toNode := cluster.FindNode(task.TargetNode().Id) tname := task.TaskName() if fromNode == nil { log.Infof(tname, "Source node %s(%s) not exist", fromNode.Addr(), fromNode.Id) return ErrNodeNotFound } if toNode == nil { log.Infof(tname, "Target node %s(%s) not exist", toNode.Addr(), toNode.Id) return ErrNodeNotFound } // 角色变化说明该分片进行了主从切换 if !fromNode.IsMaster() || !toNode.IsMaster() { log.Warningf(tname, "%s role change, cancel migration task %s\n", fromNode.Id[:6], task.TaskName()) task.SetState(StateCancelling) return ErrSourceNodeFail } // 如果是源节点挂了,直接取消,等待主从切换之后重建任务 if fromNode.Fail { log.Infof(tname, "Cancel migration task %s\n", task.TaskName()) task.SetState(StateCancelling) return ErrSourceNodeFail } // 如果目标节点挂了,需要记录当前的ReplicaSet,观察等待主从切换 if toNode.Fail { if task.CurrentState() == StateRunning { task.SetState(StateTargetNodeFailure) task.SetBackupReplicaSet(task.TargetReplicaSet()) return ErrTargetNodeFail } } else if task.CurrentState() != StateNew { task.SetState(StateRunning) task.SetBackupReplicaSet(nil) } // 如果目标节点已经进行了Failover(重新选主),我们需要找到对应的新主 // 方法是从BackupReplicaSet里取一个从来查找 if toNode.IsStandbyMaster() { brs := task.BackupReplicaSet() if brs == nil { task.SetState(StateCancelling) log.Info(tname, "No backup replicaset found, controller maybe restarted after target master failure, can not do recovery.") return ErrCanNotRecover } slaves := brs.Slaves if len(slaves) == 0 { task.SetState(StateCancelling) log.Info(tname, "The dead target master has no slave, cannot do recovery.") return ErrCanNotRecover } else { rs := cluster.FindReplicaSetByNode(slaves[0].Id) if rs == nil { task.SetState(StateCancelling) log.Info(tname, "No replicaset for slave of dead target master found") return ErrCanNotRecover } task.ReplaceTargetReplicaSet(rs) log.Infof(tname, "Recover dead target node to %s(%s)", rs.Master.Id, rs.Master.Addr()) } } return nil }