// 更新任务状态机
func (m *MigrateManager) handleTaskChange(task *MigrateTask, cluster *topo.Cluster) error {
fromNode := cluster.FindNode(task.SourceNode().Id)
toNode := cluster.FindNode(task.TargetNode().Id)
tname := task.TaskName()
if fromNode == nil {
log.Infof(tname, "Source node %s(%s) not exist", fromNode.Addr(), fromNode.Id)
return ErrNodeNotFound
}
if toNode == nil {
log.Infof(tname, "Target node %s(%s) not exist", toNode.Addr(), toNode.Id)
return ErrNodeNotFound
}
// 角色变化说明该分片进行了主从切换
if !fromNode.IsMaster() || !toNode.IsMaster() {
log.Warningf(tname, "%s role change, cancel migration task %s\n", fromNode.Id[:6], task.TaskName())
task.SetState(StateCancelling)
return ErrSourceNodeFail
}
// 如果是源节点挂了，直接取消，等待主从切换之后重建任务
if fromNode.Fail {
log.Infof(tname, "Cancel migration task %s\n", task.TaskName())
task.SetState(StateCancelling)
return ErrSourceNodeFail
}
// 如果目标节点挂了，需要记录当前的ReplicaSet，观察等待主从切换
if toNode.Fail {
if task.CurrentState() == StateRunning {
task.SetState(StateTargetNodeFailure)
task.SetBackupReplicaSet(task.TargetReplicaSet())
return ErrTargetNodeFail
}
} else if task.CurrentState() != StateNew {
task.SetState(StateRunning)
task.SetBackupReplicaSet(nil)
}
// 如果目标节点已经进行了Failover(重新选主)，我们需要找到对应的新主
// 方法是从BackupReplicaSet里取一个从来查找
if toNode.IsStandbyMaster() {
brs := task.BackupReplicaSet()
if brs == nil {
task.SetState(StateCancelling)
log.Info(tname, "No backup replicaset found, controller maybe restarted after target master failure, can not do recovery.")
return ErrCanNotRecover
}
slaves := brs.Slaves
if len(slaves) == 0 {
task.SetState(StateCancelling)
log.Info(tname, "The dead target master has no slave, cannot do recovery.")
return ErrCanNotRecover
} else {
rs := cluster.FindReplicaSetByNode(slaves[0].Id)
if rs == nil {
task.SetState(StateCancelling)
log.Info(tname, "No replicaset for slave of dead target master found")
return ErrCanNotRecover
}
task.ReplaceTargetReplicaSet(rs)
log.Infof(tname, "Recover dead target node to %s(%s)",
rs.Master.Id, rs.Master.Addr())
}
}
return nil
}
func (self *Inspector) checkClusterTopo(seed *topo.Node, cluster *topo.Cluster) error {
resp, err := redis.ClusterNodesInRegion(seed.Addr(), self.LocalRegion)
if err != nil && strings.HasPrefix(err.Error(), "ERR Wrong CLUSTER subcommand or number of arguments") {
//server version do not support 'cluster nodes extra [region]'
resp, err = redis.ClusterNodes(seed.Addr())
}
//this may lead to BuildClusterTopo update failed for a time
//the node is step into this state after check IsAlive
if err != nil && strings.HasPrefix(err.Error(), "LOADING") {
return nil
}
if err != nil {
return err
}
var summary topo.SummaryInfo
lines := strings.Split(resp, "\n")
for _, line := range lines {
if strings.HasPrefix(line, "# ") {
summary.ReadLine(line)
continue
}
line = strings.TrimSpace(line)
if line == "" {
continue
}
s, myself, err := self.buildNode(line)
if err == ErrNodeInHandShake || err == ErrNodeNoAddr {
continue
}
// Fix 'cluster nodes extra' & 'cluster nodes extra region' compatiable
if s.Region != self.LocalRegion {
continue
}
if err != nil {
return err
}
if s.Ip == "127.0.0.1" {
s.Ip = seed.Ip
}
node := cluster.FindNode(s.Id)
if node == nil {
if s.PFail {
glog.Warningf("forget dead node %s(%s) should be forgoten", s.Id, s.Addr())
//redis.ClusterForget(seed.Addr(), s.Id)
}
return fmt.Errorf("node not exist %s(%s)", s.Id, s.Addr())
}
// 对比节点数据是否相同
if !node.Compare(s) {
glog.Infof("%#v vs %#v different", s, node)
if s.Tag == "-" && node.Tag != "-" {
// 可能存在处于不被Cluster接受的节点，节点可以看见Cluster，但Cluster看不到它。
// 一种复现情况情况：某个节点已经死了，系统将其Forget，但是OP并未被摘除该节点，
// 而是恢复了该节点。
glog.Warningf("remeet node %s", seed.Addr())
self.MeetNode(seed)
}
return ErrNodesInfoNotSame
}
if len(node.Ranges) == 0 && len(s.Ranges) > 0 {
glog.Warningf("Ranges not equal, use nonempty ranges.")
node.Ranges = s.Ranges
}
if myself {
info, err := redis.FetchClusterInfo(node.Addr())
if err != nil {
return err
}
node.ClusterInfo = info
node.SummaryInfo = summary
}
if len(s.Migrating) != 0 {
node.Migrating = s.Migrating
}
if len(s.Importing) != 0 {
node.Importing = s.Importing
}
if s.PFail {
node.IncrPFailCount()
}
}
return nil
}
