// 更新任务状态机 func (m *MigrateManager) handleTaskChange(task *MigrateTask, cluster *topo.Cluster) error { fromNode := cluster.FindNode(task.SourceNode().Id) toNode := cluster.FindNode(task.TargetNode().Id) tname := task.TaskName() if fromNode == nil { log.Infof(tname, "Source node %s(%s) not exist", fromNode.Addr(), fromNode.Id) return ErrNodeNotFound } if toNode == nil { log.Infof(tname, "Target node %s(%s) not exist", toNode.Addr(), toNode.Id) return ErrNodeNotFound } // 角色变化说明该分片进行了主从切换 if !fromNode.IsMaster() || !toNode.IsMaster() { log.Warningf(tname, "%s role change, cancel migration task %s\n", fromNode.Id[:6], task.TaskName()) task.SetState(StateCancelling) return ErrSourceNodeFail } // 如果是源节点挂了,直接取消,等待主从切换之后重建任务 if fromNode.Fail { log.Infof(tname, "Cancel migration task %s\n", task.TaskName()) task.SetState(StateCancelling) return ErrSourceNodeFail } // 如果目标节点挂了,需要记录当前的ReplicaSet,观察等待主从切换 if toNode.Fail { if task.CurrentState() == StateRunning { task.SetState(StateTargetNodeFailure) task.SetBackupReplicaSet(task.TargetReplicaSet()) return ErrTargetNodeFail } } else if task.CurrentState() != StateNew { task.SetState(StateRunning) task.SetBackupReplicaSet(nil) } // 如果目标节点已经进行了Failover(重新选主),我们需要找到对应的新主 // 方法是从BackupReplicaSet里取一个从来查找 if toNode.IsStandbyMaster() { brs := task.BackupReplicaSet() if brs == nil { task.SetState(StateCancelling) log.Info(tname, "No backup replicaset found, controller maybe restarted after target master failure, can not do recovery.") return ErrCanNotRecover } slaves := brs.Slaves if len(slaves) == 0 { task.SetState(StateCancelling) log.Info(tname, "The dead target master has no slave, cannot do recovery.") return ErrCanNotRecover } else { rs := cluster.FindReplicaSetByNode(slaves[0].Id) if rs == nil { task.SetState(StateCancelling) log.Info(tname, "No replicaset for slave of dead target master found") return ErrCanNotRecover } task.ReplaceTargetReplicaSet(rs) log.Infof(tname, "Recover dead target node to %s(%s)", rs.Master.Id, rs.Master.Addr()) } } return nil }
func (self *Inspector) checkClusterTopo(seed *topo.Node, cluster *topo.Cluster) error { resp, err := redis.ClusterNodesInRegion(seed.Addr(), self.LocalRegion) if err != nil && strings.HasPrefix(err.Error(), "ERR Wrong CLUSTER subcommand or number of arguments") { //server version do not support 'cluster nodes extra [region]' resp, err = redis.ClusterNodes(seed.Addr()) } //this may lead to BuildClusterTopo update failed for a time //the node is step into this state after check IsAlive if err != nil && strings.HasPrefix(err.Error(), "LOADING") { return nil } if err != nil { return err } var summary topo.SummaryInfo lines := strings.Split(resp, "\n") for _, line := range lines { if strings.HasPrefix(line, "# ") { summary.ReadLine(line) continue } line = strings.TrimSpace(line) if line == "" { continue } s, myself, err := self.buildNode(line) if err == ErrNodeInHandShake || err == ErrNodeNoAddr { continue } // Fix 'cluster nodes extra' & 'cluster nodes extra region' compatiable if s.Region != self.LocalRegion { continue } if err != nil { return err } if s.Ip == "127.0.0.1" { s.Ip = seed.Ip } node := cluster.FindNode(s.Id) if node == nil { if s.PFail { glog.Warningf("forget dead node %s(%s) should be forgoten", s.Id, s.Addr()) //redis.ClusterForget(seed.Addr(), s.Id) } return fmt.Errorf("node not exist %s(%s)", s.Id, s.Addr()) } // 对比节点数据是否相同 if !node.Compare(s) { glog.Infof("%#v vs %#v different", s, node) if s.Tag == "-" && node.Tag != "-" { // 可能存在处于不被Cluster接受的节点,节点可以看见Cluster,但Cluster看不到它。 // 一种复现情况情况:某个节点已经死了,系统将其Forget,但是OP并未被摘除该节点, // 而是恢复了该节点。 glog.Warningf("remeet node %s", seed.Addr()) self.MeetNode(seed) } return ErrNodesInfoNotSame } if len(node.Ranges) == 0 && len(s.Ranges) > 0 { glog.Warningf("Ranges not equal, use nonempty ranges.") node.Ranges = s.Ranges } if myself { info, err := redis.FetchClusterInfo(node.Addr()) if err != nil { return err } node.ClusterInfo = info node.SummaryInfo = summary } if len(s.Migrating) != 0 { node.Migrating = s.Migrating } if len(s.Importing) != 0 { node.Importing = s.Importing } if s.PFail { node.IncrPFailCount() } } return nil }