// 似乎，只有同时进行Forget和Reset才有意义，否则都是一个不一致的状态
func (self *ForgetAndResetNodeCommand) Execute(c *cc.Controller) (cc.Result, error) {
cs := c.ClusterState
target := cs.FindNode(self.NodeId)
if target == nil {
return nil, ErrNodeNotExist
}
if !target.Free == false {
return nil, ErrNodeIsFree
}
if len(target.Ranges) > 0 {
return nil, ErrNodeNotEmpty
}
var err error
forgetCount := 0
allForgetDone := true
// 1. 所有节点发送Forget
for _, ns := range cs.AllNodeStates() {
if ns.Id() == target.Id {
continue
}
node := ns.Node()
_, err = redis.ClusterForget(ns.Addr(), target.Id)
if !node.Fail && err != nil && !strings.HasPrefix(err.Error(), "ERR Unknown node") {
allForgetDone = false
log.Warningf(target.Addr(), "Forget node %s(%s) failed, %v", ns.Addr(), ns.Id(), err)
continue
} else if !node.Fail && err != nil {
//try again
for try := redis.NUM_RETRY; try >= 0; try-- {
_, err = redis.ClusterForget(ns.Addr(), target.Id)
if err == nil {
break
}
}
//execute failed after retry
if err != nil {
allForgetDone = false
log.Warningf(target.Addr(), "Forget node %s(%s) failed after retry, %v", ns.Addr(), ns.Id(), err)
continue
}
}
log.Eventf(target.Addr(), "Forget by %s(%s).", ns.Addr(), ns.Id())
forgetCount++
}
if !allForgetDone {
return nil, fmt.Errorf("Not all forget done, only (%d/%d) success",
forgetCount, len(cs.AllNodeStates())-1)
}
// 2. 重置
if !target.Fail {
_, err = redis.ClusterReset(target.Addr(), false)
if err != nil {
return nil, fmt.Errorf("Reset node %s(%s) failed, %v", target.Id, target.Addr(), err)
}
log.Eventf(target.Addr(), "Reset.")
}
// remove seed in leader contrller
meta.RemoveSeed(target.Addr())
return nil, nil
}
func fixHandshakeAction(c *cli.Context) {
addr := context.GetLeaderAddr()
url := "http://" + addr + api.FetchReplicaSetsPath
resp, err := utils.HttpGet(url, nil, 5*time.Second)
if err != nil {
fmt.Println(err)
return
}
var rss command.FetchReplicaSetsResult
err = utils.InterfaceToStruct(resp.Body, &rss)
if err != nil {
fmt.Println(err)
return
}
sort.Sort(topo.ByMasterId(rss.ReplicaSets))
sort.Sort(topo.ByNodeState(rss.ReplicaSets))
seedNodes := []string{}
allFailedNodes := []string{}
inner := func(addr string) {
failedNodes, err := getFailedNodes(addr)
if err != nil {
fmt.Println(err)
return
}
if len(failedNodes) > 0 {
seedNodes = append(seedNodes, addr)
}
for _, fn := range failedNodes {
if nodeExists(fn, allFailedNodes) == false {
allFailedNodes = append(allFailedNodes, fn)
}
}
}
for _, rs := range rss.ReplicaSets {
n := rs.Master
inner(n.Addr())
//slaves
for _, ns := range rs.Slaves {
inner(ns.Addr())
}
}
//send forget to need fix nodes
resChan := make(chan string, len(seedNodes))
for _, failed := range allFailedNodes {
for _, seed := range seedNodes {
go func(seed, failed string) {
resp, err := redis.ClusterForget(seed, failed)
res := fmt.Sprintf("Node %s forget %s result %s %v", seed, failed, resp, err)
resChan <- res
}(seed, failed)
}
for i := 0; i < len(seedNodes); i++ {
res := <-resChan
fmt.Println(res)
}
}
}