Example #1
0
// 似乎,只有同时进行Forget和Reset才有意义,否则都是一个不一致的状态
func (self *ForgetAndResetNodeCommand) Execute(c *cc.Controller) (cc.Result, error) {
	cs := c.ClusterState
	target := cs.FindNode(self.NodeId)
	if target == nil {
		return nil, ErrNodeNotExist
	}
	if !target.Free == false {
		return nil, ErrNodeIsFree
	}
	if len(target.Ranges) > 0 {
		return nil, ErrNodeNotEmpty
	}
	var err error
	forgetCount := 0
	allForgetDone := true
	// 1. 所有节点发送Forget
	for _, ns := range cs.AllNodeStates() {
		if ns.Id() == target.Id {
			continue
		}
		node := ns.Node()
		_, err = redis.ClusterForget(ns.Addr(), target.Id)
		if !node.Fail && err != nil && !strings.HasPrefix(err.Error(), "ERR Unknown node") {
			allForgetDone = false
			log.Warningf(target.Addr(), "Forget node %s(%s) failed, %v", ns.Addr(), ns.Id(), err)
			continue
		} else if !node.Fail && err != nil {
			//try again
			for try := redis.NUM_RETRY; try >= 0; try-- {
				_, err = redis.ClusterForget(ns.Addr(), target.Id)
				if err == nil {
					break
				}
			}
			//execute failed after retry
			if err != nil {
				allForgetDone = false
				log.Warningf(target.Addr(), "Forget node %s(%s) failed after retry, %v", ns.Addr(), ns.Id(), err)
				continue
			}

		}
		log.Eventf(target.Addr(), "Forget by %s(%s).", ns.Addr(), ns.Id())
		forgetCount++
	}
	if !allForgetDone {
		return nil, fmt.Errorf("Not all forget done, only (%d/%d) success",
			forgetCount, len(cs.AllNodeStates())-1)
	}
	// 2. 重置
	if !target.Fail {
		_, err = redis.ClusterReset(target.Addr(), false)
		if err != nil {
			return nil, fmt.Errorf("Reset node %s(%s) failed, %v", target.Id, target.Addr(), err)
		}
		log.Eventf(target.Addr(), "Reset.")
	}

	// remove seed in leader contrller
	meta.RemoveSeed(target.Addr())
	return nil, nil
}
Example #2
0
func fixHandshakeAction(c *cli.Context) {
	addr := context.GetLeaderAddr()
	url := "http://" + addr + api.FetchReplicaSetsPath

	resp, err := utils.HttpGet(url, nil, 5*time.Second)
	if err != nil {
		fmt.Println(err)
		return
	}

	var rss command.FetchReplicaSetsResult
	err = utils.InterfaceToStruct(resp.Body, &rss)
	if err != nil {
		fmt.Println(err)
		return
	}
	sort.Sort(topo.ByMasterId(rss.ReplicaSets))
	sort.Sort(topo.ByNodeState(rss.ReplicaSets))

	seedNodes := []string{}
	allFailedNodes := []string{}
	inner := func(addr string) {
		failedNodes, err := getFailedNodes(addr)
		if err != nil {
			fmt.Println(err)
			return
		}
		if len(failedNodes) > 0 {
			seedNodes = append(seedNodes, addr)
		}
		for _, fn := range failedNodes {
			if nodeExists(fn, allFailedNodes) == false {
				allFailedNodes = append(allFailedNodes, fn)
			}
		}
	}
	for _, rs := range rss.ReplicaSets {
		n := rs.Master
		inner(n.Addr())

		//slaves
		for _, ns := range rs.Slaves {
			inner(ns.Addr())
		}
	}

	//send forget to need fix nodes
	resChan := make(chan string, len(seedNodes))
	for _, failed := range allFailedNodes {
		for _, seed := range seedNodes {
			go func(seed, failed string) {
				resp, err := redis.ClusterForget(seed, failed)
				res := fmt.Sprintf("Node %s forget %s result %s %v", seed, failed, resp, err)
				resChan <- res
			}(seed, failed)
		}
		for i := 0; i < len(seedNodes); i++ {
			res := <-resChan
			fmt.Println(res)
		}
	}
}