func (c *Controller) ProcessCommand(command Command, timeout time.Duration) (result Result, err error) {
switch command.Type() {
case REGION_COMMAND:
if !meta.IsRegionLeader() {
return nil, ErrNotRegionLeader
}
case CLUSTER_COMMAND:
if !meta.IsClusterLeader() {
return nil, ErrNotClusterLeader
}
}
// 一次处理一条命令，也即同一时间只能在做一个状态变换
c.mutex.Lock()
defer c.mutex.Unlock()
resultCh := make(chan Result)
errorCh := make(chan error)
//c.ClusterState.DebugDump()
go func() {
result, err := command.Execute(c)
if err != nil {
errorCh <- err
} else {
resultCh <- result
}
}()
select {
case result = <-resultCh:
case err = <-errorCh:
case <-time.After(timeout):
err = ErrProcessCommandTimedout
}
return
}
// 生成ClusterSnapshot
func (self *Inspector) BuildClusterTopo() (*topo.Cluster, []*topo.Node, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
if len(meta.Seeds()) == 0 {
return nil, nil, ErrNoSeed
}
// 过滤掉连接不上的节点
seeds := []*topo.Node{}
for _, s := range meta.Seeds() {
if redis.IsAlive(s.Addr()) {
seeds = append(seeds, s)
} else {
// remove this seed from meta seeds
// will re-add to seeds if join the cluster again
meta.RemoveSeed(s.Addr())
}
}
if len(seeds) == 0 {
return nil, seeds, ErrNoSeed
}
// 顺序选一个节点，获取nodes数据作为基准，再用其他节点的数据与基准做对比
if self.SeedIndex >= len(seeds) {
self.SeedIndex = len(seeds) - 1
}
var seed *topo.Node
for i := 0; i < len(seeds); i++ {
seed = seeds[self.SeedIndex]
self.SeedIndex++
self.SeedIndex %= len(seeds)
if seed.Free {
glog.Info("Seed node is free ", seed.Addr())
} else {
break
}
}
cluster, err := self.initClusterTopo(seed)
if err != nil {
glog.Infof("InitClusterTopo failed")
return nil, seeds, err
}
// 检查所有节点返回的信息是不是相同，如果不同说明正在变化中，直接返回等待重试
if len(seeds) > 1 {
for _, s := range seeds {
if s == seed {
continue
}
err := self.checkClusterTopo(s, cluster)
if err != nil {
free, node := self.isFreeNode(s)
if free {
node.Free = true
glog.Infof("Found free node %s", node.Addr())
cluster.AddNode(node)
} else {
glog.Infof("checkClusterTopo failed")
return cluster, seeds, err
}
} else {
s.Free = false
}
}
}
// 构造LocalRegion视图
for _, s := range cluster.LocalRegionNodes() {
if s.PFailCount() > cluster.NumLocalRegionNode()/2 {
glog.Infof("Found %d/%d PFAIL state on %s, set FAIL",
s.PFailCount(), cluster.NumLocalRegionNode(), s.Addr())
s.SetFail(true)
}
}
if meta.IsClusterLeader() {
cluster.BuildReplicaSets()
}
meta.MergeSeeds(cluster.LocalRegionNodes())
self.ClusterTopo = cluster
return cluster, seeds, nil
}