func apiGetServerGroupList() (int, string) {
conn := CreateCoordConn()
defer conn.Close()
groups, err := models.ServerGroups(conn, globalEnv.ProductName())
if err != nil {
log.Warning(err)
return 500, err.Error()
}
b, err := json.MarshalIndent(groups, " ", " ")
return 200, string(b)
}
func getLivingNodeInfos(coordConn zkhelper.Conn) ([]*NodeInfo, error) {
groups, err := models.ServerGroups(coordConn, globalEnv.ProductName())
if err != nil {
return nil, errors.Trace(err)
}
slots, err := models.Slots(coordConn, globalEnv.ProductName())
slotMap := make(map[int][]int)
for _, slot := range slots {
if slot.State.Status == models.SLOT_STATUS_ONLINE {
slotMap[slot.GroupId] = append(slotMap[slot.GroupId], slot.Id)
}
}
var ret []*NodeInfo
for _, g := range groups {
master, err := g.Master(coordConn)
if err != nil {
return nil, errors.Trace(err)
}
if master == nil {
return nil, errors.Errorf("group %d has no master", g.Id)
}
out, err := utils.GetRedisConfig(master.Addr, "MAXMEMORY", globalEnv.StoreAuth())
if err != nil {
return nil, errors.Trace(err)
}
maxMem, err := strconv.ParseInt(out, 10, 64)
if err != nil {
return nil, errors.Trace(err)
}
if maxMem <= 0 {
return nil, errors.Errorf("redis %s should set maxmemory", master.Addr)
}
node := &NodeInfo{
GroupId: g.Id,
CurSlots: slotMap[g.Id],
MaxMemory: maxMem,
}
ret = append(ret, node)
}
cnt := 0
for _, info := range ret {
cnt += len(info.CurSlots)
}
if cnt != models.DEFAULT_SLOT_NUM {
return nil, errors.New("not all slots are online")
}
return ret, nil
}
func apiOverview() (int, string) {
conn := CreateCoordConn()
defer conn.Close()
// get all server groups
groups, err := models.ServerGroups(conn, globalEnv.ProductName())
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return 500, err.Error()
}
var instances []string
for _, group := range groups {
for _, srv := range group.Servers {
if srv.Type == "master" {
instances = append(instances, srv.Addr)
}
}
}
info := make(map[string]interface{})
info["product"] = globalEnv.ProductName()
info["ops"] = proxiesSpeed
redisInfos := make([]map[string]string, 0)
if len(instances) > 0 {
for _, instance := range instances {
info, err := utils.GetRedisStat(instance, globalEnv.StoreAuth())
if err != nil {
log.Error(err)
}
redisInfos = append(redisInfos, info)
}
}
info["redis_infos"] = redisInfos
b, err := json.MarshalIndent(info, " ", " ")
return 200, string(b)
}
func (t *haTask) check() error {
groups, err := models.ServerGroups(globalConn, globalEnv.ProductName())
if err != nil {
return errors.Trace(err)
}
cnt := 0
const checkChanSize = 100
ch := make(chan interface{}, checkChanSize)
// check all servers in all groups
for _, group := range groups {
for _, server := range group.Servers {
cnt++
go t.checkGroupServer(server, ch)
}
}
var crashSlaves []*models.Server
var crashMasters []*models.Server
var svrOnline []*models.Server
for i := 0; i < cnt; i++ {
v := <-ch
switch s := v.(type) {
case nil:
case *models.Server:
if s.Type == models.SERVER_TYPE_SLAVE {
crashSlaves = append(crashSlaves, s)
} else if s.Type == models.SERVER_TYPE_MASTER {
crashMasters = append(crashMasters, s)
} else if s.Type == models.SERVER_TYPE_OFFLINE {
log.Infof("Server %s in group %d is already start append", s.Addr, s.GroupId)
svrOnline = append(svrOnline, s)
}
case error:
err = errors.Trace(s)
}
}
if err != nil {
return errors.Trace(err)
}
for _, s := range crashSlaves {
log.Infof("slave %s in group %d is down, set offline", s.Addr, s.GroupId)
group := models.NewServerGroup(globalEnv.ProductName(), s.GroupId)
s.Type = models.SERVER_TYPE_OFFLINE
if err := group.AddServer(globalConn, s, globalEnv.StoreAuth()); err != nil {
return errors.Trace(err)
}
}
for _, s := range crashMasters {
log.Infof("master %s in group %d is down, do failover", s.Addr, s.GroupId)
if err := t.doFailover(s); err != nil {
log.Errorf("master %s in group %d is down, do failover err: %v", s.Addr, s.GroupId, err)
}
}
for _, s := range svrOnline {
log.Infof("Server %s in group %d is already start", s.Addr, s.GroupId)
group := models.NewServerGroup(globalEnv.ProductName(), s.GroupId)
s.Type = models.SERVER_TYPE_SLAVE
if err := group.AddServer(globalConn, s, globalEnv.StoreAuth()); err != nil {
return errors.Trace(err)
}
}
return nil
}