func runAddServerGroup(groupId int) error {
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
var v interface{}
err := callApi(METHOD_PUT, "/api/server_groups", serverGroup, &v)
if err != nil {
return err
}
fmt.Println(jsonify(v))
return nil
}
func (s *testAgentSuite) checkStoreServerType(c *C, addr string, tp string) {
group := models.NewServerGroup(globalEnv.ProductName(), 1)
servers, err := group.GetServers(globalConn)
c.Assert(err, IsNil)
for _, server := range servers {
if server.Addr == addr {
c.Assert(server.Type, Equals, tp)
return
}
}
c.Fatalf("addr %s is not in group servers %v", addr, servers)
}
func (t *haTask) doFailover(s *models.Server) error {
// first get all slaves
group := models.NewServerGroup(globalEnv.ProductName(), s.GroupId)
var err error
group.Servers, err = group.GetServers(globalConn)
if err != nil {
return errors.Trace(err)
}
slaves := make([]*models.Server, 0, len(group.Servers))
slaveAddrs := make([]string, 0, len(group.Servers))
for _, s := range group.Servers {
if s.Type == models.SERVER_TYPE_SLAVE {
slaves = append(slaves, s)
slaveAddrs = append(slaveAddrs, s.Addr)
}
}
// elect a new master
log.Infof("elect a new master in %v", slaveAddrs)
addr, err := t.electNewMaster(slaves)
if err != nil {
return errors.Trace(err)
}
// prmote it as new master
log.Infof("promote %s as the new master", addr)
if err := group.Promote(globalConn, addr, globalEnv.StoreAuth()); err != nil {
// should we fatal here and let human intervention ???
return errors.Trace(err)
}
// let other slaves replicate from the new master
for _, slave := range slaves {
if slave.Addr == addr {
continue
}
log.Infof("let %s slaveof new master %s", slave.Addr, addr)
if err := utils.SlaveOf(slave.Addr, addr, globalEnv.StoreAuth()); err != nil {
// should we fatal here and let human intervention ???
return errors.Trace(err)
}
}
return nil
}
// add redis server to exist server group
func apiAddServerToGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
conn := CreateCoordConn()
defer conn.Close()
lock := utils.GetCoordLock(conn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("add server to group, %+v", server))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
// check group exists first
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
exists, err := serverGroup.Exists(conn)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
// create new group if not exists
if !exists {
if err := serverGroup.Create(conn); err != nil {
return 500, err.Error()
}
}
if err := serverGroup.AddServer(conn, &server, globalEnv.StoreAuth()); err != nil {
log.Warning(errors.ErrorStack(err))
return 500, err.Error()
}
return jsonRetSucc()
}
func apiRemoveServerGroup(param martini.Params) (int, string) {
conn := CreateCoordConn()
defer conn.Close()
lock := utils.GetCoordLock(conn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("removing group %s", param["id"]))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
groupId, _ := strconv.Atoi(param["id"])
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
if err := serverGroup.Remove(conn); err != nil {
log.Error(errors.ErrorStack(err))
return 500, err.Error()
}
return jsonRetSucc()
}
func apiRemoveServerFromGroup(server models.Server, param martini.Params) (int, string) {
groupId, _ := strconv.Atoi(param["id"])
conn := CreateCoordConn()
defer conn.Close()
lock := utils.GetCoordLock(conn, globalEnv.ProductName())
lock.Lock(fmt.Sprintf("removing server from group, %+v", server))
defer func() {
err := lock.Unlock()
if err != nil {
log.Warning(err)
}
}()
serverGroup := models.NewServerGroup(globalEnv.ProductName(), groupId)
err := serverGroup.RemoveServer(conn, server.Addr)
if err != nil {
log.Warning(err)
return 500, err.Error()
}
return jsonRetSucc()
}
func (s *testAgentSuite) testInitGroup(c *C) {
serverGroup := models.NewServerGroup(globalEnv.ProductName(), 1)
s.callDashboardAPI(c, nil, "/api/server_groups", "PUT", serverGroup)
}
func (t *haTask) check() error {
groups, err := models.ServerGroups(globalConn, globalEnv.ProductName())
if err != nil {
return errors.Trace(err)
}
cnt := 0
const checkChanSize = 100
ch := make(chan interface{}, checkChanSize)
// check all servers in all groups
for _, group := range groups {
for _, server := range group.Servers {
cnt++
go t.checkGroupServer(server, ch)
}
}
var crashSlaves []*models.Server
var crashMasters []*models.Server
var svrOnline []*models.Server
for i := 0; i < cnt; i++ {
v := <-ch
switch s := v.(type) {
case nil:
case *models.Server:
if s.Type == models.SERVER_TYPE_SLAVE {
crashSlaves = append(crashSlaves, s)
} else if s.Type == models.SERVER_TYPE_MASTER {
crashMasters = append(crashMasters, s)
} else if s.Type == models.SERVER_TYPE_OFFLINE {
log.Infof("Server %s in group %d is already start append", s.Addr, s.GroupId)
svrOnline = append(svrOnline, s)
}
case error:
err = errors.Trace(s)
}
}
if err != nil {
return errors.Trace(err)
}
for _, s := range crashSlaves {
log.Infof("slave %s in group %d is down, set offline", s.Addr, s.GroupId)
group := models.NewServerGroup(globalEnv.ProductName(), s.GroupId)
s.Type = models.SERVER_TYPE_OFFLINE
if err := group.AddServer(globalConn, s, globalEnv.StoreAuth()); err != nil {
return errors.Trace(err)
}
}
for _, s := range crashMasters {
log.Infof("master %s in group %d is down, do failover", s.Addr, s.GroupId)
if err := t.doFailover(s); err != nil {
log.Errorf("master %s in group %d is down, do failover err: %v", s.Addr, s.GroupId, err)
}
}
for _, s := range svrOnline {
log.Infof("Server %s in group %d is already start", s.Addr, s.GroupId)
group := models.NewServerGroup(globalEnv.ProductName(), s.GroupId)
s.Type = models.SERVER_TYPE_SLAVE
if err := group.AddServer(globalConn, s, globalEnv.StoreAuth()); err != nil {
return errors.Trace(err)
}
}
return nil
}
func (s *testProxyRouterSuite) initEnv(c *C) {
go once.Do(func() {
conn = zkhelper.NewConn()
conf = &Conf{
ProductName: "test",
CoordinatorAddr: "localhost:2181",
NetTimeout: 5,
f: func(string) (zkhelper.Conn, error) { return conn, nil },
Proto: "tcp4",
ProxyID: "proxy_test",
Addr: ":19000",
HTTPAddr: ":11000",
ProxyAuth: proxyAuth,
StoreAuth: storeAuth,
}
// init action path
prefix := models.GetWatchActionPath(conf.ProductName)
err := models.CreateActionRootPath(conn, prefix)
c.Assert(err, IsNil)
// init slot
err = models.InitSlotSet(conn, conf.ProductName, 1024)
c.Assert(err, IsNil)
// init server group
g1 := models.NewServerGroup(conf.ProductName, 1)
g1.Create(conn)
g2 := models.NewServerGroup(conf.ProductName, 2)
g2.Create(conn)
s1 := models.NewServer(models.SERVER_TYPE_MASTER, s.s1.addr)
s2 := models.NewServer(models.SERVER_TYPE_MASTER, s.s2.addr)
g1.AddServer(conn, s1, storeAuth)
g2.AddServer(conn, s2, storeAuth)
// set slot range
err = models.SetSlotRange(conn, conf.ProductName, 0, 511, 1, models.SLOT_STATUS_ONLINE)
c.Assert(err, IsNil)
err = models.SetSlotRange(conn, conf.ProductName, 512, 1023, 2, models.SLOT_STATUS_ONLINE)
c.Assert(err, IsNil)
go func() { //set proxy online
time.Sleep(3 * time.Second)
err := models.SetProxyStatus(conn, conf.ProductName, conf.ProxyID, models.PROXY_STATE_ONLINE)
c.Assert(err, IsNil)
time.Sleep(2 * time.Second)
proxyMutex.Lock()
defer proxyMutex.Unlock()
pi := ss.getProxyInfo()
c.Assert(pi.State, Equals, models.PROXY_STATE_ONLINE)
}()
proxyMutex.Lock()
ss = NewServer(conf)
proxyMutex.Unlock()
ss.Run()
})
waitonce.Do(func() {
time.Sleep(10 * time.Second)
})
}