func checkMaster(addr string) error {
if master, err := utils.GetRole(addr, ""); err != nil {
return errors.Trace(err)
} else if master != "master" {
return ErrServerIsNotMaster
} else {
return nil
}
}
func (s *testAgentSuite) TestHA(c *C) {
s.agentHA = s.testStartAgent(c, "127.0.0.1:39005", true)
defer s.testStopAgent(c, s.agentHA)
time.Sleep(3 * time.Second)
// test slave ha
procs := s.testGetProcs(c, s.agentStoreSlave)
s.testStopAgent(c, s.agentStoreSlave)
s.testKillProcs(c, procs)
time.Sleep(3 * time.Second)
s.checkStoreServerType(c, "127.0.0.1:6382", models.SERVER_TYPE_OFFLINE)
s.agentStoreSlave = s.testStartAgent(c, "127.0.0.1:39004", false)
time.Sleep(3 * time.Second)
// now agentStoreSlave restart and redis restart
role, err := utils.GetRole("127.0.0.1:6382", globalEnv.StoreAuth())
c.Assert(err, IsNil)
c.Assert(role, Equals, "master")
s.testAddStoreToGroup(c, 6382, models.SERVER_TYPE_SLAVE)
s.checkStoreServerType(c, "127.0.0.1:6382", models.SERVER_TYPE_SLAVE)
role, err = utils.GetRole("127.0.0.1:6382", globalEnv.StoreAuth())
c.Assert(err, IsNil)
c.Assert(role, Equals, "slave")
// test master ha
procs = s.testGetProcs(c, s.agentStoreMaster)
s.testStopAgent(c, s.agentStoreMaster)
s.testKillProcs(c, procs)
time.Sleep(3 * time.Second)
// now 6382 is slave, and 6381 is offline
s.checkStoreServerType(c, "127.0.0.1:6382", models.SERVER_TYPE_MASTER)
role, err = utils.GetRole("127.0.0.1:6382", globalEnv.StoreAuth())
c.Assert(err, IsNil)
c.Assert(role, Equals, "master")
s.checkStoreServerType(c, "127.0.0.1:6381", models.SERVER_TYPE_OFFLINE)
s.agentStoreMaster = s.testStartAgent(c, "127.0.0.1:39003", false)
time.Sleep(3 * time.Second)
// now agentStoreMaster restart and redis restart
role, err = utils.GetRole("127.0.0.1:6381", globalEnv.StoreAuth())
c.Assert(err, IsNil)
c.Assert(role, Equals, "master")
s.testAddStoreToGroup(c, 6381, models.SERVER_TYPE_SLAVE)
s.checkStoreServerType(c, "127.0.0.1:6381", models.SERVER_TYPE_SLAVE)
role, err = utils.GetRole("127.0.0.1:6381", globalEnv.StoreAuth())
c.Assert(err, IsNil)
c.Assert(role, Equals, "slave")
}
func (sg *ServerGroup) AddServer(coordConn zkhelper.Conn, s *Server, auth string) error {
switch s.Type {
case SERVER_TYPE_MASTER, SERVER_TYPE_SLAVE, SERVER_TYPE_OFFLINE:
default:
return errors.NotSupportedf("server type %q", s.Type)
}
// if type is offline, the server may be down, so we cannot use store function
if s.Type != SERVER_TYPE_OFFLINE {
// we only support reborn-server and qdb-server
// origin redis has no slot_info command
// atm, we can use this command to check whether server is alive or not.
if _, err := utils.SlotsInfo(s.Addr, 0, 0, auth); err != nil {
return errors.Trace(err)
}
}
s.GroupId = sg.Id
servers, err := sg.GetServers(coordConn)
if err != nil {
return errors.Trace(err)
}
var masterAddr string
for _, server := range servers {
if server.Type == SERVER_TYPE_MASTER {
masterAddr = server.Addr
}
}
// make sure there is only one master
if s.Type == SERVER_TYPE_MASTER && len(masterAddr) > 0 {
return errors.Trace(ErrNodeExists)
}
// if this group has no server.
// promote this server to master automatically if type is not offline
if len(servers) == 0 && s.Type != SERVER_TYPE_OFFLINE {
s.Type = SERVER_TYPE_MASTER
}
if s.Type == SERVER_TYPE_MASTER {
if role, err := utils.GetRole(s.Addr, auth); err != nil {
return errors.Trace(err)
} else if role != "master" {
return errors.Errorf("we need master, but server %s is %s", s.Addr, role)
}
}
val, err := json.Marshal(s)
if err != nil {
return errors.Trace(err)
}
coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d/%s", sg.ProductName, sg.Id, s.Addr)
_, err = zkhelper.CreateOrUpdate(coordConn, coordPath, string(val), 0, zkhelper.DefaultFileACLs(), true)
// update servers
servers, err = sg.GetServers(coordConn)
if err != nil {
return errors.Trace(err)
}
sg.Servers = servers
if s.Type == SERVER_TYPE_MASTER {
err = NewAction(coordConn, sg.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, sg, "", true)
if err != nil {
return errors.Trace(err)
}
} else if s.Type == SERVER_TYPE_SLAVE && len(masterAddr) > 0 {
// send command slaveof to slave
err := utils.SlaveOf(s.Addr, masterAddr, auth)
if err != nil {
return errors.Trace(err)
}
}
return nil
}