func (s *testModelSuite) TestNewAction(c *C) {
log.Info("[TestNewAction][start]")
fakeCoordConn := zkhelper.NewConn()
err := NewAction(fakeCoordConn, productName, ACTION_TYPE_SLOT_CHANGED, nil, "desc", false)
c.Assert(err, IsNil)
prefix := GetWatchActionPath(productName)
exist, _, err := fakeCoordConn.Exists(prefix)
c.Assert(exist, Equals, true)
c.Assert(err, IsNil)
// test if response node exists
d, _, err := fakeCoordConn.Get(prefix + "/0000000001")
c.Assert(err, IsNil)
// test get action data
d, _, err = fakeCoordConn.Get(GetActionResponsePath(productName) + "/0000000001")
c.Assert(err, IsNil)
var action Action
json.Unmarshal(d, &action)
c.Assert(action.Desc, Equals, "desc")
c.Assert(action.Type, Equals, ACTION_TYPE_SLOT_CHANGED)
fakeCoordConn.Close()
log.Info("[TestNewAction][end]")
}
func TestForceRemoveLock(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
zkLock := utils.GetZkLock(fakeZkConn, productName)
if zkLock == nil {
t.Error("create lock error")
}
zkLock.Lock("force remove lock")
zkPath := fmt.Sprintf("/zk/codis/db_%s/LOCK", productName)
children, _, err := fakeZkConn.Children(zkPath)
if err != nil {
t.Error(err)
}
if len(children) == 0 {
t.Error("create lock error")
}
ForceRemoveLock(fakeZkConn, productName)
children, _, err = fakeZkConn.Children(zkPath)
if err != nil {
t.Error(err)
}
if len(children) != 0 {
t.Error("remove lock error")
}
}
// NewTimestampOracle creates a tso server with special config.
func NewTimestampOracle(cfg *Config) (*TimestampOracle, error) {
if cfg.SaveInterval <= 0 {
cfg.SaveInterval = defaultSaveInterval
}
tso := &TimestampOracle{
cfg: cfg,
isLeader: 0,
}
var err error
tso.listener, err = net.Listen("tcp", cfg.Addr)
if err != nil {
return nil, errors.Trace(err)
}
if len(cfg.ZKAddr) == 0 {
// no zookeeper, use a fake zk conn instead
tso.zkConn = zkhelper.NewConn()
} else {
tso.zkConn, err = zkhelper.ConnectToZkWithTimeout(cfg.ZKAddr, zkTimeout)
if err != nil {
return nil, errors.Trace(err)
}
}
tso.conns = map[net.Conn]struct{}{}
return tso, nil
}
func TestNewAction(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
err := NewAction(fakeZkConn, productName, ACTION_TYPE_SLOT_CHANGED, nil, "desc", false)
if err != nil {
t.Error(errors.ErrorStack(err))
}
prefix := GetWatchActionPath(productName)
if exist, _, err := fakeZkConn.Exists(prefix); !exist {
t.Error(errors.ErrorStack(err))
}
//test if response node exists
d, _, err := fakeZkConn.Get(prefix + "/0000000001")
if err != nil {
t.Error(errors.ErrorStack(err))
}
//test get action data
d, _, err = fakeZkConn.Get(GetActionResponsePath(productName) + "/0000000001")
if err != nil {
t.Error(errors.ErrorStack(err))
}
var action Action
json.Unmarshal(d, &action)
if action.Desc != "desc" || action.Type != ACTION_TYPE_SLOT_CHANGED {
t.Error("create action error")
}
}
func TestProxyOfflineInWaitActionReceiver(t *testing.T) {
log.Info("test proxy offline when waiting action response")
fakeZkConn := zkhelper.NewConn()
for i := 1; i <= 4; i++ {
CreateProxyInfo(fakeZkConn, productName, &ProxyInfo{
Id: strconv.Itoa(i),
State: PROXY_STATE_ONLINE,
})
go waitForProxyMarkOffline(fakeZkConn, strconv.Itoa(i))
}
lst, _ := ProxyList(fakeZkConn, productName, nil)
if len(lst) != 4 {
t.Error("create proxy info error")
}
go func() {
time.Sleep(500 * time.Millisecond)
actionPath := path.Join(GetActionResponsePath(productName), fakeZkConn.Seq2Str(1))
//create test response for proxy 4, means proxy 1,2,3 are timeout
fakeZkConn.Create(path.Join(actionPath, "4"), nil,
0, zkhelper.DefaultFileACLs())
}()
err := NewActionWithTimeout(fakeZkConn, productName, ACTION_TYPE_SLOT_CHANGED, nil, "desc", true, 3*1000)
if err != nil && err.Error() != ErrReceiverTimeout.Error() {
t.Error(errors.ErrorStack(err))
}
for i := 1; i <= 3; i++ {
if info, _ := GetProxyInfo(fakeZkConn, productName, strconv.Itoa(i)); info.State != PROXY_STATE_OFFLINE {
t.Error("shutdown offline proxy error")
}
}
}
func resetEnv() {
conn = zkhelper.NewConn()
once.Do(func() {
go runFakeRedisSrv("127.0.0.1:1111")
go runFakeRedisSrv("127.0.0.1:2222")
time.Sleep(1 * time.Second)
})
}
func TestSlots(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
path := GetSlotBasePath(productName)
children, _, _ := fakeZkConn.Children(path)
if len(children) != 0 {
t.Error("slot is no empty")
}
err := InitSlotSet(fakeZkConn, productName, 1024)
if err != nil {
t.Error(err)
}
children, _, _ = fakeZkConn.Children(path)
if len(children) != 1024 {
t.Error("init slots error")
}
s, err := GetSlot(fakeZkConn, productName, 1)
if err != nil {
t.Error(err)
}
if s.GroupId != -1 {
t.Error("init slots error")
}
err = SetSlotRange(fakeZkConn, productName, 0, 1023, 1, SLOT_STATUS_ONLINE)
if err != nil {
t.Error(err)
}
s, err = GetSlot(fakeZkConn, productName, 1)
if err != nil {
t.Error(err)
}
if s.GroupId != 1 {
t.Error("range set error")
}
err = s.SetMigrateStatus(fakeZkConn, 1, 2)
if err != nil {
t.Error(err)
}
if s.GroupId != 2 || s.State.Status != SLOT_STATUS_MIGRATE {
t.Error("migrate error")
}
}
func (s *testModelSuite) TestAddSlaveToEmptyGroup(c *C) {
log.Info("[TestAddSlaveToEmptyGroup][start]")
fakeCoordConn := zkhelper.NewConn()
g := NewServerGroup(productName, 1)
g.Create(fakeCoordConn)
s1 := NewServer(SERVER_TYPE_SLAVE, s.s1.addr)
err := g.AddServer(fakeCoordConn, s1, auth)
c.Assert(err, IsNil)
c.Assert(g.Servers[0].Type, Equals, SERVER_TYPE_MASTER)
fakeCoordConn.Close()
log.Info("[TestAddSlaveToEmptyGroup][end]")
}
func (s *testModelSuite) TestServerGroup(c *C) {
log.Info("[TestServerGroup][start]")
fakeCoordConn := zkhelper.NewConn()
g := NewServerGroup(productName, 1)
g.Create(fakeCoordConn)
// test create new group
groups, err := ServerGroups(fakeCoordConn, productName)
c.Assert(err, IsNil)
c.Assert(len(groups), Not(Equals), 0)
ok, err := g.Exists(fakeCoordConn)
c.Assert(err, IsNil)
c.Assert(ok, Equals, true)
gg, err := GetGroup(fakeCoordConn, productName, 1)
c.Assert(err, IsNil)
c.Assert(gg, NotNil)
c.Assert(gg.Id, Equals, g.Id)
s1 := NewServer(SERVER_TYPE_MASTER, s.s1.addr)
s2 := NewServer(SERVER_TYPE_MASTER, s.s2.addr)
err = g.AddServer(fakeCoordConn, s1, auth)
c.Assert(err, IsNil)
servers, err := g.GetServers(fakeCoordConn)
c.Assert(err, IsNil)
c.Assert(len(servers), Equals, 1)
g.AddServer(fakeCoordConn, s2, auth)
c.Assert(len(g.Servers), Equals, 1)
s2.Type = SERVER_TYPE_SLAVE
g.AddServer(fakeCoordConn, s2, auth)
c.Assert(len(g.Servers), Equals, 2)
err = g.Promote(fakeCoordConn, s2.Addr, auth)
c.Assert(err, IsNil)
m, err := g.Master(fakeCoordConn)
c.Assert(err, IsNil)
c.Assert(m.Addr, Equals, s2.Addr)
fakeCoordConn.Close()
log.Info("[TestServerGroup][stop]")
}
func (s *testModelSuite) TestSlots(c *C) {
log.Info("[TestSlots][start]")
fakeCoordConn := zkhelper.NewConn()
path := GetSlotBasePath(productName)
children, _, _ := fakeCoordConn.Children(path)
c.Assert(len(children), Equals, 0)
err := InitSlotSet(fakeCoordConn, productName, 1024)
c.Assert(err, IsNil)
children, _, err = fakeCoordConn.Children(path)
c.Assert(err, IsNil)
c.Assert(len(children), Equals, 1024)
sl, err := GetSlot(fakeCoordConn, productName, 1)
c.Assert(err, IsNil)
c.Assert(sl.GroupId, Equals, -1)
g := NewServerGroup(productName, 1)
g.Create(fakeCoordConn)
// test create new group
_, err = ServerGroups(fakeCoordConn, productName)
c.Assert(err, IsNil)
ok, err := g.Exists(fakeCoordConn)
c.Assert(err, IsNil)
c.Assert(ok, Equals, true)
err = SetSlotRange(fakeCoordConn, productName, 0, 1023, 1, SLOT_STATUS_ONLINE)
c.Assert(err, IsNil)
sl, err = GetSlot(fakeCoordConn, productName, 1)
c.Assert(err, IsNil)
c.Assert(sl.GroupId, Equals, 1)
err = sl.SetMigrateStatus(fakeCoordConn, 1, 2)
c.Assert(err, IsNil)
c.Assert(sl.GroupId, Equals, 2)
c.Assert(sl.State.Status, Equals, SLOT_STATUS_MIGRATE)
fakeCoordConn.Close()
log.Info("[TestSlots][end]")
}
func (s *testModelSuite) TestForceRemoveLock(c *C) {
log.Info("[TestForceRemoveLock][start]")
fakeCoordConn := zkhelper.NewConn()
zkLock := utils.GetCoordLock(fakeCoordConn, productName)
c.Assert(zkLock, NotNil)
zkLock.Lock("force remove lock")
coordPath := fmt.Sprintf("/zk/reborn/db_%s/LOCK", productName)
children, _, err := fakeCoordConn.Children(coordPath)
c.Assert(err, IsNil)
c.Assert(len(children), Not(Equals), 0)
ForceRemoveLock(fakeCoordConn, productName)
children, _, err = fakeCoordConn.Children(coordPath)
c.Assert(err, IsNil)
c.Assert(len(children), Equals, 0)
fakeCoordConn.Close()
log.Info("[TestForceRemoveLock][end]")
}
func TestMigrateManager(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
mgr := NewMigrateManager(fakeZkConn, testProductName, nil)
if mgr == nil {
t.Error("mgr is null")
}
nodePath := fmt.Sprintf("/zk/codis/db_%s/migrate_manager", testProductName)
b, _, err := fakeZkConn.Exists(nodePath)
if !b || err != nil {
t.Error("create migrate mgr node error")
}
err = mgr.removeNode()
if err != nil {
t.Error(err)
}
b, _, err = fakeZkConn.Exists(nodePath)
if b {
t.Error("remove migrate mgr node error")
}
}
func TestProxy(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
pi := &ProxyInfo{
Id: "proxy_1",
Addr: "localhost:1234",
State: PROXY_STATE_OFFLINE,
}
_, err := CreateProxyInfo(fakeZkConn, productName, pi)
if err != nil {
t.Error(err)
}
ps, err := ProxyList(fakeZkConn, productName, nil)
if err != nil {
t.Error(err)
}
if len(ps) != 1 || ps[0].Id != "proxy_1" {
t.Error("create proxy error")
}
err = SetProxyStatus(fakeZkConn, productName, pi.Id, PROXY_STATE_ONLINE)
if err != nil {
t.Error(err)
}
p, err := GetProxyInfo(fakeZkConn, productName, pi.Id)
if err != nil {
t.Error(err)
}
if p.State != PROXY_STATE_ONLINE {
t.Error("change status error")
}
}
func (s *testModelSuite) TestProxyOfflineInWaitActionReceiver(c *C) {
log.Info("[TestProxyOfflineInWaitActionReceiver][start]")
fakeCoordConn := zkhelper.NewConn()
proxyNum := 4
for i := 1; i <= proxyNum; i++ {
CreateProxyInfo(fakeCoordConn, productName, &ProxyInfo{
ID: strconv.Itoa(i),
State: PROXY_STATE_ONLINE,
})
go waitForProxyMarkOffline(fakeCoordConn, strconv.Itoa(i))
}
lst, _ := ProxyList(fakeCoordConn, productName, nil)
c.Assert(len(lst), Equals, proxyNum)
go func() {
time.Sleep(500 * time.Millisecond)
actionPath := path.Join(GetActionResponsePath(productName), fakeCoordConn.Seq2Str(1))
// create test response for proxy 4, means proxy 1,2,3 are timeout
fakeCoordConn.Create(path.Join(actionPath, "4"), nil,
0, zkhelper.DefaultFileACLs())
}()
err := NewActionWithTimeout(fakeCoordConn, productName, ACTION_TYPE_SLOT_CHANGED, nil, "desc", true, 3*1000)
if c.Check(err, NotNil) {
c.Assert(err.Error(), Equals, ErrReceiverTimeout.Error())
}
for i := 1; i <= proxyNum-1; i++ {
info, _ := GetProxyInfo(fakeCoordConn, productName, strconv.Itoa(i))
c.Assert(info.State, Equals, PROXY_STATE_OFFLINE)
}
fakeCoordConn.Close()
log.Info("[TestProxyOfflineInWaitActionReceiver][end]")
}
func InitEnv() {
go once.Do(func() {
conn = zkhelper.NewConn()
conf = &Conf{
proxyId: "proxy_test",
productName: "test",
zkAddr: "localhost:2181",
netTimeout: 5,
f: func(string) (zkhelper.Conn, error) { return conn, nil },
proto: "tcp4",
}
//init action path
prefix := models.GetWatchActionPath(conf.productName)
err := models.CreateActionRootPath(conn, prefix)
if err != nil {
log.Fatal(err)
}
//init slot
err = models.InitSlotSet(conn, conf.productName, 1024)
if err != nil {
log.Fatal(err)
}
//init server group
g1 := models.NewServerGroup(conf.productName, 1)
g1.Create(conn)
g2 := models.NewServerGroup(conf.productName, 2)
g2.Create(conn)
redis1, _ = miniredis.Run()
redis2, _ = miniredis.Run()
s1 := models.NewServer(models.SERVER_TYPE_MASTER, redis1.Addr())
s2 := models.NewServer(models.SERVER_TYPE_MASTER, redis2.Addr())
g1.AddServer(conn, s1)
g2.AddServer(conn, s2)
//set slot range
err = models.SetSlotRange(conn, conf.productName, 0, 511, 1, models.SLOT_STATUS_ONLINE)
if err != nil {
log.Fatal(err)
}
err = models.SetSlotRange(conn, conf.productName, 512, 1023, 2, models.SLOT_STATUS_ONLINE)
if err != nil {
log.Fatal(err)
}
go func() { //set proxy online
time.Sleep(3 * time.Second)
err := models.SetProxyStatus(conn, conf.productName, conf.proxyId, models.PROXY_STATE_ONLINE)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
time.Sleep(2 * time.Second)
proxyMutex.Lock()
defer proxyMutex.Unlock()
pi := s.getProxyInfo()
if pi.State != models.PROXY_STATE_ONLINE {
log.Fatalf("should be online, we got %s", pi.State)
}
}()
proxyMutex.Lock()
s = NewServer(":19000", ":11000",
conf,
)
proxyMutex.Unlock()
s.Run()
})
waitonce.Do(func() {
time.Sleep(10 * time.Second)
})
}
func (s *testModelSuite) TestProxy(c *C) {
log.Info("[TestProxy][start]")
fakeCoordConn := zkhelper.NewConn()
path := GetSlotBasePath(productName)
children, _, _ := fakeCoordConn.Children(path)
c.Assert(len(children), Equals, 0)
g := NewServerGroup(productName, 1)
g.Create(fakeCoordConn)
// test create new group
_, err := ServerGroups(fakeCoordConn, productName)
c.Assert(err, IsNil)
ok, err := g.Exists(fakeCoordConn)
c.Assert(err, IsNil)
c.Assert(ok, Equals, true)
s1 := NewServer(SERVER_TYPE_MASTER, "localhost:1111")
g.AddServer(fakeCoordConn, s1, auth)
err = InitSlotSet(fakeCoordConn, productName, 1024)
c.Assert(err, IsNil)
children, _, err = fakeCoordConn.Children(path)
c.Assert(err, IsNil)
c.Assert(len(children), Equals, 1024)
sl, err := GetSlot(fakeCoordConn, productName, 1)
c.Assert(err, IsNil)
c.Assert(sl.GroupId, Equals, -1)
err = SetSlotRange(fakeCoordConn, productName, 0, 1023, 1, SLOT_STATUS_ONLINE)
c.Assert(err, IsNil)
pi := &ProxyInfo{
ID: "proxy_1",
Addr: "localhost:1234",
State: PROXY_STATE_OFFLINE,
}
_, err = CreateProxyInfo(fakeCoordConn, productName, pi)
c.Assert(err, IsNil)
ps, err := ProxyList(fakeCoordConn, productName, nil)
c.Assert(err, IsNil)
c.Assert(len(ps), Equals, 1)
c.Assert(ps[0].ID, Equals, "proxy_1")
err = SetProxyStatus(fakeCoordConn, productName, pi.ID, PROXY_STATE_ONLINE)
c.Assert(err, IsNil)
p, err := GetProxyInfo(fakeCoordConn, productName, pi.ID)
c.Assert(err, IsNil)
c.Assert(p.State, Equals, PROXY_STATE_ONLINE)
fakeCoordConn.Close()
log.Info("[TestProxy][end]")
}
func TestProxy(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
path := GetSlotBasePath(productName)
children, _, _ := fakeZkConn.Children(path)
if len(children) != 0 {
t.Error("slot is no empty")
}
g := NewServerGroup(productName, 1)
g.Create(fakeZkConn)
// test create new group
_, err := ServerGroups(fakeZkConn, productName)
if err != nil {
t.Error(err)
}
ok, err := g.Exists(fakeZkConn)
if !ok || err != nil {
t.Error("create group error")
}
s1 := NewServer(SERVER_TYPE_MASTER, "localhost:1111")
g.AddServer(fakeZkConn, s1)
err = InitSlotSet(fakeZkConn, productName, 1024)
if err != nil {
t.Error(err)
}
children, _, _ = fakeZkConn.Children(path)
if len(children) != 1024 {
t.Error("init slots error")
}
s, err := GetSlot(fakeZkConn, productName, 1)
if err != nil {
t.Error(err)
}
if s.GroupId != -1 {
t.Error("init slots error")
}
err = SetSlotRange(fakeZkConn, productName, 0, 1023, 1, SLOT_STATUS_ONLINE)
if err != nil {
t.Error(err)
}
pi := &ProxyInfo{
Id: "proxy_1",
Addr: "localhost:1234",
State: PROXY_STATE_OFFLINE,
}
_, err = CreateProxyInfo(fakeZkConn, productName, pi)
if err != nil {
t.Error(err)
}
ps, err := ProxyList(fakeZkConn, productName, nil)
if err != nil {
t.Error(err)
}
if len(ps) != 1 || ps[0].Id != "proxy_1" {
t.Error("create proxy error")
}
err = SetProxyStatus(fakeZkConn, productName, pi.Id, PROXY_STATE_ONLINE)
if err != nil {
t.Error(err)
}
p, err := GetProxyInfo(fakeZkConn, productName, pi.Id)
if err != nil {
t.Error(err)
}
if p.State != PROXY_STATE_ONLINE {
t.Error("change status error")
}
}
func TestServerGroup(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
g := NewServerGroup(productName, 1)
g.Create(fakeZkConn)
// test create new group
groups, err := ServerGroups(fakeZkConn, productName)
if err != nil {
t.Error(err)
}
if len(groups) == 0 {
t.Error("create group error")
}
ok, err := g.Exists(fakeZkConn)
if !ok || err != nil {
t.Error("create group error")
}
gg, err := GetGroup(fakeZkConn, productName, 1)
if err != nil {
t.Error(err)
}
if gg == nil || gg.Id != g.Id {
t.Error("get group error")
}
s1 := NewServer(SERVER_TYPE_MASTER, "localhost:1111")
s2 := NewServer(SERVER_TYPE_MASTER, "localhost:2222")
g.AddServer(fakeZkConn, s1)
servers, err := g.GetServers(fakeZkConn)
if err != nil {
t.Error("add server error")
}
if len(servers) != 1 {
t.Error("add server error", len(servers))
}
g.AddServer(fakeZkConn, s1)
if len(g.Servers) != 1 {
t.Error("create group error")
}
g.AddServer(fakeZkConn, s2)
// add another master
if len(g.Servers) != 1 {
t.Error("add server error")
}
s2.Type = SERVER_TYPE_SLAVE
g.AddServer(fakeZkConn, s2)
if len(g.Servers) != 2 {
t.Error("add server error")
}
g.Promote(fakeZkConn, s2.Addr)
s, err := g.Master(fakeZkConn)
if err != nil {
t.Error(err)
}
// already exist master
if s.Addr != s1.Addr {
t.Error("prompt error")
}
s, err = g.Master(fakeZkConn)
if err != nil {
t.Error(err)
}
// already exist master
if s.Addr != s1.Addr {
t.Error("master error")
}
}
func (s *testProxyRouterSuite) initEnv(c *C) {
go once.Do(func() {
conn = zkhelper.NewConn()
conf = &Conf{
ProductName: "test",
CoordinatorAddr: "localhost:2181",
NetTimeout: 5,
f: func(string) (zkhelper.Conn, error) { return conn, nil },
Proto: "tcp4",
ProxyID: "proxy_test",
Addr: ":19000",
HTTPAddr: ":11000",
ProxyAuth: proxyAuth,
StoreAuth: storeAuth,
}
// init action path
prefix := models.GetWatchActionPath(conf.ProductName)
err := models.CreateActionRootPath(conn, prefix)
c.Assert(err, IsNil)
// init slot
err = models.InitSlotSet(conn, conf.ProductName, 1024)
c.Assert(err, IsNil)
// init server group
g1 := models.NewServerGroup(conf.ProductName, 1)
g1.Create(conn)
g2 := models.NewServerGroup(conf.ProductName, 2)
g2.Create(conn)
s1 := models.NewServer(models.SERVER_TYPE_MASTER, s.s1.addr)
s2 := models.NewServer(models.SERVER_TYPE_MASTER, s.s2.addr)
g1.AddServer(conn, s1, storeAuth)
g2.AddServer(conn, s2, storeAuth)
// set slot range
err = models.SetSlotRange(conn, conf.ProductName, 0, 511, 1, models.SLOT_STATUS_ONLINE)
c.Assert(err, IsNil)
err = models.SetSlotRange(conn, conf.ProductName, 512, 1023, 2, models.SLOT_STATUS_ONLINE)
c.Assert(err, IsNil)
go func() { //set proxy online
time.Sleep(3 * time.Second)
err := models.SetProxyStatus(conn, conf.ProductName, conf.ProxyID, models.PROXY_STATE_ONLINE)
c.Assert(err, IsNil)
time.Sleep(2 * time.Second)
proxyMutex.Lock()
defer proxyMutex.Unlock()
pi := ss.getProxyInfo()
c.Assert(pi.State, Equals, models.PROXY_STATE_ONLINE)
}()
proxyMutex.Lock()
ss = NewServer(conf)
proxyMutex.Unlock()
ss.Run()
})
waitonce.Do(func() {
time.Sleep(10 * time.Second)
})
}