func TestProxyOfflineInWaitActionReceiver(t *testing.T) {
log.Infof("test proxy offline when waiting action response")
fakeZkConn := zkhelper.NewConn()
for i := 1; i <= 4; i++ {
CreateProxyInfo(fakeZkConn, productName, &ProxyInfo{
Id: strconv.Itoa(i),
State: PROXY_STATE_ONLINE,
})
go waitForProxyMarkOffline(fakeZkConn, strconv.Itoa(i))
}
lst, _ := ProxyList(fakeZkConn, productName, nil)
assert.Must(len(lst) == 4)
go func() {
time.Sleep(500 * time.Millisecond)
actionPath := path.Join(GetActionResponsePath(productName), fakeZkConn.Seq2Str(1))
//create test response for proxy 4, means proxy 1,2,3 are timeout
fakeZkConn.Create(path.Join(actionPath, "4"), nil,
0, zkhelper.DefaultFileACLs())
}()
err := NewActionWithTimeout(fakeZkConn, productName, ACTION_TYPE_SLOT_CHANGED, nil, "desc", true, 3*1000)
if err != nil {
assert.Must(err.Error() == ErrReceiverTimeout.Error())
}
for i := 1; i <= 3; i++ {
info, _ := GetProxyInfo(fakeZkConn, productName, strconv.Itoa(i))
assert.Must(info.State == PROXY_STATE_OFFLINE)
}
}
func resetEnv() {
conn = zkhelper.NewConn()
once.Do(func() {
go runFakeRedisSrv("127.0.0.1:1111")
go runFakeRedisSrv("127.0.0.1:2222")
time.Sleep(1 * time.Second)
})
}
func TestProxy(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
path := GetSlotBasePath(productName)
children, _, _ := fakeZkConn.Children(path)
assert.Must(len(children) == 0)
g := NewServerGroup(productName, 1)
g.Create(fakeZkConn)
// test create new group
_, err := ServerGroups(fakeZkConn, productName)
assert.MustNoError(err)
ok, err := g.Exists(fakeZkConn)
assert.MustNoError(err)
assert.Must(ok)
s1 := NewServer(SERVER_TYPE_MASTER, "localhost:1111")
g.AddServer(fakeZkConn, s1, "")
err = InitSlotSet(fakeZkConn, productName, 1024)
assert.MustNoError(err)
children, _, _ = fakeZkConn.Children(path)
assert.Must(len(children) == 1024)
s, err := GetSlot(fakeZkConn, productName, 1)
assert.MustNoError(err)
assert.Must(s.GroupId == -1)
err = SetSlotRange(fakeZkConn, productName, 0, 1023, 1, SLOT_STATUS_ONLINE)
assert.MustNoError(err)
pi := &ProxyInfo{
Id: "proxy_1",
Addr: "localhost:1234",
State: PROXY_STATE_OFFLINE,
}
_, err = CreateProxyInfo(fakeZkConn, productName, pi)
assert.MustNoError(err)
ps, err := ProxyList(fakeZkConn, productName, nil)
assert.MustNoError(err)
assert.Must(len(ps) == 1)
assert.Must(ps[0].Id == "proxy_1")
err = SetProxyStatus(fakeZkConn, productName, pi.Id, PROXY_STATE_ONLINE)
assert.MustNoError(err)
p, err := GetProxyInfo(fakeZkConn, productName, pi.Id)
assert.MustNoError(err)
assert.Must(p.State == PROXY_STATE_ONLINE)
}
func TestWaitForReceiver(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
proxies := []ProxyInfo{}
for i := 0; i < 5; i++ {
proxies = append(proxies, ProxyInfo{
Id: fmt.Sprintf("proxy_%d", i),
Addr: fmt.Sprintf("localhost:%d", i+1234),
State: PROXY_STATE_ONLINE,
})
CreateProxyInfo(fakeZkConn, productName, &proxies[i])
}
zkhelper.CreateRecursive(fakeZkConn, GetActionResponsePath(productName)+"/1", "", 0, zkhelper.DefaultDirACLs())
go func() {
time.Sleep(time.Second * 2)
doResponseForTest(fakeZkConn, "1", &proxies[0])
doResponseForTest(fakeZkConn, "1", &proxies[1])
doResponseForTest(fakeZkConn, "1", &proxies[2])
doResponseForTest(fakeZkConn, "1", &proxies[3])
doResponseForTest(fakeZkConn, "1", &proxies[4])
for {
for i := 0; i < 5; i++ {
pname := fmt.Sprintf("proxy_%d", i)
p, _ := GetProxyInfo(fakeZkConn, productName, pname)
if p != nil && p.State == PROXY_STATE_MARK_OFFLINE {
zkhelper.DeleteRecursive(fakeZkConn, path.Join(GetProxyPath(productName), pname), -1)
}
}
}
}()
err := WaitForReceiverWithTimeout(fakeZkConn, productName, GetActionResponsePath(productName)+"/1", proxies, 1000*10)
if err != nil {
t.Error("there is error not as expected")
}
p, _ := GetProxyInfo(fakeZkConn, productName, "proxy_0")
if p == nil || p.State != PROXY_STATE_ONLINE {
t.Error("proxy_0 status is not as expected")
}
p, _ = GetProxyInfo(fakeZkConn, productName, "proxy_1")
if p == nil || p.State != PROXY_STATE_ONLINE {
t.Error("proxy_1 status is not as expected")
}
p, _ = GetProxyInfo(fakeZkConn, productName, "proxy_2")
if p == nil || p.State != PROXY_STATE_ONLINE {
t.Error("proxy_2 status is not as expected")
}
p, _ = GetProxyInfo(fakeZkConn, productName, "proxy_3")
if p == nil || p.State != PROXY_STATE_ONLINE {
t.Error("proxy_3 status is not as expected")
}
p, _ = GetProxyInfo(fakeZkConn, productName, "proxy_4")
if p == nil || p.State != PROXY_STATE_ONLINE {
t.Error("proxy_4 status is not as expected")
}
}
func TestForceRemoveLock(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
zkLock := utils.GetZkLock(fakeZkConn, productName)
assert.Must(zkLock != nil)
zkLock.Lock("force remove lock")
zkPath := fmt.Sprintf("/zk/codis/db_%s/LOCK", productName)
children, _, err := fakeZkConn.Children(zkPath)
assert.MustNoError(err)
assert.Must(len(children) != 0)
ForceRemoveLock(fakeZkConn, productName)
children, _, err = fakeZkConn.Children(zkPath)
assert.MustNoError(err)
assert.Must(len(children) == 0)
}
func init() {
conn = zkhelper.NewConn()
conf = &Config{
proxyId: "proxy_test",
productName: "test",
zkAddr: "localhost:2181",
fact: func(string, int) (zkhelper.Conn, error) { return conn, nil },
proto: "tcp4",
}
//init action path
prefix := models.GetWatchActionPath(conf.productName)
err := models.CreateActionRootPath(conn, prefix)
assert.MustNoError(err)
//init slot
err = models.InitSlotSet(conn, conf.productName, 1024)
assert.MustNoError(err)
//init server group
g1 := models.NewServerGroup(conf.productName, 1)
g1.Create(conn)
g2 := models.NewServerGroup(conf.productName, 2)
g2.Create(conn)
redis1, _ = miniredis.Run()
redis2, _ = miniredis.Run()
s1 := models.NewServer(models.SERVER_TYPE_MASTER, redis1.Addr())
s2 := models.NewServer(models.SERVER_TYPE_MASTER, redis2.Addr())
g1.AddServer(conn, s1, "")
g2.AddServer(conn, s2, "")
//set slot range
err = models.SetSlotRange(conn, conf.productName, 0, 511, 1, models.SLOT_STATUS_ONLINE)
assert.MustNoError(err)
err = models.SetSlotRange(conn, conf.productName, 512, 1023, 2, models.SLOT_STATUS_ONLINE)
assert.MustNoError(err)
s = New(":19000", ":11000", conf)
err = models.SetProxyStatus(conn, conf.productName, conf.proxyId, models.PROXY_STATE_ONLINE)
assert.MustNoError(err)
}
func initReal() {
conn = zkhelper.NewConn()
conf = &Config{
proxyId: "proxy_test",
productName: "test",
zkAddr: "192.168.28.191:2181",
fact: nil,
proto: "tcp4",
provider: "zookeeper",
zkSessionTimeout: 30,
zkReadTimeout: 30,
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT, os.Kill)
go func() {
<-c
log.Info("ctrl-c or SIGTERM found, bye bye...")
s.Close()
}()
go func() {
log.Info(http.ListenAndServe("192.168.28.192:6060", nil))
}()
go func() {
time.Sleep(10 * time.Second)
zkConn, err := zkhelper.ConnectToZk(conf.zkAddr, 20000)
if err != nil {
log.Errorf("connect to zk: %+v", errors.Trace(err))
} else {
err = models.SetProxyStatus(zkConn, conf.productName, conf.proxyId, models.PROXY_STATE_ONLINE)
if err != nil {
log.Errorf("set proxy error: %+v", errors.Trace(err))
}
}
zkConn.Close()
}()
go func() {
err := http.ListenAndServe("192.168.28.192:11001", nil)
log.PanicError(err, "http debug server quit")
}()
s = New("192.168.28.192:19001", "192.168.28.192:11001", conf)
}
func TestSlots(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
path := GetSlotBasePath(productName)
children, _, _ := fakeZkConn.Children(path)
assert.Must(len(children) == 0)
err := InitSlotSet(fakeZkConn, productName, 1024)
assert.MustNoError(err)
children, _, _ = fakeZkConn.Children(path)
assert.Must(len(children) == 1024)
s, err := GetSlot(fakeZkConn, productName, 1)
assert.MustNoError(err)
assert.Must(s.GroupId == -1)
g := NewServerGroup(productName, 1)
g.Create(fakeZkConn)
// test create new group
_, err = ServerGroups(fakeZkConn, productName)
assert.MustNoError(err)
ok, err := g.Exists(fakeZkConn)
assert.MustNoError(err)
assert.Must(ok)
err = SetSlotRange(fakeZkConn, productName, 0, 1023, 1, SLOT_STATUS_ONLINE)
assert.MustNoError(err)
s, err = GetSlot(fakeZkConn, productName, 1)
assert.MustNoError(err)
assert.Must(s.GroupId == 1)
err = s.SetMigrateStatus(fakeZkConn, 1, 2)
assert.MustNoError(err)
assert.Must(s.GroupId == 2)
assert.Must(s.State.Status == SLOT_STATUS_MIGRATE)
}
func TestNewAction(t *testing.T) {
fakeZkConn := zkhelper.NewConn()
err := NewAction(fakeZkConn, productName, ACTION_TYPE_SLOT_CHANGED, nil, "desc", false)
assert.MustNoError(err)
prefix := GetWatchActionPath(productName)
exist, _, err := fakeZkConn.Exists(prefix)
assert.MustNoError(err)
assert.Must(exist)
//test if response node exists
d, _, err := fakeZkConn.Get(prefix + "/0000000001")
assert.MustNoError(err)
//test get action data
d, _, err = fakeZkConn.Get(GetActionResponsePath(productName) + "/0000000001")
assert.MustNoError(err)
var action Action
err = json.Unmarshal(d, &action)
assert.MustNoError(err)
assert.Must(action.Desc == "desc")
assert.Must(action.Type == ACTION_TYPE_SLOT_CHANGED)
}
func InitEnv() {
go once.Do(func() {
conn = zkhelper.NewConn()
conf = &Config{
proxyId: "proxy_test",
productName: "test",
zkAddr: "localhost:2181",
fact: func(string) (zkhelper.Conn, error) { return conn, nil },
proto: "tcp4",
}
//init action path
prefix := models.GetWatchActionPath(conf.productName)
err := models.CreateActionRootPath(conn, prefix)
assert.MustNoError(err)
//init slot
err = models.InitSlotSet(conn, conf.productName, 1024)
assert.MustNoError(err)
//init server group
g1 := models.NewServerGroup(conf.productName, 1)
g1.Create(conn)
g2 := models.NewServerGroup(conf.productName, 2)
g2.Create(conn)
redis1, _ = miniredis.Run()
redis2, _ = miniredis.Run()
s1 := models.NewServer(models.SERVER_TYPE_MASTER, redis1.Addr())
s2 := models.NewServer(models.SERVER_TYPE_MASTER, redis2.Addr())
g1.AddServer(conn, s1, "")
g2.AddServer(conn, s2, "")
//set slot range
err = models.SetSlotRange(conn, conf.productName, 0, 511, 1, models.SLOT_STATUS_ONLINE)
assert.MustNoError(err)
err = models.SetSlotRange(conn, conf.productName, 512, 1023, 2, models.SLOT_STATUS_ONLINE)
assert.MustNoError(err)
go func() { //set proxy online
time.Sleep(3 * time.Second)
err := models.SetProxyStatus(conn, conf.productName, conf.proxyId, models.PROXY_STATE_ONLINE)
assert.MustNoError(err)
time.Sleep(2 * time.Second)
proxyMutex.Lock()
defer proxyMutex.Unlock()
assert.Must(s.info.State == models.PROXY_STATE_ONLINE)
}()
proxyMutex.Lock()
s, err = NewServer(":19000", ":11000", conf)
assert.MustNoError(err)
proxyMutex.Unlock()
s.Serve()
})
waitonce.Do(func() {
time.Sleep(10 * time.Second)
})
}