func (s *testAgentSuite) SetUpSuite(c *C) {
// initial whole test environment
configFile = "config.ini"
resetAbsPath(&configFile)
cfg, err := utils.InitConfigFromFile(configFile)
c.Assert(err, IsNil)
globalEnv = env.LoadRebornEnv(cfg)
globalConn, err = globalEnv.NewCoordConn()
c.Assert(err, IsNil)
s.testSetExecPath(c)
// remove all infos in coordinator first
err = zkhelper.DeleteRecursive(globalConn, fmt.Sprintf("/zk/reborn/db_%s", globalEnv.ProductName()), -1)
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
c.Assert(err, IsNil)
}
s.agentDashboard = s.testStartAgent(c, "127.0.0.1:39001", false)
s.agentProxy = s.testStartAgent(c, "127.0.0.1:39002", false)
s.agentStoreMaster = s.testStartAgent(c, "127.0.0.1:39003", false)
s.agentStoreSlave = s.testStartAgent(c, "127.0.0.1:39004", false)
s.testDashboard(c)
s.testInitGroup(c)
s.testInitSlots(c)
s.testStoreAddServer(c)
}
func ServerGroups(coordConn zkhelper.Conn, productName string) ([]*ServerGroup, error) {
var ret []*ServerGroup
root := fmt.Sprintf("/zk/reborn/db_%s/servers", productName)
groups, _, err := coordConn.Children(root)
// if ErrNoNode, we may return an empty slice like ProxyList
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return nil, errors.Trace(err)
}
// Buggy :X
//zkhelper.ChildrenRecursive(*coordConn, root)
for _, group := range groups {
// parse group_1 => 1
groupId, err := strconv.Atoi(strings.Split(group, "_")[1])
if err != nil {
return nil, errors.Trace(err)
}
g, err := GetGroup(coordConn, productName, groupId)
if err != nil {
return nil, errors.Trace(err)
}
ret = append(ret, g)
}
return ret, nil
}
func SetProxyStatus(zkConn zkhelper.Conn, productName string, proxyName string, status string) error {
p, err := GetProxyInfo(zkConn, productName, proxyName)
if err != nil {
return errors.Trace(err)
}
if status != PROXY_STATE_ONLINE && status != PROXY_STATE_MARK_OFFLINE && status != PROXY_STATE_OFFLINE {
return errors.Errorf("%v, %s", ErrUnknownProxyStatus, status)
}
p.State = status
b, _ := json.Marshal(p)
_, err = zkConn.Set(path.Join(GetProxyPath(productName), proxyName), b, -1)
if err != nil {
return errors.Trace(err)
}
if status == PROXY_STATE_MARK_OFFLINE {
// wait for the proxy down
for {
_, _, c, err := zkConn.GetW(path.Join(GetProxyPath(productName), proxyName))
if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return nil
} else if err != nil {
return errors.Trace(err)
}
<-c
info, err := GetProxyInfo(zkConn, productName, proxyName)
log.Info("mark_offline, check proxy status:", proxyName, info, err)
if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
log.Info("shutdown proxy successful")
return nil
} else if err != nil {
return errors.Trace(err)
}
if info.State == PROXY_STATE_OFFLINE {
log.Info("proxy:", proxyName, "offline success!")
return nil
}
}
}
return nil
}
func loadTimestamp(conn zkhelper.Conn, rootPath string) (int64, error) {
data, _, err := conn.Get(getTimestampPath(rootPath))
if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return 0, zk.ErrNoNode
} else if err != nil {
return 0, errors.Trace(err)
} else if len(data) != 8 {
return 0, errors.Errorf("invalid timestamp data, must 8 bytes, but %d", len(data))
}
return int64(binary.BigEndian.Uint64(data)), nil
}
func saveTimestamp(conn zkhelper.Conn, rootPath string, ts int64) error {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], uint64(ts))
tsPath := getTimestampPath(rootPath)
_, err := conn.Set(tsPath, buf[:], -1)
if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
_, err = conn.Create(tsPath, buf[:], 0, zk.WorldACL(zkhelper.PERM_FILE))
}
return errors.Trace(err)
}
func apiSetProxyStatus(proxy models.ProxyInfo, param martini.Params) (int, string) {
conn := CreateZkConn()
defer conn.Close()
err := models.SetProxyStatus(conn, globalEnv.ProductName(), proxy.Id, proxy.State)
if err != nil {
// if this proxy is not online, just return success
if proxy.State == models.PROXY_STATE_MARK_OFFLINE && zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return jsonRetSucc()
}
log.Warning(errors.ErrorStack(err))
return 500, err.Error()
}
return jsonRetSucc()
}
func apiMigrateStatus() (int, string) {
conn := CreateZkConn()
defer conn.Close()
migrateSlots, err := models.GetMigratingSlots(conn, globalEnv.ProductName())
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return 500, err.Error()
}
b, err := json.MarshalIndent(map[string]interface{}{
"migrate_slots": migrateSlots,
"migrate_task": globalMigrateManager.runningTask,
}, " ", " ")
return 200, string(b)
}
func ForceRemoveLock(zkConn zkhelper.Conn, productName string) error {
lockPath := fmt.Sprintf("/zk/codis/db_%s/LOCK", productName)
children, _, err := zkConn.Children(lockPath)
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return errors.Trace(err)
}
for _, c := range children {
fullPath := path.Join(lockPath, c)
log.Info("deleting..", fullPath)
err := zkConn.Delete(fullPath, 0)
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func ProxyList(zkConn zkhelper.Conn, productName string, filter func(*ProxyInfo) bool) ([]ProxyInfo, error) {
ret := make([]ProxyInfo, 0)
root := GetProxyPath(productName)
proxies, _, err := zkConn.Children(root)
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return nil, errors.Trace(err)
}
for _, proxyName := range proxies {
pi, err := GetProxyInfo(zkConn, productName, proxyName)
if err != nil {
return nil, errors.Trace(err)
}
if filter == nil || filter(pi) == true {
ret = append(ret, *pi)
}
}
return ret, nil
}
func getAgents() ([]*agentInfo, error) {
basePath := agentPath()
children, _, err := globalConn.Children(basePath)
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return nil, errors.Trace(err)
}
sort.Strings(children)
var agents []*agentInfo
for _, child := range children {
if agent, err := getAgent(child); err != nil {
return nil, errors.Trace(err)
} else {
agents = append(agents, agent)
}
}
return agents, nil
}
func (tso *TimestampOracle) syncTimestamp() error {
last, err := loadTimestamp(tso.zkConn, tso.cfg.RootPath)
if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
// no timestamp node, create later
err = nil
}
if err != nil {
return errors.Trace(err)
}
var now time.Time
for {
now = time.Now()
since := (now.UnixNano() - last) / 1e6
if since <= 0 {
return errors.Errorf("%s <= last saved time %s", now, time.Unix(0, last))
}
if wait := 2*tso.cfg.SaveInterval - since; wait > 0 {
log.Warnf("wait %d milliseconds to guarantee valid generated timestamp", wait)
time.Sleep(time.Duration(wait) * time.Millisecond)
continue
}
break
}
if err = tso.saveTimestamp(now); err != nil {
return errors.Trace(err)
}
current := &atomicObject{
physical: now,
}
tso.ts.Store(current)
return nil
}
func apiOverview() (int, string) {
conn := CreateCoordConn()
defer conn.Close()
// get all server groups
groups, err := models.ServerGroups(conn, globalEnv.ProductName())
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return 500, err.Error()
}
var instances []string
for _, group := range groups {
for _, srv := range group.Servers {
if srv.Type == "master" {
instances = append(instances, srv.Addr)
}
}
}
info := make(map[string]interface{})
info["product"] = globalEnv.ProductName()
info["ops"] = proxiesSpeed
redisInfos := make([]map[string]string, 0)
if len(instances) > 0 {
for _, instance := range instances {
info, err := utils.GetRedisStat(instance, globalEnv.StoreAuth())
if err != nil {
log.Error(err)
}
redisInfos = append(redisInfos, info)
}
}
info["redis_infos"] = redisInfos
b, err := json.MarshalIndent(info, " ", " ")
return 200, string(b)
}
func SetProxyStatus(zkConn zkhelper.Conn, productName string, proxyName string, status string) error {
p, err := GetProxyInfo(zkConn, productName, proxyName)
if err != nil {
return errors.Trace(err)
}
if status != PROXY_STATE_ONLINE && status != PROXY_STATE_MARK_OFFLINE && status != PROXY_STATE_OFFLINE {
return errors.Errorf("%v, %s", ErrUnknownProxyStatus, status)
}
// check slot status before setting proxy online
if status == PROXY_STATE_ONLINE {
slots, err := Slots(zkConn, productName)
if err != nil {
return errors.Trace(err)
}
for _, slot := range slots {
if slot.State.Status != SLOT_STATUS_ONLINE {
return errors.Errorf("slot %v is not online", slot)
}
if slot.GroupId == INVALID_ID {
return errors.Errorf("slot %v has invalid group id", slot)
}
}
}
p.State = status
b, _ := json.Marshal(p)
_, err = zkConn.Set(path.Join(GetProxyPath(productName), proxyName), b, -1)
if err != nil {
return errors.Trace(err)
}
if status == PROXY_STATE_MARK_OFFLINE {
// wait for the proxy down
for {
_, _, c, err := zkConn.GetW(path.Join(GetProxyPath(productName), proxyName))
if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return nil
} else if err != nil {
return errors.Trace(err)
}
<-c
info, err := GetProxyInfo(zkConn, productName, proxyName)
log.Info("mark_offline, check proxy status:", proxyName, info, err)
if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
log.Info("shutdown proxy successful")
return nil
} else if err != nil {
return errors.Trace(err)
}
if info.State == PROXY_STATE_OFFLINE {
log.Info("proxy:", proxyName, "offline success!")
return nil
}
}
}
return nil
}
func ActionGC(zkConn zkhelper.Conn, productName string, gcType int, keep int) error {
prefix := GetWatchActionPath(productName)
respPrefix := GetActionResponsePath(productName)
exists, err := zkhelper.NodeExists(zkConn, prefix)
if err != nil {
return errors.Trace(err)
}
if !exists {
// if action path not exists just return nil
return nil
}
actions, _, err := zkConn.Children(prefix)
if err != nil {
return errors.Trace(err)
}
var act Action
currentTs := time.Now().Unix()
if gcType == GC_TYPE_N {
sort.Strings(actions)
// keep last 500 actions
if len(actions)-500 <= keep {
return nil
}
for _, action := range actions[:len(actions)-keep-500] {
if err := zkhelper.DeleteRecursive(zkConn, path.Join(prefix, action), -1); err != nil {
return errors.Trace(err)
}
err := zkhelper.DeleteRecursive(zkConn, path.Join(respPrefix, action), -1)
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return errors.Trace(err)
}
}
} else if gcType == GC_TYPE_SEC {
secs := keep
for _, action := range actions {
b, _, err := zkConn.Get(path.Join(prefix, action))
if err != nil {
return errors.Trace(err)
}
if err := json.Unmarshal(b, &act); err != nil {
return errors.Trace(err)
}
log.Info(action, act.Ts)
ts, _ := strconv.ParseInt(act.Ts, 10, 64)
if currentTs-ts > int64(secs) {
if err := zkhelper.DeleteRecursive(zkConn, path.Join(prefix, action), -1); err != nil {
return errors.Trace(err)
}
err := zkhelper.DeleteRecursive(zkConn, path.Join(respPrefix, action), -1)
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return errors.Trace(err)
}
}
}
}
return nil
}