func LoadConf(configFile string) (*Config, error) {
c, err := utils.InitConfigFromFile(configFile)
if err != nil {
log.PanicErrorf(err, "load config '%s' failed", configFile)
}
conf := &Config{}
conf.productName, _ = c.ReadString("product", "test")
if len(conf.productName) == 0 {
log.Panicf("invalid config: product entry is missing in %s", configFile)
}
conf.zkAddr, _ = c.ReadString("zk", "")
if len(conf.zkAddr) == 0 {
log.Panicf("invalid config: need zk entry is missing in %s", configFile)
}
conf.zkAddr = strings.TrimSpace(conf.zkAddr)
conf.proxyId, _ = c.ReadString("proxy_id", "")
if len(conf.proxyId) == 0 {
log.Panicf("invalid config: need proxy_id entry is missing in %s", configFile)
}
conf.netTimeout, _ = c.ReadInt("net_timeout", 5)
conf.proto, _ = c.ReadString("proto", "tcp")
conf.provider, _ = c.ReadString("coordinator", "zookeeper")
return conf, nil
}
func LoadConf(configFile string) (*Config, error) {
c := cfg.NewCfg(configFile)
if err := c.Load(); err != nil {
log.PanicErrorf(err, "load config '%s' failed", configFile)
}
conf := &Config{}
conf.productName, _ = c.ReadString("product", "test")
if len(conf.productName) == 0 {
log.Panicf("invalid config: product entry is missing in %s", configFile)
}
conf.dashboardAddr, _ = c.ReadString("dashboard_addr", "")
if conf.dashboardAddr == "" {
log.Panicf("invalid config: dashboard_addr is missing in %s", configFile)
}
conf.zkAddr, _ = c.ReadString("zk", "")
if len(conf.zkAddr) == 0 {
log.Panicf("invalid config: need zk entry is missing in %s", configFile)
}
conf.zkAddr = strings.TrimSpace(conf.zkAddr)
conf.passwd, _ = c.ReadString("password", "")
conf.proxyId, _ = c.ReadString("proxy_id", "")
if len(conf.proxyId) == 0 {
log.Panicf("invalid config: need proxy_id entry is missing in %s", configFile)
}
conf.proto, _ = c.ReadString("proto", "tcp")
conf.provider, _ = c.ReadString("coordinator", "zookeeper")
loadConfInt := func(entry string, defval int) int {
v, _ := c.ReadInt(entry, defval)
if v < 0 {
log.Panicf("invalid config: read %s = %d", entry, v)
}
return v
}
conf.pingPeriod = loadConfInt("backend_ping_period", 5)
conf.maxTimeout = loadConfInt("session_max_timeout", 1800)
conf.maxBufSize = loadConfInt("session_max_bufsize", 131072)
conf.maxPipeline = loadConfInt("session_max_pipeline", 1024)
conf.zkSessionTimeout = loadConfInt("zk_session_timeout", 30000)
if conf.zkSessionTimeout <= 100 {
conf.zkSessionTimeout *= 1000
log.Warn("zkSessionTimeout is to small, it is ms not second")
}
return conf, nil
}
func groupMaster(groupInfo models.ServerGroup) string {
var master string
for _, server := range groupInfo.Servers {
if server.Type == models.SERVER_TYPE_MASTER {
if master != "" {
log.Panicf("two master not allowed: %+v", groupInfo)
}
master = server.Addr
}
}
if master == "" {
log.Panicf("master not found: %+v", groupInfo)
}
return master
}
func LoadCodisEnv(cfg *cfg.Cfg) Env {
if cfg == nil {
log.Panicf("config is nil")
}
productName, err := cfg.ReadString("product", "test")
if err != nil {
log.PanicErrorf(err, "read product name failed")
}
zkAddr, err := cfg.ReadString("zk", "localhost:2181")
if err != nil {
log.PanicErrorf(err, "read zk address failed")
}
hostname, _ := os.Hostname()
dashboardAddr, err := cfg.ReadString("dashboard_addr", hostname+":18087")
if err != nil {
log.PanicErrorf(err, "read dashboard address failed")
}
provider, err := cfg.ReadString("coordinator", "zookeeper")
if err != nil {
log.PanicErrorf(err, "read coordinator failed")
}
return &CodisEnv{
zkAddr: zkAddr,
dashboardAddr: dashboardAddr,
productName: productName,
provider: provider,
}
}
func NewTopoWithNetConf(ProductName string, zkAddr string, f ZkFactory, provider string, connTimeout int, readTimeout int, zkSessionTimeout int) *Topology {
t := &Topology{
zkAddr: zkAddr,
ProductName: ProductName,
fact: f,
provider: provider,
zkSessionTimeout: zkSessionTimeout,
readTimeout: readTimeout,
connTimeout: connTimeout,
watchSuspend: &atomic2.Bool{},
}
t.watchSuspend.Set(false)
if t.fact == nil {
switch t.provider {
case "etcd":
t.fact = &zkhelper.EtcdConnector{}
case "zookeeper":
t.fact = &zkhelper.ZkConnector{}
default:
log.Panicf("coordinator not found in config")
}
}
t.InitZkConn()
return t
}
func (s *Server) checkAndDoTopoChange(seq int) bool {
act, err := s.topo.GetActionWithSeq(int64(seq))
if err != nil { //todo: error is not "not exist"
log.PanicErrorf(err, "action failed, seq = %d", seq)
}
if !needResponse(act.Receivers, s.info) { //no need to response
return false
}
log.Warnf("action %v receivers %v", seq, act.Receivers)
switch act.Type {
case models.ACTION_TYPE_SLOT_MIGRATE, models.ACTION_TYPE_SLOT_CHANGED,
models.ACTION_TYPE_SLOT_PREMIGRATE:
slot := &models.Slot{}
s.getActionObject(seq, slot)
s.fillSlot(slot.Id)
case models.ACTION_TYPE_SERVER_GROUP_CHANGED:
serverGroup := &models.ServerGroup{}
s.getActionObject(seq, serverGroup)
s.onGroupChange(serverGroup.Id)
case models.ACTION_TYPE_SERVER_GROUP_REMOVE:
//do not care
case models.ACTION_TYPE_MULTI_SLOT_CHANGED:
param := &models.SlotMultiSetParam{}
s.getActionObject(seq, param)
s.onSlotRangeChange(param)
default:
log.Panicf("unknown action %+v", act)
}
return true
}
func (s *Server) processAction(e interface{}) {
if strings.Index(getEventPath(e), models.GetProxyPath(s.topo.ProductName)) == 0 {
info, err := s.topo.GetProxyInfo(s.info.Id)
if err != nil {
log.PanicErrorf(err, "get proxy info failed: %s", s.info.Id)
}
switch info.State {
case models.PROXY_STATE_MARK_OFFLINE:
log.Infof("mark offline, proxy got offline event: %s", s.info.Id)
s.markOffline()
case models.PROXY_STATE_ONLINE:
s.rewatchProxy()
default:
log.Panicf("unknown proxy state %v", info)
}
return
}
//re-watch
nodes := s.rewatchNodes()
seqs, err := models.ExtraSeqList(nodes)
if err != nil {
log.PanicErrorf(err, "get seq list failed")
}
if len(seqs) == 0 || !s.topo.IsChildrenChangedEvent(e) {
return
}
//get last pos
index := -1
for i, seq := range seqs {
if s.lastActionSeq < seq {
index = i
//break
//only handle latest action
}
}
if index < 0 {
return
}
actions := seqs[index:]
for _, seq := range actions {
exist, err := s.topo.Exist(path.Join(s.topo.GetActionResponsePath(seq), s.info.Id))
if err != nil {
log.PanicErrorf(err, "get action failed")
}
if exist {
continue
}
if s.checkAndDoTopoChange(seq) {
s.responseAction(int64(seq))
}
}
s.lastActionSeq = seqs[len(seqs)-1]
}
func (s *SharedBackendConn) IncrRefcnt() {
s.mu.Lock()
defer s.mu.Unlock()
if s.refcnt == 0 {
log.Panicf("shared backend conn has been closed")
}
s.refcnt++
}
func (s *Server) handleMarkOffline() {
s.topo.Close(s.info.Id)
if s.OnSuicide == nil {
s.OnSuicide = func() error {
log.Panicf("proxy exit: %+v", s.info)
return nil
}
}
s.OnSuicide()
}
func checkUlimit(min int) {
ulimitN, err := exec.Command("/bin/sh", "-c", "ulimit -n").Output()
if err != nil {
log.WarnErrorf(err, "get ulimit failed")
}
n, err := strconv.Atoi(strings.TrimSpace(string(ulimitN)))
if err != nil || n < min {
log.Panicf("ulimit too small: %d, should be at least %d", n, min)
}
}
func (s *SharedBackendConn) Close() bool {
s.mu.Lock()
defer s.mu.Unlock()
if s.refcnt <= 0 {
log.Panicf("shared backend conn has been closed, close too many times")
}
if s.refcnt == 1 {
s.BackendConn.Close()
}
s.refcnt--
return s.refcnt == 0
}
func LoadConf(configFile string) (*Config, error) {
c, err := utils.InitConfigFromFile(configFile)
if err != nil {
log.PanicErrorf(err, "load config '%s' failed", configFile)
}
conf := &Config{}
conf.productName, _ = c.ReadString("product", "test")
if len(conf.productName) == 0 {
log.Panicf("invalid config: product entry is missing in %s", configFile)
}
conf.zkAddr, _ = c.ReadString("zk", "")
if len(conf.zkAddr) == 0 {
log.Panicf("invalid config: need zk entry is missing in %s", configFile)
}
conf.zkAddr = strings.TrimSpace(conf.zkAddr)
conf.passwd, _ = c.ReadString("requirepass", "")
conf.proxyId, _ = c.ReadString("proxy_id", "")
if len(conf.proxyId) == 0 {
log.Panicf("invalid config: need proxy_id entry is missing in %s", configFile)
}
conf.proto, _ = c.ReadString("proto", "tcp")
conf.provider, _ = c.ReadString("coordinator", "zookeeper")
loadConfInt := func(entry string, defInt int) int {
v, _ := c.ReadInt(entry, defInt)
if v < 0 {
log.Panicf("invalid config: read %s = %d", entry, v)
}
return v
}
conf.pingPeriod = loadConfInt("backend_ping_period", 5)
conf.maxTimeout = loadConfInt("session_max_timeout", 1800)
conf.maxBufSize = loadConfInt("session_max_bufsize", 1024*32)
conf.maxPipeline = loadConfInt("session_max_pipeline", 128)
return conf, nil
}
func (s *Server) onSlotRangeChange(param *models.SlotMultiSetParam) {
log.Infof("slotRangeChange %+v", param)
for i := param.From; i <= param.To; i++ {
switch param.Status {
case models.SLOT_STATUS_OFFLINE:
s.resetSlot(i)
case models.SLOT_STATUS_ONLINE:
s.fillSlot(i)
default:
log.Panicf("can not handle status %v", param.Status)
}
}
}
func (b *connBuilder) resetConnection() {
b.lock.Lock()
defer b.lock.Unlock()
if b.builder == nil {
log.Panicf("no connection builder")
}
if time.Now().Before(b.createdOn.Add(time.Second)) {
return
}
if b.connection != nil {
b.connection.Close()
}
var err error
b.connection, err = b.builder() // this is asnyc
if err == nil {
b.safeConnInstance.Conn = b.connection
b.unsafeConnInstance.Conn = b.connection
b.createdOn = time.Now()
return
}
log.Panicf("can not build new zk session, exit")
}
func NewGroup(groupInfo models.ServerGroup) *Group {
g := &Group{
redisServers: make(map[string]*models.Server),
}
for _, server := range groupInfo.Servers {
if server.Type == models.SERVER_TYPE_MASTER {
if len(g.master) > 0 {
log.Panicf("two master not allowed: %+v", groupInfo)
}
g.master = server.Addr
}
g.redisServers[server.Addr] = server
}
if len(g.master) == 0 {
log.Panicf("master not found: %+v", groupInfo)
}
return g
}
func NewTopo(ProductName string, zkAddr string, f ZkFactory, provider string) *Topology {
t := &Topology{zkAddr: zkAddr, ProductName: ProductName, fact: f, provider: provider}
if t.fact == nil {
switch t.provider {
case "etcd":
t.fact = zkhelper.NewEtcdConn
case "zookeeper":
t.fact = zkhelper.ConnectToZk
default:
log.Panicf("coordinator not found in config")
}
}
t.InitZkConn()
return t
}
func (top *Topology) doWatch(evtch <-chan topo.Event, evtbus chan interface{}) {
e := <-evtch
if e.State == topo.StateExpired || e.Type == topo.EventNotWatching {
log.Panicf("session expired: %+v", e)
}
log.Warnf("topo event %+v", e)
switch e.Type {
//case topo.EventNodeCreated:
//case topo.EventNodeDataChanged:
case topo.EventNodeChildrenChanged: //only care children changed
//todo:get changed node and decode event
default:
log.Warnf("%+v", e)
}
evtbus <- e
}
func (s *Server) fillSlot(i int) {
slotInfo, slotGroup, err := s.topo.GetSlotByIndex(i)
if err != nil {
log.PanicErrorf(err, "get slot by index failed", i)
}
var from string
var addr = groupMaster(*slotGroup)
if slotInfo.State.Status == models.SLOT_STATUS_MIGRATE {
fromGroup, err := s.topo.GetGroup(slotInfo.State.MigrateStatus.From)
if err != nil {
log.PanicErrorf(err, "get migrate from failed")
}
from = groupMaster(*fromGroup)
if from == addr {
log.Panicf("set slot %04d migrate from %s to %s", i, from, addr)
}
}
s.groups[i] = slotInfo.GroupId
s.router.FillSlot(i, addr, from,
slotInfo.State.Status == models.SLOT_STATUS_PRE_MIGRATE)
}
func LoadCodisEnv(cfg *cfg.Cfg) Env {
if cfg == nil {
log.Panicf("config is nil")
}
productName, err := cfg.ReadString("product", "test")
if err != nil {
log.PanicErrorf(err, "config: 'product' not found")
}
zkAddr, err := cfg.ReadString("zk", "localhost:2181")
if err != nil {
log.PanicErrorf(err, "config: 'zk' not found")
}
hostname, _ := os.Hostname()
dashboardAddr, err := cfg.ReadString("dashboard_addr", hostname+":18087")
if err != nil {
log.PanicErrorf(err, "config: 'dashboard_addr' not found")
}
provider, err := cfg.ReadString("coordinator", "zookeeper")
if err != nil {
log.PanicErrorf(err, "config: 'coordinator' not found")
}
passwd, _ := cfg.ReadString("password", "")
return &CodisEnv{
zkAddr: zkAddr,
passwd: passwd,
dashboardAddr: dashboardAddr,
productName: productName,
provider: provider,
}
}
func (c *unsafeConn) Close() {
log.Panicf("do not close zk connection by yourself")
}
func (s *Server) fillSlot(i int, force bool) {
if !s.isValidSlot(i) {
return
}
slot := s.slots[i]
if !force && slot.backend.bc != nil {
log.Panicf("slot %04d already filled, slot: %+v", i, slot)
}
slotInfo, slotGroup, err := s.topo.GetSlotByIndex(i)
if err != nil {
log.PanicErrorf(err, "get slot by index failed", i)
}
var from string
var addr = group.NewGroup(*slotGroup).Master()
if slotInfo.State.Status == models.SLOT_STATUS_MIGRATE {
fromGroup, err := s.topo.GetGroup(slotInfo.State.MigrateStatus.From)
if err != nil {
log.PanicErrorf(err, "get migrate from failed")
}
from = group.NewGroup(*fromGroup).Master()
if from == addr {
log.Panicf("set slot %04d migrate from %s to %s", i, from, addr)
}
}
slot.blockAndWait()
s.putBackendConn(slot.backend.bc)
s.putBackendConn(slot.migrate.bc)
slot.reset()
slot.Info, slot.Group = slotInfo, slotGroup
if len(addr) != 0 {
xx := strings.Split(addr, ":")
if len(xx) >= 1 {
slot.backend.host = []byte(xx[0])
}
if len(xx) >= 2 {
slot.backend.port = []byte(xx[1])
}
slot.backend.addr = addr
slot.backend.bc = s.getBackendConn(addr)
}
if len(from) != 0 {
slot.migrate.from = from
slot.migrate.bc = s.getBackendConn(from)
}
if slotInfo.State.Status != models.SLOT_STATUS_PRE_MIGRATE {
slot.unblock()
}
if slot.migrate.bc != nil {
log.Infof("fill slot %04d, force %v, backend.addr = %s, migrate.from = %s",
i, force, slot.backend.addr, slot.migrate.from)
} else {
log.Infof("fill slot %04d, force %v, backend.addr = %s",
i, force, slot.backend.addr)
}
}
func main() {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
signal.Notify(c, syscall.SIGTERM)
go func() {
<-c
if createdDashboardNode {
releaseDashboardNode()
}
log.Panicf("ctrl-c or SIGTERM found, exit")
}()
args, err := docopt.Parse(usage, nil, true, utils.Version, true)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
// set output log file
if s, ok := args["-L"].(string); ok && s != "" {
f, err := os.OpenFile(s, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
log.PanicErrorf(err, "open log file failed: %s", s)
} else {
defer f.Close()
log.StdLog = log.New(f, "")
}
}
log.SetLevel(log.LEVEL_INFO)
log.SetFlags(log.Flags() | log.Lshortfile)
// set log level
if s, ok := args["--log-level"].(string); ok && s != "" {
setLogLevel(s)
}
// set config file
var configFile string
if args["-c"] != nil {
configFile = args["-c"].(string)
} else {
configFile = "config.ini"
}
config := cfg.NewCfg(configFile)
if err := config.Load(); err != nil {
log.PanicErrorf(err, "load config file error")
}
// load global vars
globalEnv = LoadCodisEnv(config)
cmd := args["<command>"].(string)
cmdArgs := args["<args>"].([]string)
go http.ListenAndServe(":10086", nil)
err = runCommand(cmd, cmdArgs)
if err != nil {
log.PanicErrorf(err, "run sub-command failed")
}
}