func (self *ServerGroup) RemoveServer(zkConn zkhelper.Conn, addr string) error {
zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, addr)
data, _, err := zkConn.Get(zkPath)
if err != nil {
return errors.Trace(err)
}
var s Server
err = json.Unmarshal(data, &s)
if err != nil {
return errors.Trace(err)
}
log.Info(s)
if s.Type == SERVER_TYPE_MASTER {
return errors.Errorf("cannot remove master, use promote first")
}
err = zkConn.Delete(zkPath, -1)
if err != nil {
return errors.Trace(err)
}
// update server list
for i := 0; i < len(self.Servers); i++ {
if self.Servers[i].Addr == s.Addr {
self.Servers = append(self.Servers[:i], self.Servers[i+1:]...)
break
}
}
// remove slave won't need proxy confirm
err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
return errors.Trace(err)
}
func ForceRemoveDeadFence(zkConn zkhelper.Conn, productName string) error {
proxies, err := ProxyList(zkConn, productName, func(p *ProxyInfo) bool {
return p.State == PROXY_STATE_ONLINE
})
if err != nil {
return errors.Trace(err)
}
fenceProxies, err := GetFenceProxyMap(zkConn, productName)
if err != nil {
return errors.Trace(err)
}
// remove online proxies's fence
for _, proxy := range proxies {
delete(fenceProxies, proxy.Addr)
}
// delete dead fence in zookeeper
path := GetProxyFencePath(productName)
for remainFence, _ := range fenceProxies {
fencePath := filepath.Join(path, remainFence)
log.Info("removing fence: ", fencePath)
if err := zkhelper.DeleteRecursive(zkConn, fencePath, -1); err != nil {
return errors.Trace(err)
}
}
return nil
}
func (s *Server) registerSignal() {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM, os.Kill)
go func() {
<-c
log.Info("ctrl-c or SIGTERM found, mark offline server")
s.evtbus <- &killEvent{}
}()
}
func initReal() {
conn = zkhelper.NewConn()
conf = &Config{
proxyId: "proxy_test",
productName: "test",
zkAddr: "192.168.28.191:2181",
fact: nil,
proto: "tcp4",
provider: "zookeeper",
zkSessionTimeout: 30,
zkReadTimeout: 30,
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT, os.Kill)
go func() {
<-c
log.Info("ctrl-c or SIGTERM found, bye bye...")
s.Close()
}()
go func() {
log.Info(http.ListenAndServe("192.168.28.192:6060", nil))
}()
go func() {
time.Sleep(10 * time.Second)
zkConn, err := zkhelper.ConnectToZk(conf.zkAddr, 20000)
if err != nil {
log.Errorf("connect to zk: %+v", errors.Trace(err))
} else {
err = models.SetProxyStatus(zkConn, conf.productName, conf.proxyId, models.PROXY_STATE_ONLINE)
if err != nil {
log.Errorf("set proxy error: %+v", errors.Trace(err))
}
}
zkConn.Close()
}()
go func() {
err := http.ListenAndServe("192.168.28.192:11001", nil)
log.PanicError(err, "http debug server quit")
}()
s = New("192.168.28.192:19001", "192.168.28.192:11001", conf)
}
func (s *Session) Reset(c net.Conn, auth string, timeout int) {
s.Conn.Reset(c)
s.Ops = 0
s.CreateUnix = time.Now().Unix()
s.LastOpUnix = 0
s.auth = auth
s.authorized = false
s.quit = false
s.failed.Set(false)
s.closed.Set(false)
s.Conn.ReaderTimeout = time.Second * time.Duration(timeout)
s.Conn.WriterTimeout = time.Second * 30
log.Info("session [%p] reused: %s", s, s)
}
func (s *Server) SetMyselfOnline() error {
log.Info("mark myself online")
info := models.ProxyInfo{
Id: s.conf.proxyId,
State: models.PROXY_STATE_ONLINE,
}
b, _ := json.Marshal(info)
url := "http://" + s.conf.dashboardAddr + "/api/proxy"
res, err := http.Post(url, "application/json", bytes.NewReader(b))
if err != nil {
return err
}
if res.StatusCode != 200 {
return errors.New("response code is not 200")
}
return nil
}
func ForceRemoveLock(zkConn zkhelper.Conn, productName string) error {
lockPath := fmt.Sprintf("/zk/codis/db_%s/LOCK", productName)
children, _, err := zkConn.Children(lockPath)
if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return errors.Trace(err)
}
for _, c := range children {
fullPath := path.Join(lockPath, c)
log.Info("deleting..", fullPath)
err := zkConn.Delete(fullPath, 0)
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (s *Server) serve() {
defer s.close()
if !s.waitOnline() {
return
}
s.rewatchNodes()
for i := 0; i < router.MaxSlotNum; i++ {
s.fillSlot(i)
}
log.Info("proxy is serving")
go func() {
defer s.close()
s.handleConns()
}()
s.loopEvents()
}
func TestRefreshZkConn(t *testing.T) {
initReal()
time.Sleep(50 * time.Second)
if s != nil {
//s.top.RefreshZkConn()
pi, err := s.topo.GetProxyInfo(s.info.Id)
if err != nil {
log.Error(err)
} else {
log.Infof("get proxy: %+v", pi)
}
log.Info("begin get")
/*
time.Sleep(15 * time.Second)
content, err := s.top.WatchChildren(models.GetWatchActionPath(s.top.ProductName), s.evtbus)
if err != nil {
log.Error(fmt.Sprintf(errors.ErrorStack(err)+" zk error %+v", s.top.IsFatalErr(err)))
} else {
log.Infof("watch action: %+v", content)
}
e := <-s.evtbus
log.Infof("event: %+v", e)
*/
for {
pi, err = s.topo.GetProxyInfo(s.info.Id)
if err != nil {
log.Error(err)
} else {
log.Infof("get proxy: %+v", pi)
}
time.Sleep(5 * time.Second)
}
}
}
func SetProxyStatus(zkConn zkhelper.Conn, productName string, proxyName string, status string) error {
p, err := GetProxyInfo(zkConn, productName, proxyName)
if err != nil {
return errors.Trace(err)
}
if status != PROXY_STATE_ONLINE && status != PROXY_STATE_MARK_OFFLINE && status != PROXY_STATE_OFFLINE {
return errors.Errorf("%v, %s", ErrUnknownProxyStatus, status)
}
// check slot status before setting proxy online
if status == PROXY_STATE_ONLINE {
slots, err := Slots(zkConn, productName)
if err != nil {
return errors.Trace(err)
}
for _, slot := range slots {
if slot.State.Status != SLOT_STATUS_ONLINE && slot.State.Status != SLOT_STATUS_MIGRATE {
return errors.Errorf("slot %v is not online or migrate", slot)
}
if slot.GroupId == INVALID_ID {
return errors.Errorf("slot %v has invalid group id", slot)
}
}
}
p.State = status
b, _ := json.Marshal(p)
_, err = zkConn.Set(path.Join(GetProxyPath(productName), proxyName), b, -1)
if err != nil {
return errors.Trace(err)
}
if status == PROXY_STATE_MARK_OFFLINE {
// wait for the proxy down
for {
_, _, c, err := zkConn.GetW(path.Join(GetProxyPath(productName), proxyName))
if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
return nil
} else if err != nil {
return errors.Trace(err)
}
<-c
info, err := GetProxyInfo(zkConn, productName, proxyName)
log.Info("mark_offline, check proxy status:", proxyName, info, err)
if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) {
log.Info("shutdown proxy successful")
return nil
} else if err != nil {
return errors.Trace(err)
}
if info.State == PROXY_STATE_OFFLINE {
log.Infof("proxy: %s offline success!", proxyName)
return nil
}
}
}
return nil
}
func main() {
fmt.Print(banner)
args, err := docopt.Parse(usage, nil, true, "codis proxy v0.1", true)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
// set config file
if args["-c"] != nil {
configFile = args["-c"].(string)
}
var maxFileFrag = 10
var maxFragSize int64 = bytesize.GB * 1
if s, ok := args["--log-filesize"].(string); ok && s != "" {
v, err := bytesize.Parse(s)
if err != nil {
log.PanicErrorf(err, "invalid max log file size = %s", s)
}
maxFragSize = v
}
// set output log file
if s, ok := args["-L"].(string); ok && s != "" {
f, err := log.NewRollingFile(s, maxFileFrag, maxFragSize)
if err != nil {
log.PanicErrorf(err, "open rolling log file failed: %s", s)
} else {
defer f.Close()
log.StdLog = log.New(f, "")
}
}
log.SetLevel(log.LEVEL_INFO)
// set log level
if s, ok := args["--log-level"].(string); ok && s != "" {
setLogLevel(s)
}
// set cpu
if args["--cpu"] != nil {
cpus, err = strconv.Atoi(args["--cpu"].(string))
if err != nil {
log.PanicErrorf(err, "parse cpu number failed")
}
}
// set addr
if args["--addr"] != nil {
addr = args["--addr"].(string)
}
// set http addr
if args["--http-addr"] != nil {
httpAddr = args["--http-addr"].(string)
}
checkUlimit(1024)
runtime.GOMAXPROCS(cpus)
http.HandleFunc("/setloglevel", handleSetLogLevel)
go http.ListenAndServe(httpAddr, nil)
log.Info("running on ", addr)
conf, err := router.LoadConf(configFile)
if err != nil {
log.PanicErrorf(err, "load config failed")
}
s, err := router.NewServer(addr, httpAddr, conf)
if err != nil {
log.PanicErrorf(err, "create new server failed")
}
if err := s.Serve(); err != nil {
log.PanicErrorf(err, "serve failed")
}
panic("exit")
}
func NewServer(addr string, debugVarAddr string, conf *Config) (*Server, error) {
log.Infof("start proxy with config: %+v", conf)
s := &Server{
evtbus: make(chan interface{}, 1000),
conf: conf,
topo: topology.NewTopo(conf.productName, conf.zkAddr, conf.fact, conf.provider),
pool: make(map[string]*SharedBackendConn),
lastActionSeq: -1,
}
for i := 0; i < MaxSlotNum; i++ {
s.slots[i] = &Slot{Id: i}
}
proxyHost := strings.Split(addr, ":")[0]
debugHost := strings.Split(debugVarAddr, ":")[0]
hostname, err := os.Hostname()
if err != nil {
log.PanicErrorf(err, "get host name failed")
}
if proxyHost == "0.0.0.0" || strings.HasPrefix(proxyHost, "127.0.0.") {
proxyHost = hostname
}
if debugHost == "0.0.0.0" || strings.HasPrefix(debugHost, "127.0.0.") {
debugHost = hostname
}
s.info.Id = conf.proxyId
s.info.State = models.PROXY_STATE_OFFLINE
s.info.Addr = proxyHost + ":" + strings.Split(addr, ":")[1]
s.info.DebugVarAddr = debugHost + ":" + strings.Split(debugVarAddr, ":")[1]
s.info.Pid = os.Getpid()
s.info.StartAt = time.Now().String()
log.Infof("proxy info = %+v", s.info)
if l, err := net.Listen(conf.proto, addr); err != nil {
return nil, errors.Trace(err)
} else {
s.Listener = l
}
stats.Publish("evtbus", stats.StringFunc(func() string {
return strconv.Itoa(len(s.evtbus))
}))
stats.PublishJSONFunc("router", func() string {
var m = make(map[string]interface{})
m["ops"] = cmdstats.requests.Get()
m["cmds"] = getAllOpStats()
m["info"] = s.info
m["build"] = map[string]interface{}{
"version": utils.Version,
"compile": utils.Compile,
}
b, _ := json.Marshal(m)
return string(b)
})
s.RegisterAndWait()
_, err = s.topo.WatchChildren(models.GetWatchActionPath(conf.productName), s.evtbus)
if err != nil {
log.PanicErrorf(err, "watch children failed")
}
for i := 0; i < MaxSlotNum; i++ {
s.fillSlot(i, false)
}
go s.handleTopoEvent()
log.Info("proxy start ok")
return s, nil
}
func main() {
args, err := docopt.Parse(usage, nil, true, utils.Version, true)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Print(banner)
// set config file
if args["-c"] != nil {
configFile = args["-c"].(string)
}
var maxFileFrag = 10000000
var maxFragSize int64 = bytesize.GB * 1
if s, ok := args["--log-filesize"].(string); ok && s != "" {
v, err := bytesize.Parse(s)
if err != nil {
log.PanicErrorf(err, "invalid max log file size = %s", s)
}
maxFragSize = v
}
// set output log file
if s, ok := args["-L"].(string); ok && s != "" {
f, err := log.NewRollingFile(s, maxFileFrag, maxFragSize)
if err != nil {
log.PanicErrorf(err, "open rolling log file failed: %s", s)
} else {
defer f.Close()
log.StdLog = log.New(f, "")
}
}
log.SetLevel(log.LEVEL_INFO)
log.SetFlags(log.Flags() | log.Lshortfile)
// set log level
if s, ok := args["--log-level"].(string); ok && s != "" {
setLogLevel(s)
}
cpus = runtime.NumCPU()
// set cpu
if args["--cpu"] != nil {
cpus, err = strconv.Atoi(args["--cpu"].(string))
if err != nil {
log.PanicErrorf(err, "parse cpu number failed")
}
}
// set addr
if args["--addr"] != nil {
addr = args["--addr"].(string)
}
// set http addr
if args["--http-addr"] != nil {
httpAddr = args["--http-addr"].(string)
}
checkUlimit(1024)
runtime.GOMAXPROCS(cpus)
http.HandleFunc("/setloglevel", handleSetLogLevel)
go func() {
err := http.ListenAndServe(httpAddr, nil)
log.PanicError(err, "http debug server quit")
}()
log.Info("running on ", addr)
conf, err := proxy.LoadConf(configFile)
if err != nil {
log.PanicErrorf(err, "load config failed")
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM, os.Kill)
s := proxy.New(addr, httpAddr, conf)
defer s.Close()
stats.PublishJSONFunc("router", func() string {
var m = make(map[string]interface{})
m["ops"] = router.OpCounts()
m["cmds"] = router.GetAllOpStats()
m["info"] = s.Info()
m["build"] = map[string]interface{}{
"version": utils.Version,
"compile": utils.Compile,
}
b, _ := json.Marshal(m)
return string(b)
})
go func() {
<-c
log.Info("ctrl-c or SIGTERM found, bye bye...")
s.Close()
}()
time.Sleep(time.Second)
if err := s.SetMyselfOnline(); err != nil {
log.WarnError(err, "mark myself online fail, you need mark online manually by dashboard")
}
s.Join()
log.Infof("proxy exit!! :(")
}