func (self *ServerGroup) RemoveServer(zkConn zkhelper.Conn, addr string) error { zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, addr) data, _, err := zkConn.Get(zkPath) if err != nil { return errors.Trace(err) } var s Server err = json.Unmarshal(data, &s) if err != nil { return errors.Trace(err) } log.Info(s) if s.Type == SERVER_TYPE_MASTER { return errors.Errorf("cannot remove master, use promote first") } err = zkConn.Delete(zkPath, -1) if err != nil { return errors.Trace(err) } // update server list for i := 0; i < len(self.Servers); i++ { if self.Servers[i].Addr == s.Addr { self.Servers = append(self.Servers[:i], self.Servers[i+1:]...) break } } // remove slave won't need proxy confirm err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false) return errors.Trace(err) }
func ForceRemoveDeadFence(zkConn zkhelper.Conn, productName string) error { proxies, err := ProxyList(zkConn, productName, func(p *ProxyInfo) bool { return p.State == PROXY_STATE_ONLINE }) if err != nil { return errors.Trace(err) } fenceProxies, err := GetFenceProxyMap(zkConn, productName) if err != nil { return errors.Trace(err) } // remove online proxies's fence for _, proxy := range proxies { delete(fenceProxies, proxy.Addr) } // delete dead fence in zookeeper path := GetProxyFencePath(productName) for remainFence, _ := range fenceProxies { fencePath := filepath.Join(path, remainFence) log.Info("removing fence: ", fencePath) if err := zkhelper.DeleteRecursive(zkConn, fencePath, -1); err != nil { return errors.Trace(err) } } return nil }
func (s *Server) registerSignal() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM, os.Kill) go func() { <-c log.Info("ctrl-c or SIGTERM found, mark offline server") s.evtbus <- &killEvent{} }() }
func initReal() { conn = zkhelper.NewConn() conf = &Config{ proxyId: "proxy_test", productName: "test", zkAddr: "192.168.28.191:2181", fact: nil, proto: "tcp4", provider: "zookeeper", zkSessionTimeout: 30, zkReadTimeout: 30, } c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT, os.Kill) go func() { <-c log.Info("ctrl-c or SIGTERM found, bye bye...") s.Close() }() go func() { log.Info(http.ListenAndServe("192.168.28.192:6060", nil)) }() go func() { time.Sleep(10 * time.Second) zkConn, err := zkhelper.ConnectToZk(conf.zkAddr, 20000) if err != nil { log.Errorf("connect to zk: %+v", errors.Trace(err)) } else { err = models.SetProxyStatus(zkConn, conf.productName, conf.proxyId, models.PROXY_STATE_ONLINE) if err != nil { log.Errorf("set proxy error: %+v", errors.Trace(err)) } } zkConn.Close() }() go func() { err := http.ListenAndServe("192.168.28.192:11001", nil) log.PanicError(err, "http debug server quit") }() s = New("192.168.28.192:19001", "192.168.28.192:11001", conf) }
func (s *Session) Reset(c net.Conn, auth string, timeout int) { s.Conn.Reset(c) s.Ops = 0 s.CreateUnix = time.Now().Unix() s.LastOpUnix = 0 s.auth = auth s.authorized = false s.quit = false s.failed.Set(false) s.closed.Set(false) s.Conn.ReaderTimeout = time.Second * time.Duration(timeout) s.Conn.WriterTimeout = time.Second * 30 log.Info("session [%p] reused: %s", s, s) }
func (s *Server) SetMyselfOnline() error { log.Info("mark myself online") info := models.ProxyInfo{ Id: s.conf.proxyId, State: models.PROXY_STATE_ONLINE, } b, _ := json.Marshal(info) url := "http://" + s.conf.dashboardAddr + "/api/proxy" res, err := http.Post(url, "application/json", bytes.NewReader(b)) if err != nil { return err } if res.StatusCode != 200 { return errors.New("response code is not 200") } return nil }
func ForceRemoveLock(zkConn zkhelper.Conn, productName string) error { lockPath := fmt.Sprintf("/zk/codis/db_%s/LOCK", productName) children, _, err := zkConn.Children(lockPath) if err != nil && !zkhelper.ZkErrorEqual(err, zk.ErrNoNode) { return errors.Trace(err) } for _, c := range children { fullPath := path.Join(lockPath, c) log.Info("deleting..", fullPath) err := zkConn.Delete(fullPath, 0) if err != nil { return errors.Trace(err) } } return nil }
func (s *Server) serve() { defer s.close() if !s.waitOnline() { return } s.rewatchNodes() for i := 0; i < router.MaxSlotNum; i++ { s.fillSlot(i) } log.Info("proxy is serving") go func() { defer s.close() s.handleConns() }() s.loopEvents() }
func TestRefreshZkConn(t *testing.T) { initReal() time.Sleep(50 * time.Second) if s != nil { //s.top.RefreshZkConn() pi, err := s.topo.GetProxyInfo(s.info.Id) if err != nil { log.Error(err) } else { log.Infof("get proxy: %+v", pi) } log.Info("begin get") /* time.Sleep(15 * time.Second) content, err := s.top.WatchChildren(models.GetWatchActionPath(s.top.ProductName), s.evtbus) if err != nil { log.Error(fmt.Sprintf(errors.ErrorStack(err)+" zk error %+v", s.top.IsFatalErr(err))) } else { log.Infof("watch action: %+v", content) } e := <-s.evtbus log.Infof("event: %+v", e) */ for { pi, err = s.topo.GetProxyInfo(s.info.Id) if err != nil { log.Error(err) } else { log.Infof("get proxy: %+v", pi) } time.Sleep(5 * time.Second) } } }
func SetProxyStatus(zkConn zkhelper.Conn, productName string, proxyName string, status string) error { p, err := GetProxyInfo(zkConn, productName, proxyName) if err != nil { return errors.Trace(err) } if status != PROXY_STATE_ONLINE && status != PROXY_STATE_MARK_OFFLINE && status != PROXY_STATE_OFFLINE { return errors.Errorf("%v, %s", ErrUnknownProxyStatus, status) } // check slot status before setting proxy online if status == PROXY_STATE_ONLINE { slots, err := Slots(zkConn, productName) if err != nil { return errors.Trace(err) } for _, slot := range slots { if slot.State.Status != SLOT_STATUS_ONLINE && slot.State.Status != SLOT_STATUS_MIGRATE { return errors.Errorf("slot %v is not online or migrate", slot) } if slot.GroupId == INVALID_ID { return errors.Errorf("slot %v has invalid group id", slot) } } } p.State = status b, _ := json.Marshal(p) _, err = zkConn.Set(path.Join(GetProxyPath(productName), proxyName), b, -1) if err != nil { return errors.Trace(err) } if status == PROXY_STATE_MARK_OFFLINE { // wait for the proxy down for { _, _, c, err := zkConn.GetW(path.Join(GetProxyPath(productName), proxyName)) if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) { return nil } else if err != nil { return errors.Trace(err) } <-c info, err := GetProxyInfo(zkConn, productName, proxyName) log.Info("mark_offline, check proxy status:", proxyName, info, err) if zkhelper.ZkErrorEqual(err, zk.ErrNoNode) { log.Info("shutdown proxy successful") return nil } else if err != nil { return errors.Trace(err) } if info.State == PROXY_STATE_OFFLINE { log.Infof("proxy: %s offline success!", proxyName) return nil } } } return nil }
func main() { fmt.Print(banner) args, err := docopt.Parse(usage, nil, true, "codis proxy v0.1", true) if err != nil { fmt.Println(err) os.Exit(1) } // set config file if args["-c"] != nil { configFile = args["-c"].(string) } var maxFileFrag = 10 var maxFragSize int64 = bytesize.GB * 1 if s, ok := args["--log-filesize"].(string); ok && s != "" { v, err := bytesize.Parse(s) if err != nil { log.PanicErrorf(err, "invalid max log file size = %s", s) } maxFragSize = v } // set output log file if s, ok := args["-L"].(string); ok && s != "" { f, err := log.NewRollingFile(s, maxFileFrag, maxFragSize) if err != nil { log.PanicErrorf(err, "open rolling log file failed: %s", s) } else { defer f.Close() log.StdLog = log.New(f, "") } } log.SetLevel(log.LEVEL_INFO) // set log level if s, ok := args["--log-level"].(string); ok && s != "" { setLogLevel(s) } // set cpu if args["--cpu"] != nil { cpus, err = strconv.Atoi(args["--cpu"].(string)) if err != nil { log.PanicErrorf(err, "parse cpu number failed") } } // set addr if args["--addr"] != nil { addr = args["--addr"].(string) } // set http addr if args["--http-addr"] != nil { httpAddr = args["--http-addr"].(string) } checkUlimit(1024) runtime.GOMAXPROCS(cpus) http.HandleFunc("/setloglevel", handleSetLogLevel) go http.ListenAndServe(httpAddr, nil) log.Info("running on ", addr) conf, err := router.LoadConf(configFile) if err != nil { log.PanicErrorf(err, "load config failed") } s, err := router.NewServer(addr, httpAddr, conf) if err != nil { log.PanicErrorf(err, "create new server failed") } if err := s.Serve(); err != nil { log.PanicErrorf(err, "serve failed") } panic("exit") }
func NewServer(addr string, debugVarAddr string, conf *Config) (*Server, error) { log.Infof("start proxy with config: %+v", conf) s := &Server{ evtbus: make(chan interface{}, 1000), conf: conf, topo: topology.NewTopo(conf.productName, conf.zkAddr, conf.fact, conf.provider), pool: make(map[string]*SharedBackendConn), lastActionSeq: -1, } for i := 0; i < MaxSlotNum; i++ { s.slots[i] = &Slot{Id: i} } proxyHost := strings.Split(addr, ":")[0] debugHost := strings.Split(debugVarAddr, ":")[0] hostname, err := os.Hostname() if err != nil { log.PanicErrorf(err, "get host name failed") } if proxyHost == "0.0.0.0" || strings.HasPrefix(proxyHost, "127.0.0.") { proxyHost = hostname } if debugHost == "0.0.0.0" || strings.HasPrefix(debugHost, "127.0.0.") { debugHost = hostname } s.info.Id = conf.proxyId s.info.State = models.PROXY_STATE_OFFLINE s.info.Addr = proxyHost + ":" + strings.Split(addr, ":")[1] s.info.DebugVarAddr = debugHost + ":" + strings.Split(debugVarAddr, ":")[1] s.info.Pid = os.Getpid() s.info.StartAt = time.Now().String() log.Infof("proxy info = %+v", s.info) if l, err := net.Listen(conf.proto, addr); err != nil { return nil, errors.Trace(err) } else { s.Listener = l } stats.Publish("evtbus", stats.StringFunc(func() string { return strconv.Itoa(len(s.evtbus)) })) stats.PublishJSONFunc("router", func() string { var m = make(map[string]interface{}) m["ops"] = cmdstats.requests.Get() m["cmds"] = getAllOpStats() m["info"] = s.info m["build"] = map[string]interface{}{ "version": utils.Version, "compile": utils.Compile, } b, _ := json.Marshal(m) return string(b) }) s.RegisterAndWait() _, err = s.topo.WatchChildren(models.GetWatchActionPath(conf.productName), s.evtbus) if err != nil { log.PanicErrorf(err, "watch children failed") } for i := 0; i < MaxSlotNum; i++ { s.fillSlot(i, false) } go s.handleTopoEvent() log.Info("proxy start ok") return s, nil }
func main() { args, err := docopt.Parse(usage, nil, true, utils.Version, true) if err != nil { fmt.Println(err) os.Exit(1) } fmt.Print(banner) // set config file if args["-c"] != nil { configFile = args["-c"].(string) } var maxFileFrag = 10000000 var maxFragSize int64 = bytesize.GB * 1 if s, ok := args["--log-filesize"].(string); ok && s != "" { v, err := bytesize.Parse(s) if err != nil { log.PanicErrorf(err, "invalid max log file size = %s", s) } maxFragSize = v } // set output log file if s, ok := args["-L"].(string); ok && s != "" { f, err := log.NewRollingFile(s, maxFileFrag, maxFragSize) if err != nil { log.PanicErrorf(err, "open rolling log file failed: %s", s) } else { defer f.Close() log.StdLog = log.New(f, "") } } log.SetLevel(log.LEVEL_INFO) log.SetFlags(log.Flags() | log.Lshortfile) // set log level if s, ok := args["--log-level"].(string); ok && s != "" { setLogLevel(s) } cpus = runtime.NumCPU() // set cpu if args["--cpu"] != nil { cpus, err = strconv.Atoi(args["--cpu"].(string)) if err != nil { log.PanicErrorf(err, "parse cpu number failed") } } // set addr if args["--addr"] != nil { addr = args["--addr"].(string) } // set http addr if args["--http-addr"] != nil { httpAddr = args["--http-addr"].(string) } checkUlimit(1024) runtime.GOMAXPROCS(cpus) http.HandleFunc("/setloglevel", handleSetLogLevel) go func() { err := http.ListenAndServe(httpAddr, nil) log.PanicError(err, "http debug server quit") }() log.Info("running on ", addr) conf, err := proxy.LoadConf(configFile) if err != nil { log.PanicErrorf(err, "load config failed") } c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM, os.Kill) s := proxy.New(addr, httpAddr, conf) defer s.Close() stats.PublishJSONFunc("router", func() string { var m = make(map[string]interface{}) m["ops"] = router.OpCounts() m["cmds"] = router.GetAllOpStats() m["info"] = s.Info() m["build"] = map[string]interface{}{ "version": utils.Version, "compile": utils.Compile, } b, _ := json.Marshal(m) return string(b) }) go func() { <-c log.Info("ctrl-c or SIGTERM found, bye bye...") s.Close() }() time.Sleep(time.Second) if err := s.SetMyselfOnline(); err != nil { log.WarnError(err, "mark myself online fail, you need mark online manually by dashboard") } s.Join() log.Infof("proxy exit!! :(") }