func (s *Server) handleConns() {
ch := make(chan net.Conn, 4096)
defer close(ch)
go func() {
for c := range ch {
x := router.NewSessionSize(c, s.conf.passwd, s.conf.maxBufSize, s.conf.maxTimeout)
go x.Serve(s.router, s.conf.maxPipeline)
}
}()
for {
c, err := s.listener.Accept()
if err != nil {
if ne, ok := err.(net.Error); ok && ne.Temporary() {
log.WarnErrorf(err, "[%p] proxy accept new connection failed, get temporary error", s)
time.Sleep(time.Millisecond * 10)
continue
}
log.WarnErrorf(err, "[%p] proxy accept new connection failed, get non-temporary error, must shutdown", s)
return
} else {
ch <- c
}
}
}
func jsonRet(output map[string]interface{}) (int, string) {
b, err := json.Marshal(output)
if err != nil {
log.WarnErrorf(err, "to json failed")
}
return 200, string(b)
}
func getAllProxySlowop() map[string][]*router.SlowOpInfo {
proxies, err := models.ProxyList(unsafeZkConn, globalEnv.ProductName(), nil)
if err != nil {
log.ErrorErrorf(err, "get proxy list failed")
return nil
}
ret := make(map[string][]*router.SlowOpInfo)
for _, p := range proxies {
m, err := p.DebugVars()
if err != nil {
log.WarnErrorf(err, "get proxy debug varsfailed")
}
if m["router"] != nil {
route := m["router"].(map[string]interface{})
if route["slowop"] != nil {
ops := make([]*router.SlowOpInfo, 0, 10)
for _, elem := range route["slowop"].([]interface{}) {
op := elem.(map[string]interface{})
s := &router.SlowOpInfo{
Time: op["time"].(string),
Key: op["key"].(string),
Duration: int64(op["duration"].(float64)),
Reqs: int64(op["req"].(float64)),
}
ops = append(ops, s)
}
ret[p.Id] = ops
}
}
}
return ret
}
func (s *Server) handleTopoEvent() {
ticker := time.NewTicker(time.Second * 5)
defer ticker.Stop()
for {
select {
case e := <-s.evtbus:
switch e.(type) {
case *killEvent:
s.handleMarkOffline()
default:
evtPath := getEventPath(e)
log.Infof("got event %s, %v, lastActionSeq %d", s.info.Id, e, s.lastActionSeq)
if strings.Index(evtPath, models.GetActionResponsePath(s.conf.productName)) == 0 {
seq, err := strconv.Atoi(path.Base(evtPath))
if err != nil {
log.WarnErrorf(err, "parse action seq failed")
} else {
if seq < s.lastActionSeq {
log.Infof("ignore seq = %d", seq)
continue
}
}
}
log.Infof("got event %s, %v, lastActionSeq %d", s.info.Id, e, s.lastActionSeq)
s.processAction(e)
}
case <-ticker.C:
for _, bc := range s.pool {
bc.KeepAlive()
}
}
}
}
func checkUlimit(min int) {
ulimitN, err := exec.Command("/bin/sh", "-c", "ulimit -n").Output()
if err != nil {
log.WarnErrorf(err, "get ulimit failed")
}
n, err := strconv.Atoi(strings.TrimSpace(string(ulimitN)))
if err != nil || n < min {
log.Panicf("ulimit too small: %d, should be at least %d", n, min)
}
}
func (s *Server) RegisterAndWait() {
_, err := s.topo.CreateProxyInfo(&s.info)
if err != nil {
log.PanicErrorf(err, "create proxy node failed")
}
_, err = s.topo.CreateProxyFenceNode(&s.info)
if err != nil {
log.WarnErrorf(err, "create fence node failed")
}
s.registerSignal()
s.waitOnline()
}
func apiGetProxySlowop() (int, string) {
m := getAllProxySlowop()
if m == nil {
return 500, "Error getting proxy debug vars"
}
b, err := json.MarshalIndent(m, " ", " ")
if err != nil {
log.WarnErrorf(err, "to json failed")
return 500, err.Error()
}
return 200, string(b)
}
func (bc *BackendConn) Run() {
log.Infof("backend conn [%p] to %s, start service", bc, bc.addr)
for k := 0; ; k++ {
err := bc.loopWriter()
if err == nil {
break
} else {
for i := len(bc.input); i != 0; i-- {
r := <-bc.input
bc.setResponse(r, nil, err)
}
}
log.WarnErrorf(err, "backend conn [%p] to %s, restart [%d]", bc, bc.addr, k)
time.Sleep(time.Millisecond * 50)
}
log.Infof("backend conn [%p] to %s, stop and exit", bc, bc.addr)
}
func getAllProxyOps() int64 {
proxies, err := models.ProxyList(unsafeZkConn, globalEnv.ProductName(), nil)
if err != nil {
log.ErrorErrorf(err, "get proxy list failed")
return -1
}
var total int64
for _, p := range proxies {
i, err := p.Ops()
if err != nil {
log.WarnErrorf(err, "get proxy ops failed")
}
total += i
}
return total
}
// for debug
func getAllProxyDebugVars() map[string]map[string]interface{} {
proxies, err := models.ProxyList(unsafeZkConn, globalEnv.ProductName(), nil)
if err != nil {
log.ErrorErrorf(err, "get proxy list failed")
return nil
}
ret := make(map[string]map[string]interface{})
for _, p := range proxies {
m, err := p.DebugVars()
if err != nil {
log.WarnErrorf(err, "get proxy debug varsfailed")
}
ret[p.Id] = m
}
return ret
}
func runDashboard(addr string, httpLogFile string) {
log.Infof("dashboard listening on addr: %s", addr)
m := martini.Classic()
f, err := os.OpenFile(httpLogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.PanicErrorf(err, "open http log file failed")
}
defer f.Close()
m.Map(stdlog.New(f, "[martini]", stdlog.LstdFlags))
binRoot, err := filepath.Abs(filepath.Dir(os.Args[0]))
if err != nil {
log.PanicErrorf(err, "get binroot path failed")
}
m.Use(martini.Static(filepath.Join(binRoot, "assets/statics")))
m.Use(render.Renderer(render.Options{
Directory: filepath.Join(binRoot, "assets/template"),
Extensions: []string{".tmpl", ".html"},
Charset: "UTF-8",
IndentJSON: true,
}))
m.Use(cors.Allow(&cors.Options{
AllowOrigins: []string{"*"},
AllowMethods: []string{"POST", "GET", "DELETE", "PUT"},
AllowHeaders: []string{"Origin", "x-requested-with", "Content-Type", "Content-Range", "Content-Disposition", "Content-Description"},
ExposeHeaders: []string{"Content-Length"},
AllowCredentials: false,
}))
m.Get("/api/server_groups", apiGetServerGroupList)
m.Get("/api/overview", apiOverview)
m.Get("/api/redis/:addr/stat", apiRedisStat)
m.Get("/api/redis/:addr/:id/slotinfo", apiGetRedisSlotInfo)
m.Get("/api/redis/group/:group_id/:slot_id/slotinfo", apiGetRedisSlotInfoFromGroupId)
m.Put("/api/server_groups", binding.Json(models.ServerGroup{}), apiAddServerGroup)
m.Put("/api/server_group/(?P<id>[0-9]+)/addServer", binding.Json(models.Server{}), apiAddServerToGroup)
m.Delete("/api/server_group/(?P<id>[0-9]+)", apiRemoveServerGroup)
m.Put("/api/server_group/(?P<id>[0-9]+)/removeServer", binding.Json(models.Server{}), apiRemoveServerFromGroup)
m.Get("/api/server_group/(?P<id>[0-9]+)", apiGetServerGroup)
m.Post("/api/server_group/(?P<id>[0-9]+)/promote", binding.Json(models.Server{}), apiPromoteServer)
m.Get("/api/migrate/status", apiMigrateStatus)
m.Get("/api/migrate/tasks", apiGetMigrateTasks)
m.Post("/api/migrate", binding.Json(migrateTaskForm{}), apiDoMigrate)
m.Post("/api/rebalance", apiRebalance)
m.Get("/api/slot/list", apiGetSlots)
m.Get("/api/slot/:id", apiGetSingleSlot)
m.Post("/api/slots/init", apiInitSlots)
m.Get("/api/slots", apiGetSlots)
m.Post("/api/slot", binding.Json(RangeSetTask{}), apiSlotRangeSet)
m.Get("/api/proxy/list", apiGetProxyList)
m.Get("/api/proxy/debug/vars", apiGetProxyDebugVars)
m.Post("/api/proxy", binding.Json(models.ProxyInfo{}), apiSetProxyStatus)
m.Get("/api/action/gc", apiActionGC)
m.Get("/api/force_remove_locks", apiForceRemoveLocks)
m.Get("/api/remove_fence", apiRemoveFence)
//m.Get("/api/action/gc", apiActionGC)
m.Get("/slots", pageSlots)
m.Get("/", func(r render.Render) {
r.Redirect("/admin")
})
//check key slot correspondence
m.Get("/api/keyslot/(?P<key>.+)", apiKeySlot)
m.Get("/api/remove_migration", apiRemoveMigration)
m.Get("/api/remove_migration_fail", apiRemoveMigrationForFail)
m.Get("/api/proxy/slowop", apiGetProxySlowop)
zkBuilder := utils.NewConnBuilder(globalEnv.NewZkConn)
safeZkConn = zkBuilder.GetSafeConn()
unsafeZkConn = zkBuilder.GetUnsafeConn()
// create temp node in ZK
if err := createDashboardNode(); err != nil {
log.WarnErrorf(err, "create zk node failed") // do not release dashborad node here
}
// create long live migrate manager
globalMigrateManager = NewMigrateManager(safeZkConn, globalEnv.ProductName())
go func() {
tick := time.Tick(time.Second)
var lastCnt, qps int64
for _ = range tick {
cnt := getAllProxyOps()
if cnt > 0 {
qps = cnt - lastCnt
lastCnt = cnt
} else {
qps = 0
}
atomic.StoreInt64(&proxiesSpeed, qps)
}
}()
go func() {
for {
err := models.ActionGC(safeZkConn, globalEnv.ProductName(), models.GC_TYPE_SEC, 60*60*24)
if err != nil {
log.Warnf("clean actions failed %+v", err)
}
time.Sleep(60 * 60 * 24 * time.Second)
}
}()
m.RunOnAddr(addr)
}