// no need lock
func NewSwitchManager(context *cli.Context) error {
config, err := NewConfig(context)
if err != nil {
return err
}
sm = &switchManager{
config: config,
rwMutex: new(sync.RWMutex),
steps: []string{},
closed: true,
}
if sm.config.Domain == "" {
return errors.New("need Domain specify")
}
if sm.config.Name == "" {
return errors.New("need Name specify")
}
if sm.config.Port == "" {
return errors.New("need Port specify")
}
if sm.config.ProxyPort == "" {
return errors.New("need ProxyPort specify")
}
prefix = sm.config.Domain + "/" + sm.config.Name + "/"
cs, err := consul.NewConsul(context)
if err != nil {
sm.steps = append(sm.steps, fmt.Sprintf("Step1. New Consul fail:%s\n", err.Error()))
return err
}
sm.steps = append(sm.steps, "Step1. New Consul success\n")
sm.consul = cs
swm, err := swarm.NewSwarm(context, cs)
if err != nil {
sm.steps = append(sm.steps, fmt.Sprintf("Step2. New Swarm fail:%s\n", err.Error()))
return err
}
sm.steps = append(sm.steps, "Step2. New Swarm success\n")
sm.swarm = swm
// init from consul
isInitalized, err := sm.consul.GetVal(prefix + consts.InitKey)
if err == nil && isInitalized == "true" {
log.Info("init from consul")
err = initProxy()
if err != nil {
log.WithFields(log.Fields{
"err:": err.Error(),
}).Error("pull proxy from consul fail, return")
sm.steps = append(sm.steps, fmt.Sprintf("Step3. Init from consul fail:%s\n", err.Error()))
return err
}
err = initTopo()
if err != nil {
log.WithFields(log.Fields{
"err:": err.Error(),
}).Error("pull topology from consul fail, continue")
sm.steps = append(sm.steps, fmt.Sprintf("Step3. Init from consul fail:%s\n", err.Error()))
return err
}
err = initSwarm()
if err != nil {
log.WithFields(log.Fields{
"err:": err.Error(),
}).Error("pull swarm from consul fail, continue")
sm.steps = append(sm.steps, fmt.Sprintf("Step3. Init from consul fail:%s\n", err.Error()))
return err
}
sm.steps = append(sm.steps, "Step3. Init from consul success\n")
} else {
sm.steps = append(sm.steps, "Step3. No init from consul\n")
}
sm.closed = false
sm.state = 0
// recover action no need lock
actionVal, err := sm.consul.GetVal(prefix + consts.ActionKey)
if actionVal != "" {
log.Info("recover action:%s\n", actionVal)
split := strings.Split(actionVal, ",")
actionVal := split[0]
name := split[1]
if actionVal == consts.ActionIsolateProxyVal {
if sm.proxys[name].Status != consts.ProxyClose {
err = CloseProxy(name)
if err != nil {
sm.steps = append(sm.steps, fmt.Sprintf("Step4. Init recover action %s fail:%s\n", actionVal, err.Error()))
return err
}
bproxys, _ := json.Marshal(sm.proxys)
err = sm.consul.PutVal(prefix+consts.ProxyKey, bproxys)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.ProxyKey,
"val": string(bproxys),
"err": err.Error(),
}).Error("push proxy to consul fail, continue")
}
err = presistent.ProxyPresist(string(bproxys))
if err != nil {
log.WithFields(log.Fields{
"val": string(bproxys),
"err": err.Error(),
}).Error("proxy presist fail, continue")
}
}
err := sm.consul.PutVal(prefix+consts.ActionKey, []byte(""))
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.ActionKey,
"val": "",
"err": err.Error(),
}).Error("push action key to consul fail, continue")
}
sm.steps = append(sm.steps, "Step4. Init recover action %s success\n")
} else if actionVal == consts.ActionIsolateDbVal {
if sm.topology.DataNodeGroup["default"][name].Status != consts.Abnormal {
err, _ = ha.IsolateDB(name, sm.rwMutex, sm.topology, sm.swarm)
if err != nil {
sm.steps = append(sm.steps, fmt.Sprintf("Step4. Init recover action %s fail:%s\n", actionVal, err.Error()))
return err
}
btopology, _ := json.Marshal(sm.topology)
err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.TopologyKey,
"val": string(btopology),
"err": err.Error(),
}).Error("push topology to consul fail, continue")
}
err = presistent.TopologyPresist(string(btopology))
if err != nil {
log.WithFields(log.Fields{
"val": string(btopology),
"err": err.Error(),
}).Error("topology presist fail, continue")
}
}
err := sm.consul.PutVal(prefix+consts.ActionKey, []byte(""))
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.ActionKey,
"val": "",
"err": err.Error(),
}).Error("push action key to consul fail, continue")
}
sm.steps = append(sm.steps, "Step4. Init recover action %s success\n")
} else if actionVal == consts.ActionRecoverDbVal {
if sm.topology.DataNodeGroup["default"][name].Status != consts.Normal {
err, _ = ha.RecoverDb(name, sm.rwMutex, sm.topology, sm.swarm)
if err != nil {
sm.steps = append(sm.steps, fmt.Sprintf("Step4. Init recover action %s fail:%s\n", actionVal, err.Error()))
return err
}
btopology, _ := json.Marshal(sm.topology)
err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.TopologyKey,
"val": string(btopology),
"err": err.Error(),
}).Error("push topology to consul fail, continue")
}
err = presistent.TopologyPresist(string(btopology))
if err != nil {
log.WithFields(log.Fields{
"val": string(btopology),
"err": err.Error(),
}).Error("topology presist fail, continue")
}
}
err := sm.consul.PutVal(prefix+consts.ActionKey, []byte(""))
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.ActionKey,
"val": "",
"err": err.Error(),
}).Error("push action key to consul fail, continue")
}
sm.steps = append(sm.steps, "Step4. Init recover action %s success\n")
}
} else {
sm.steps = append(sm.steps, "Step4. No init recover action\n")
}
log.Info("sm init success")
return nil
}
func recover(w http.ResponseWriter, req *http.Request) {
var steps []string
vars := mux.Vars(req)
n := vars["name"]
log.WithFields(log.Fields{
"RemoteAddr": req.RemoteAddr,
"URL": req.URL,
"Name": n,
}).Info("handle `/recover/{name}` start")
var err error
defer func() {
log.WithFields(log.Fields{
"RemoteAddr": req.RemoteAddr,
"URL": req.URL,
"Name": n,
}).Info("handle `/recover/{name}` end")
if err != nil {
w.Write([]byte(err.Error()))
}
if len(sm.steps) != 0 {
s := ""
for _, v := range steps {
s = s + v
}
w.Write([]byte(s))
}
atomic.CompareAndSwapInt64(&sm.state, 2, 0)
}()
if !getLock(2) {
w.WriteHeader(http.StatusNotModified)
err = errors.New("http recover get lock fail")
return
}
var t string
sm.rwMutex.RLock()
_, ok := sm.proxys[n]
if ok {
if sm.proxys[n].Status != consts.ProxyClose {
w.WriteHeader(http.StatusBadRequest)
} else {
t = consts.Type_Proxy
}
} else {
_, ok = sm.topology.DataNodeGroup["default"][n]
if ok {
status := sm.topology.DataNodeGroup["default"][n].Status
if status == consts.Abnormal {
t = consts.Type_Db
} else {
w.WriteHeader(http.StatusBadRequest)
}
}
}
sm.rwMutex.RUnlock()
if t == consts.Type_Proxy {
sm.rwMutex.Lock()
if ok := sm.swarm.UpproxyStart(n); !ok {
sm.rwMutex.Unlock()
return
}
sm.proxys[n] = &structs.ProxyInfo{}
bproxy, _ := json.Marshal(sm.proxys)
sm.rwMutex.Unlock()
err = sm.consul.PutVal(prefix+consts.ProxyKey, bproxy)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.ProxyKey,
"val": string(bproxy),
"err": err.Error(),
}).Error("push proxy to consul fail, continue")
}
err = presistent.ProxyPresist(string(bproxy))
if err != nil {
log.WithFields(log.Fields{
"val": string(bproxy),
"err": err.Error(),
}).Error("proxy presist fail, continue")
}
w.WriteHeader(http.StatusOK)
} else if t == consts.Type_Db {
err = sm.consul.PutVal(prefix+consts.ActionKey, []byte(consts.ActionRecoverDbVal+","+n))
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
err, steps = ha.RecoverDb(n, sm.rwMutex, sm.topology, sm.swarm)
sm.steps = steps
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
} else {
w.WriteHeader(http.StatusOK)
}
sm.rwMutex.RLock()
btopology, _ := json.Marshal(sm.topology)
sm.rwMutex.RUnlock()
err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.TopologyKey,
"val": string(btopology),
"err": err.Error(),
}).Error("push topology to consul fail, continue")
}
err = presistent.TopologyPresist(string(btopology))
if err != nil {
log.WithFields(log.Fields{
"val": string(btopology),
"err": err.Error(),
}).Error("topology presist fail, continue")
}
err = sm.consul.PutVal(prefix+consts.ActionKey, []byte(""))
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.ActionKey,
"val": "",
"err": err.Error(),
}).Error("push action key to consul fail, continue")
}
} else {
w.WriteHeader(http.StatusBadRequest)
err = errors.New(http.StatusText(http.StatusBadRequest))
}
}
func initSm(w http.ResponseWriter, req *http.Request) {
log.WithFields(log.Fields{
"RemoteAddr": req.RemoteAddr,
"URL": req.URL,
}).Info("handle `/init` start")
var err error
var steps []string
defer func() {
log.WithFields(log.Fields{
"RemoteAddr": req.RemoteAddr,
"URL": req.URL,
}).Info("handle `/init` end")
if err != nil {
log.Error(err)
w.Write([]byte(err.Error()))
}
if len(sm.steps) != 0 {
s := ""
for _, v := range steps {
s = s + v
}
w.Write([]byte(s))
}
atomic.CompareAndSwapInt64(&sm.state, 2, 0)
}()
if !getLock(2) {
err = errors.New("init get lock fail")
return
}
var mgmPost structs.MgmPost
if err = json.NewDecoder(req.Body).Decode(&mgmPost); err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
log.WithFields(log.Fields{
"mgmPost": mgmPost,
}).Infoln("******mgmPost**********")
proxys := make(map[string]*structs.ProxyInfo)
for _, proxyName := range mgmPost.ProxyNames {
proxys[proxyName] = &structs.ProxyInfo{}
}
sm.rwMutex.Lock()
sm.swarm.ApiVersion = mgmPost.SwarmApiVersion
sm.swarm.DbRootUser = mgmPost.DbRootUser
sm.swarm.DbRootPassword = mgmPost.DbRootPassword
sm.swarm.DbReplicateUser = mgmPost.DbReplicateUser
sm.swarm.DbReplicatePassword = mgmPost.DbReplicatePassword
sm.rwMutex.Unlock()
var topo *structs.Topology
topo, err, steps = ha.InitTopo(mgmPost, sm.swarm)
sm.steps = steps
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
sm.rwMutex.Lock()
sm.proxys = proxys
sm.topology = topo
sm.rwMutex.Unlock()
w.WriteHeader(http.StatusOK)
bproxy, _ := json.Marshal(proxys)
// push to consul
err = sm.consul.PutVal(prefix+consts.ProxyKey, bproxy)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.ProxyKey,
"val": string(bproxy),
"err": err.Error(),
}).Error("push proxy to consul fail, continue")
}
err = presistent.ProxyPresist(string(bproxy))
if err != nil {
log.WithFields(log.Fields{
"val": string(bproxy),
"err": err.Error(),
}).Error("proxy presist fail, continue")
}
btopology, _ := json.Marshal(topo)
err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.TopologyKey,
"val": string(btopology),
"err": err.Error(),
}).Error("push topology to consul fail, continue")
}
err = presistent.TopologyPresist(string(btopology))
if err != nil {
log.WithFields(log.Fields{
"val": string(btopology),
"err": err.Error(),
}).Error("topology presist fail, continue")
}
bswarm, _ := json.Marshal(sm.swarm)
err = sm.consul.PutVal(prefix+consts.SwarmKey, bswarm)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.SwarmKey,
"val": string(bswarm),
"err": err.Error(),
}).Error("push swarm to consul fail, continue")
}
err = presistent.SwarmPresist(string(bswarm))
if err != nil {
log.WithFields(log.Fields{
"val": string(bswarm),
"err": err.Error(),
}).Error("swarm presist fail, continue")
}
err = sm.consul.PutVal(prefix+consts.InitKey, []byte("true"))
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.InitKey,
"val": "true",
"err": err.Error(),
}).Error("push init key to consul fail, continue")
}
}
func delUser(w http.ResponseWriter, req *http.Request) {
log.WithFields(log.Fields{
"RemoteAddr": req.RemoteAddr,
"URL": req.URL,
}).Info("handle `/delUser` start")
var user structs.User
var err error
defer func() {
log.WithFields(log.Fields{
"RemoteAddr": req.RemoteAddr,
"URL": req.URL,
}).Info("handle `/delUser` end")
if err != nil {
log.WithFields(log.Fields{
"user": user,
}).Error(err.Error())
w.Write([]byte(err.Error()))
}
}()
if err := json.NewDecoder(req.Body).Decode(&user); err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
sm.rwMutex.Lock()
defer sm.rwMutex.Unlock()
_, ok := sm.topology.ProxyUsers[user.UserName]
if !ok {
w.WriteHeader(http.StatusBadRequest)
err = fmt.Errorf("user %s not exist", user.UserName)
return
}
// del
delete(sm.topology.ProxyUsers, user.UserName)
for k, _ := range sm.topology.DatabaseAuth.ProxyDatabaseUserMap {
if k == user.UserName {
delete(sm.topology.DatabaseAuth.ProxyDatabaseUserMap, k)
}
}
// version+1
iversion, _ := strconv.Atoi(sm.topology.Version)
sm.topology.Version = strconv.Itoa(iversion + 1)
btopology, _ := json.Marshal(sm.topology)
err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.TopologyKey,
"val": string(btopology),
"err": err.Error(),
}).Error("push topology to consul fail, continue")
}
err = presistent.TopologyPresist(string(btopology))
if err != nil {
log.WithFields(log.Fields{
"val": string(btopology),
"err": err.Error(),
}).Error("topology presist fail, continue")
}
w.WriteHeader(http.StatusOK)
}
func uptUser(w http.ResponseWriter, req *http.Request) {
log.WithFields(log.Fields{
"RemoteAddr": req.RemoteAddr,
"URL": req.URL,
}).Info("handle `/uptUser` start")
var user structs.User
var err error
defer func() {
log.WithFields(log.Fields{
"RemoteAddr": req.RemoteAddr,
"URL": req.URL,
}).Info("handle `/uptUser` end")
if err != nil {
log.WithFields(log.Fields{
"user": user,
}).Error(err.Error())
w.Write([]byte(err.Error()))
}
}()
if err := json.NewDecoder(req.Body).Decode(&user); err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
sm.rwMutex.Lock()
defer sm.rwMutex.Unlock()
_, ok := sm.topology.ProxyUsers[user.UserName]
if !ok {
w.WriteHeader(http.StatusBadRequest)
err = fmt.Errorf("user %s not exist", user.UserName)
return
}
sm.topology.ProxyUsers[user.UserName] = &structs.AuthInfo{
Password: user.Password,
}
if user.ReadOnly {
sm.topology.ProxyUsers[user.UserName].ProxyMode = &structs.ProxyModeInfo{
IsShard: false,
IsRwSplit: false,
IsOnly: user.ReadOnly,
Datanode: "default",
}
}
if len(user.WhiteList) != 0 {
sm.topology.ProxyUsers[user.UserName].WhiteList = user.WhiteList
}
if len(user.BlackList) != 0 {
sm.topology.ProxyUsers[user.UserName].BlackList = user.BlackList
}
// del old map
for k, _ := range sm.topology.DatabaseAuth.ProxyDatabaseUserMap {
if k == user.UserName {
delete(sm.topology.DatabaseAuth.ProxyDatabaseUserMap, k)
}
}
// add new map
for k, v := range sm.topology.DatabaseAuth.DatabaseUsers {
if v.Role == user.Role {
sm.topology.DatabaseAuth.ProxyDatabaseUserMap[user.UserName] = k
}
}
// version+1
iversion, _ := strconv.Atoi(sm.topology.Version)
sm.topology.Version = strconv.Itoa(iversion + 1)
btopology, _ := json.Marshal(sm.topology)
err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.TopologyKey,
"val": string(btopology),
"err": err.Error(),
}).Error("push topology to consul fail, continue")
}
err = presistent.TopologyPresist(string(btopology))
if err != nil {
log.WithFields(log.Fields{
"val": string(btopology),
"err": err.Error(),
}).Error("topology presist fail, continue")
}
w.WriteHeader(http.StatusOK)
}
func HealthCheck() {
m := cmap.New()
for {
if atomic.LoadInt64(&sm.state) != 0 {
time.Sleep(sm.config.HealthCheckInterval)
continue
}
sm.rwMutex.RLock()
if sm.topology == nil {
sm.rwMutex.RUnlock()
time.Sleep(sm.config.HealthCheckInterval)
continue
}
for k, v := range sm.topology.DataNodeGroup["default"] {
serviceName := k
dbName := k
if v.Status == consts.Normal {
go func() {
health, err := sm.consul.HealthCheck(serviceName)
if err != nil {
log.WithFields(log.Fields{
"dbName": dbName,
}).Error("consul health check error, direct use swarm dbcheck")
health, err = sm.swarm.DbHealthCheck(serviceName)
if err != nil {
log.WithFields(log.Fields{
"dbName": dbName,
}).Error("swarm DbHealthCheck error")
}
}
if !health || err != nil {
log.WithFields(log.Fields{
"fail times": m.GetFail(dbName),
"dbName": dbName,
}).Warn("health check fail")
if ok := m.Fail(dbName); ok {
return
}
log.Error("toggle health check auto isolate")
if getLock(1) {
sm.steps = []string{}
// maybe already isolated at before goroute?
// no need rwMutex?
if sm.topology.DataNodeGroup["default"][dbName].Status != consts.Normal {
log.WithFields(log.Fields{
"dbName": dbName,
"status": sm.topology.DataNodeGroup["default"][dbName].Status,
}).Warn("db node already isolated")
atomic.CompareAndSwapInt64(&sm.state, 1, 0)
return
}
err, steps := ha.IsolateDB(dbName, sm.rwMutex, sm.topology, sm.swarm)
sm.steps = steps
atomic.CompareAndSwapInt64(&sm.state, 1, 0)
if err != nil {
log.WithFields(log.Fields{
"dbName": dbName,
}).Error("health check auto isolate fail")
}
sm.rwMutex.RLock()
btopology, _ := json.Marshal(sm.topology)
sm.rwMutex.RUnlock()
err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
if err != nil {
log.WithFields(log.Fields{
"key": prefix + consts.TopologyKey,
"val": string(btopology),
"err": err.Error(),
}).Error("push topology to consul fail, continue")
}
err = presistent.TopologyPresist(string(btopology))
if err != nil {
log.WithFields(log.Fields{
"val": string(btopology),
"err": err.Error(),
}).Error("topology presist fail, continue")
}
} else {
log.WithFields(log.Fields{
"dbname": dbName,
}).Error("health check auto isolate get lock fail")
}
} else {
m.Success(dbName)
}
}()
}
}
sm.rwMutex.RUnlock()
time.Sleep(sm.config.HealthCheckInterval)
}
}