Example #1
0
File: sm.go Project: yiduoyunQ/sm
// no need lock
func NewSwitchManager(context *cli.Context) error {
	config, err := NewConfig(context)
	if err != nil {
		return err
	}
	sm = &switchManager{
		config:  config,
		rwMutex: new(sync.RWMutex),
		steps:   []string{},
		closed:  true,
	}

	if sm.config.Domain == "" {
		return errors.New("need Domain specify")
	}
	if sm.config.Name == "" {
		return errors.New("need Name specify")
	}
	if sm.config.Port == "" {
		return errors.New("need Port specify")
	}
	if sm.config.ProxyPort == "" {
		return errors.New("need ProxyPort specify")
	}

	prefix = sm.config.Domain + "/" + sm.config.Name + "/"

	cs, err := consul.NewConsul(context)
	if err != nil {
		sm.steps = append(sm.steps, fmt.Sprintf("Step1. New Consul fail:%s\n", err.Error()))
		return err
	}
	sm.steps = append(sm.steps, "Step1. New Consul success\n")
	sm.consul = cs
	swm, err := swarm.NewSwarm(context, cs)
	if err != nil {
		sm.steps = append(sm.steps, fmt.Sprintf("Step2. New Swarm fail:%s\n", err.Error()))
		return err
	}
	sm.steps = append(sm.steps, "Step2. New Swarm success\n")
	sm.swarm = swm

	// init from consul
	isInitalized, err := sm.consul.GetVal(prefix + consts.InitKey)
	if err == nil && isInitalized == "true" {
		log.Info("init from consul")
		err = initProxy()
		if err != nil {
			log.WithFields(log.Fields{
				"err:": err.Error(),
			}).Error("pull proxy from consul fail, return")
			sm.steps = append(sm.steps, fmt.Sprintf("Step3. Init from consul fail:%s\n", err.Error()))
			return err
		}
		err = initTopo()
		if err != nil {
			log.WithFields(log.Fields{
				"err:": err.Error(),
			}).Error("pull topology from consul fail, continue")
			sm.steps = append(sm.steps, fmt.Sprintf("Step3. Init from consul fail:%s\n", err.Error()))
			return err
		}
		err = initSwarm()
		if err != nil {
			log.WithFields(log.Fields{
				"err:": err.Error(),
			}).Error("pull swarm from consul fail, continue")
			sm.steps = append(sm.steps, fmt.Sprintf("Step3. Init from consul fail:%s\n", err.Error()))
			return err
		}
		sm.steps = append(sm.steps, "Step3. Init from consul success\n")
	} else {
		sm.steps = append(sm.steps, "Step3. No init from consul\n")
	}

	sm.closed = false
	sm.state = 0

	// recover action no need lock
	actionVal, err := sm.consul.GetVal(prefix + consts.ActionKey)
	if actionVal != "" {
		log.Info("recover action:%s\n", actionVal)
		split := strings.Split(actionVal, ",")
		actionVal := split[0]
		name := split[1]
		if actionVal == consts.ActionIsolateProxyVal {
			if sm.proxys[name].Status != consts.ProxyClose {
				err = CloseProxy(name)
				if err != nil {
					sm.steps = append(sm.steps, fmt.Sprintf("Step4. Init recover action %s fail:%s\n", actionVal, err.Error()))
					return err
				}
				bproxys, _ := json.Marshal(sm.proxys)
				err = sm.consul.PutVal(prefix+consts.ProxyKey, bproxys)
				if err != nil {
					log.WithFields(log.Fields{
						"key": prefix + consts.ProxyKey,
						"val": string(bproxys),
						"err": err.Error(),
					}).Error("push proxy to consul fail, continue")
				}
				err = presistent.ProxyPresist(string(bproxys))
				if err != nil {
					log.WithFields(log.Fields{
						"val": string(bproxys),
						"err": err.Error(),
					}).Error("proxy presist fail, continue")
				}
			}
			err := sm.consul.PutVal(prefix+consts.ActionKey, []byte(""))
			if err != nil {
				log.WithFields(log.Fields{
					"key": prefix + consts.ActionKey,
					"val": "",
					"err": err.Error(),
				}).Error("push action key to consul fail, continue")
			}
			sm.steps = append(sm.steps, "Step4. Init recover action %s success\n")
		} else if actionVal == consts.ActionIsolateDbVal {
			if sm.topology.DataNodeGroup["default"][name].Status != consts.Abnormal {
				err, _ = ha.IsolateDB(name, sm.rwMutex, sm.topology, sm.swarm)
				if err != nil {
					sm.steps = append(sm.steps, fmt.Sprintf("Step4. Init recover action %s fail:%s\n", actionVal, err.Error()))
					return err
				}
				btopology, _ := json.Marshal(sm.topology)
				err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
				if err != nil {
					log.WithFields(log.Fields{
						"key": prefix + consts.TopologyKey,
						"val": string(btopology),
						"err": err.Error(),
					}).Error("push topology to consul fail, continue")
				}
				err = presistent.TopologyPresist(string(btopology))
				if err != nil {
					log.WithFields(log.Fields{
						"val": string(btopology),
						"err": err.Error(),
					}).Error("topology presist fail, continue")
				}
			}
			err := sm.consul.PutVal(prefix+consts.ActionKey, []byte(""))
			if err != nil {
				log.WithFields(log.Fields{
					"key": prefix + consts.ActionKey,
					"val": "",
					"err": err.Error(),
				}).Error("push action key to consul fail, continue")
			}
			sm.steps = append(sm.steps, "Step4. Init recover action %s success\n")
		} else if actionVal == consts.ActionRecoverDbVal {
			if sm.topology.DataNodeGroup["default"][name].Status != consts.Normal {
				err, _ = ha.RecoverDb(name, sm.rwMutex, sm.topology, sm.swarm)
				if err != nil {
					sm.steps = append(sm.steps, fmt.Sprintf("Step4. Init recover action %s fail:%s\n", actionVal, err.Error()))
					return err
				}
				btopology, _ := json.Marshal(sm.topology)
				err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
				if err != nil {
					log.WithFields(log.Fields{
						"key": prefix + consts.TopologyKey,
						"val": string(btopology),
						"err": err.Error(),
					}).Error("push topology to consul fail, continue")
				}
				err = presistent.TopologyPresist(string(btopology))
				if err != nil {
					log.WithFields(log.Fields{
						"val": string(btopology),
						"err": err.Error(),
					}).Error("topology presist fail, continue")
				}
			}
			err := sm.consul.PutVal(prefix+consts.ActionKey, []byte(""))
			if err != nil {
				log.WithFields(log.Fields{
					"key": prefix + consts.ActionKey,
					"val": "",
					"err": err.Error(),
				}).Error("push action key to consul fail, continue")
			}
			sm.steps = append(sm.steps, "Step4. Init recover action %s success\n")
		}
	} else {
		sm.steps = append(sm.steps, "Step4. No init recover action\n")
	}

	log.Info("sm init success")

	return nil
}
Example #2
0
func recover(w http.ResponseWriter, req *http.Request) {
	var steps []string
	vars := mux.Vars(req)
	n := vars["name"]
	log.WithFields(log.Fields{
		"RemoteAddr": req.RemoteAddr,
		"URL":        req.URL,
		"Name":       n,
	}).Info("handle `/recover/{name}` start")
	var err error
	defer func() {
		log.WithFields(log.Fields{
			"RemoteAddr": req.RemoteAddr,
			"URL":        req.URL,
			"Name":       n,
		}).Info("handle `/recover/{name}` end")

		if err != nil {
			w.Write([]byte(err.Error()))
		}

		if len(sm.steps) != 0 {
			s := ""
			for _, v := range steps {
				s = s + v
			}
			w.Write([]byte(s))
		}

		atomic.CompareAndSwapInt64(&sm.state, 2, 0)
	}()

	if !getLock(2) {
		w.WriteHeader(http.StatusNotModified)
		err = errors.New("http recover get lock fail")
		return
	}

	var t string
	sm.rwMutex.RLock()
	_, ok := sm.proxys[n]
	if ok {
		if sm.proxys[n].Status != consts.ProxyClose {
			w.WriteHeader(http.StatusBadRequest)
		} else {
			t = consts.Type_Proxy
		}

	} else {
		_, ok = sm.topology.DataNodeGroup["default"][n]
		if ok {
			status := sm.topology.DataNodeGroup["default"][n].Status
			if status == consts.Abnormal {
				t = consts.Type_Db
			} else {
				w.WriteHeader(http.StatusBadRequest)
			}
		}
	}
	sm.rwMutex.RUnlock()

	if t == consts.Type_Proxy {
		sm.rwMutex.Lock()
		if ok := sm.swarm.UpproxyStart(n); !ok {
			sm.rwMutex.Unlock()
			return
		}
		sm.proxys[n] = &structs.ProxyInfo{}
		bproxy, _ := json.Marshal(sm.proxys)
		sm.rwMutex.Unlock()
		err = sm.consul.PutVal(prefix+consts.ProxyKey, bproxy)
		if err != nil {
			log.WithFields(log.Fields{
				"key": prefix + consts.ProxyKey,
				"val": string(bproxy),
				"err": err.Error(),
			}).Error("push proxy to consul fail, continue")
		}
		err = presistent.ProxyPresist(string(bproxy))
		if err != nil {
			log.WithFields(log.Fields{
				"val": string(bproxy),
				"err": err.Error(),
			}).Error("proxy presist fail, continue")
		}

		w.WriteHeader(http.StatusOK)
	} else if t == consts.Type_Db {
		err = sm.consul.PutVal(prefix+consts.ActionKey, []byte(consts.ActionRecoverDbVal+","+n))
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		err, steps = ha.RecoverDb(n, sm.rwMutex, sm.topology, sm.swarm)
		sm.steps = steps
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
		} else {
			w.WriteHeader(http.StatusOK)
		}
		sm.rwMutex.RLock()
		btopology, _ := json.Marshal(sm.topology)
		sm.rwMutex.RUnlock()
		err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
		if err != nil {
			log.WithFields(log.Fields{
				"key": prefix + consts.TopologyKey,
				"val": string(btopology),
				"err": err.Error(),
			}).Error("push topology to consul fail, continue")
		}
		err = presistent.TopologyPresist(string(btopology))
		if err != nil {
			log.WithFields(log.Fields{
				"val": string(btopology),
				"err": err.Error(),
			}).Error("topology presist fail, continue")
		}
		err = sm.consul.PutVal(prefix+consts.ActionKey, []byte(""))
		if err != nil {
			log.WithFields(log.Fields{
				"key": prefix + consts.ActionKey,
				"val": "",
				"err": err.Error(),
			}).Error("push action key to consul fail, continue")
		}
	} else {
		w.WriteHeader(http.StatusBadRequest)
		err = errors.New(http.StatusText(http.StatusBadRequest))
	}

}
Example #3
0
func initSm(w http.ResponseWriter, req *http.Request) {
	log.WithFields(log.Fields{
		"RemoteAddr": req.RemoteAddr,
		"URL":        req.URL,
	}).Info("handle `/init` start")
	var err error
	var steps []string
	defer func() {
		log.WithFields(log.Fields{
			"RemoteAddr": req.RemoteAddr,
			"URL":        req.URL,
		}).Info("handle `/init` end")

		if err != nil {
			log.Error(err)
			w.Write([]byte(err.Error()))
		}

		if len(sm.steps) != 0 {
			s := ""
			for _, v := range steps {
				s = s + v
			}
			w.Write([]byte(s))
		}
		atomic.CompareAndSwapInt64(&sm.state, 2, 0)
	}()

	if !getLock(2) {
		err = errors.New("init get lock fail")
		return
	}

	var mgmPost structs.MgmPost
	if err = json.NewDecoder(req.Body).Decode(&mgmPost); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	log.WithFields(log.Fields{
		"mgmPost": mgmPost,
	}).Infoln("******mgmPost**********")

	proxys := make(map[string]*structs.ProxyInfo)
	for _, proxyName := range mgmPost.ProxyNames {
		proxys[proxyName] = &structs.ProxyInfo{}
	}

	sm.rwMutex.Lock()
	sm.swarm.ApiVersion = mgmPost.SwarmApiVersion
	sm.swarm.DbRootUser = mgmPost.DbRootUser
	sm.swarm.DbRootPassword = mgmPost.DbRootPassword
	sm.swarm.DbReplicateUser = mgmPost.DbReplicateUser
	sm.swarm.DbReplicatePassword = mgmPost.DbReplicatePassword
	sm.rwMutex.Unlock()

	var topo *structs.Topology
	topo, err, steps = ha.InitTopo(mgmPost, sm.swarm)
	sm.steps = steps
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	sm.rwMutex.Lock()
	sm.proxys = proxys
	sm.topology = topo
	sm.rwMutex.Unlock()

	w.WriteHeader(http.StatusOK)

	bproxy, _ := json.Marshal(proxys)
	// push to consul
	err = sm.consul.PutVal(prefix+consts.ProxyKey, bproxy)
	if err != nil {
		log.WithFields(log.Fields{
			"key": prefix + consts.ProxyKey,
			"val": string(bproxy),
			"err": err.Error(),
		}).Error("push proxy to consul fail, continue")
	}
	err = presistent.ProxyPresist(string(bproxy))
	if err != nil {
		log.WithFields(log.Fields{
			"val": string(bproxy),
			"err": err.Error(),
		}).Error("proxy presist fail, continue")
	}

	btopology, _ := json.Marshal(topo)
	err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
	if err != nil {
		log.WithFields(log.Fields{
			"key": prefix + consts.TopologyKey,
			"val": string(btopology),
			"err": err.Error(),
		}).Error("push topology to consul fail, continue")
	}
	err = presistent.TopologyPresist(string(btopology))
	if err != nil {
		log.WithFields(log.Fields{
			"val": string(btopology),
			"err": err.Error(),
		}).Error("topology presist fail, continue")
	}

	bswarm, _ := json.Marshal(sm.swarm)
	err = sm.consul.PutVal(prefix+consts.SwarmKey, bswarm)
	if err != nil {
		log.WithFields(log.Fields{
			"key": prefix + consts.SwarmKey,
			"val": string(bswarm),
			"err": err.Error(),
		}).Error("push swarm to consul fail, continue")
	}
	err = presistent.SwarmPresist(string(bswarm))
	if err != nil {
		log.WithFields(log.Fields{
			"val": string(bswarm),
			"err": err.Error(),
		}).Error("swarm presist fail, continue")
	}

	err = sm.consul.PutVal(prefix+consts.InitKey, []byte("true"))
	if err != nil {
		log.WithFields(log.Fields{
			"key": prefix + consts.InitKey,
			"val": "true",
			"err": err.Error(),
		}).Error("push init key to consul fail, continue")
	}
}
Example #4
0
func delUser(w http.ResponseWriter, req *http.Request) {
	log.WithFields(log.Fields{
		"RemoteAddr": req.RemoteAddr,
		"URL":        req.URL,
	}).Info("handle `/delUser` start")
	var user structs.User
	var err error
	defer func() {
		log.WithFields(log.Fields{
			"RemoteAddr": req.RemoteAddr,
			"URL":        req.URL,
		}).Info("handle `/delUser` end")
		if err != nil {
			log.WithFields(log.Fields{
				"user": user,
			}).Error(err.Error())
			w.Write([]byte(err.Error()))
		}
	}()

	if err := json.NewDecoder(req.Body).Decode(&user); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	sm.rwMutex.Lock()
	defer sm.rwMutex.Unlock()
	_, ok := sm.topology.ProxyUsers[user.UserName]
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		err = fmt.Errorf("user %s not exist", user.UserName)
		return
	}

	// del
	delete(sm.topology.ProxyUsers, user.UserName)
	for k, _ := range sm.topology.DatabaseAuth.ProxyDatabaseUserMap {
		if k == user.UserName {
			delete(sm.topology.DatabaseAuth.ProxyDatabaseUserMap, k)
		}
	}

	// version+1
	iversion, _ := strconv.Atoi(sm.topology.Version)
	sm.topology.Version = strconv.Itoa(iversion + 1)

	btopology, _ := json.Marshal(sm.topology)

	err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
	if err != nil {
		log.WithFields(log.Fields{
			"key": prefix + consts.TopologyKey,
			"val": string(btopology),
			"err": err.Error(),
		}).Error("push topology to consul fail, continue")
	}
	err = presistent.TopologyPresist(string(btopology))
	if err != nil {
		log.WithFields(log.Fields{
			"val": string(btopology),
			"err": err.Error(),
		}).Error("topology presist fail, continue")
	}

	w.WriteHeader(http.StatusOK)
}
Example #5
0
func uptUser(w http.ResponseWriter, req *http.Request) {
	log.WithFields(log.Fields{
		"RemoteAddr": req.RemoteAddr,
		"URL":        req.URL,
	}).Info("handle `/uptUser` start")
	var user structs.User
	var err error
	defer func() {
		log.WithFields(log.Fields{
			"RemoteAddr": req.RemoteAddr,
			"URL":        req.URL,
		}).Info("handle `/uptUser` end")
		if err != nil {
			log.WithFields(log.Fields{
				"user": user,
			}).Error(err.Error())
			w.Write([]byte(err.Error()))
		}
	}()

	if err := json.NewDecoder(req.Body).Decode(&user); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	sm.rwMutex.Lock()
	defer sm.rwMutex.Unlock()
	_, ok := sm.topology.ProxyUsers[user.UserName]
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		err = fmt.Errorf("user %s not exist", user.UserName)
		return
	}

	sm.topology.ProxyUsers[user.UserName] = &structs.AuthInfo{
		Password: user.Password,
	}
	if user.ReadOnly {
		sm.topology.ProxyUsers[user.UserName].ProxyMode = &structs.ProxyModeInfo{
			IsShard:   false,
			IsRwSplit: false,
			IsOnly:    user.ReadOnly,
			Datanode:  "default",
		}
	}

	if len(user.WhiteList) != 0 {
		sm.topology.ProxyUsers[user.UserName].WhiteList = user.WhiteList
	}
	if len(user.BlackList) != 0 {
		sm.topology.ProxyUsers[user.UserName].BlackList = user.BlackList
	}

	// del old map
	for k, _ := range sm.topology.DatabaseAuth.ProxyDatabaseUserMap {
		if k == user.UserName {
			delete(sm.topology.DatabaseAuth.ProxyDatabaseUserMap, k)
		}
	}
	// add new map
	for k, v := range sm.topology.DatabaseAuth.DatabaseUsers {
		if v.Role == user.Role {
			sm.topology.DatabaseAuth.ProxyDatabaseUserMap[user.UserName] = k
		}
	}

	// version+1
	iversion, _ := strconv.Atoi(sm.topology.Version)
	sm.topology.Version = strconv.Itoa(iversion + 1)

	btopology, _ := json.Marshal(sm.topology)

	err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
	if err != nil {
		log.WithFields(log.Fields{
			"key": prefix + consts.TopologyKey,
			"val": string(btopology),
			"err": err.Error(),
		}).Error("push topology to consul fail, continue")
	}
	err = presistent.TopologyPresist(string(btopology))
	if err != nil {
		log.WithFields(log.Fields{
			"val": string(btopology),
			"err": err.Error(),
		}).Error("topology presist fail, continue")
	}

	w.WriteHeader(http.StatusOK)
}
Example #6
0
func HealthCheck() {
	m := cmap.New()

	for {
		if atomic.LoadInt64(&sm.state) != 0 {
			time.Sleep(sm.config.HealthCheckInterval)
			continue
		}

		sm.rwMutex.RLock()
		if sm.topology == nil {
			sm.rwMutex.RUnlock()
			time.Sleep(sm.config.HealthCheckInterval)
			continue
		}

		for k, v := range sm.topology.DataNodeGroup["default"] {
			serviceName := k
			dbName := k
			if v.Status == consts.Normal {
				go func() {
					health, err := sm.consul.HealthCheck(serviceName)
					if err != nil {
						log.WithFields(log.Fields{
							"dbName": dbName,
						}).Error("consul health check error, direct use swarm dbcheck")
						health, err = sm.swarm.DbHealthCheck(serviceName)
						if err != nil {
							log.WithFields(log.Fields{
								"dbName": dbName,
							}).Error("swarm DbHealthCheck error")
						}
					}
					if !health || err != nil {
						log.WithFields(log.Fields{
							"fail times": m.GetFail(dbName),
							"dbName":     dbName,
						}).Warn("health check fail")
						if ok := m.Fail(dbName); ok {
							return
						}
						log.Error("toggle health check auto isolate")
						if getLock(1) {
							sm.steps = []string{}
							// maybe already isolated at before goroute?
							// no need rwMutex?
							if sm.topology.DataNodeGroup["default"][dbName].Status != consts.Normal {
								log.WithFields(log.Fields{
									"dbName": dbName,
									"status": sm.topology.DataNodeGroup["default"][dbName].Status,
								}).Warn("db node already isolated")
								atomic.CompareAndSwapInt64(&sm.state, 1, 0)
								return
							}
							err, steps := ha.IsolateDB(dbName, sm.rwMutex, sm.topology, sm.swarm)
							sm.steps = steps
							atomic.CompareAndSwapInt64(&sm.state, 1, 0)
							if err != nil {
								log.WithFields(log.Fields{
									"dbName": dbName,
								}).Error("health check auto isolate fail")
							}
							sm.rwMutex.RLock()
							btopology, _ := json.Marshal(sm.topology)
							sm.rwMutex.RUnlock()
							err = sm.consul.PutVal(prefix+consts.TopologyKey, btopology)
							if err != nil {
								log.WithFields(log.Fields{
									"key": prefix + consts.TopologyKey,
									"val": string(btopology),
									"err": err.Error(),
								}).Error("push topology to consul fail, continue")
							}
							err = presistent.TopologyPresist(string(btopology))
							if err != nil {
								log.WithFields(log.Fields{
									"val": string(btopology),
									"err": err.Error(),
								}).Error("topology presist fail, continue")
							}
						} else {
							log.WithFields(log.Fields{
								"dbname": dbName,
							}).Error("health check auto isolate get lock fail")
						}
					} else {
						m.Success(dbName)
					}
				}()
			}
		}
		sm.rwMutex.RUnlock()

		time.Sleep(sm.config.HealthCheckInterval)
	}
}