func initNodeData() {
if memConfNodes[conf.ClientAddr] == nil {
bs, _ := json.Marshal(memConfDataVersion)
node := &models.Node{
URL: conf.ClientAddr,
NodeURL: conf.NodeAddr,
Type: conf.NodeType,
DataVersion: memConfDataVersion,
DataVersionStr: string(bs),
CreatedUTC: utils.GetNowSecond(),
}
if err := models.InsertRow(nil, node); err != nil {
log.Panicf("Failed to init node data: %s", err.Error())
}
memConfNodes[conf.ClientAddr] = node
}
node := memConfNodes[conf.ClientAddr]
if node.Type != conf.NodeType {
node.Type = conf.NodeType
if err := models.UpdateDBModel(nil, node); err != nil {
log.Panicf("Failed to update node data: %s", err.Error())
}
}
}
func cloneConfigsFromApp(from, to, aux_info, userKey string) (*models.App, []*models.Config, error) {
memConfMux.RLock()
fromApp := memConfAppsByName[from]
memConfMux.RUnlock()
fromConfigs, err := models.GetConfigsByAppKey(nil, fromApp.Key)
if err != nil {
return nil, nil, err
}
app := &models.App{
Key: utils.GenerateKey(),
Name: to,
UserKey: userKey,
AuxInfo: aux_info,
CreatedUTC: utils.GetNowSecond(),
Type: fromApp.Type,
}
for _, config := range fromConfigs {
config.Key = utils.GenerateKey()
config.CreatorKey = app.UserKey
config.AppKey = app.Key
config.CreatedUTC = app.CreatedUTC
}
if err := cloneConfigs(app, fromConfigs, userKey); err != nil {
return nil, nil, err
}
return app, fromConfigs, nil
}
func newUserWithNewUserData(data *newUserData, userKey, creatorKey string) (*models.User, error) {
user := &models.User{
Name: data.Name,
PassCode: encryptUserPassCode(data.PassCode),
CreatorKey: creatorKey,
CreatedUTC: utils.GetNowSecond(),
AuxInfo: data.AuxInfo,
Status: models.USER_STATUS_ACTIVE,
Key: userKey}
return updateUser(user, nil)
}
func newConfigWithNewConfigData(data *newConfigData, userKey string) (*models.Config, error) {
config := &models.Config{
Key: utils.GenerateKey(),
AppKey: data.AppKey,
K: data.K,
V: data.V,
VType: data.VType,
CreatedUTC: utils.GetNowSecond(),
CreatorKey: userKey,
Des: data.Des,
Status: models.CONF_STATUS_ACTIVE,
}
return updateConfig(config, userKey, nil, nil)
}
func UpdateMasterLastDataUpdateUTC(c *gin.Context) {
if !getServiceStatus(c) ||
(c.Request.Method != http.MethodPost && c.Request.Method != http.MethodPut && c.Request.Method != http.MethodPatch && c.Request.Method != http.MethodDelete) {
return
}
memConfMux.RLock()
localNode := *memConfNodes[conf.ClientAddr] // must be master node
memConfMux.RUnlock()
// use master node's LastCheckUTC to store last update utc
localNode.LastCheckUTC = utils.GetNowSecond()
if models.UpdateDBModel(nil, &localNode) == nil {
memConfMux.Lock()
memConfNodes[localNode.URL] = &localNode
memConfMux.Unlock()
}
}
func syncData2Slave(node *models.Node, data interface{}, dataVer *models.DataVersion, opUserKey string) error {
kind := ""
switch data.(type) {
case *models.User:
kind = NODE_REQUEST_SYNC_TYPE_USER
case *models.App:
kind = NODE_REQUEST_SYNC_TYPE_APP
case *models.WebHook:
kind = NODE_REQUEST_SYNC_TYPE_WEBHOOK
case *models.Config:
kind = NODE_REQUEST_SYNC_TYPE_CONFIG
case *models.Node:
kind = NODE_REQUEST_SYNC_TYPE_NODE
case *cloneData:
kind = NODE_REQUEST_SYNC_TYPE_CLONE
default:
log.Panicln("unknown node data sync type: ", reflect.TypeOf(data))
}
bs, _ := json.Marshal(data)
syncDataString, _ := json.Marshal(&syncDataT{
DataVersion: dataVer,
Kind: kind,
Data: string(bs),
OpUserKey: opUserKey,
})
reqData := nodeRequestDataT{
Auth: nodeAuthString,
Data: string(syncDataString),
}
_, err := nodeRequest(node.NodeURL, NODE_REQUEST_TYPE_SYNCSLAVE, reqData)
if err == nil && kind != NODE_REQUEST_SYNC_TYPE_NODE {
// update slave data version here
dataVersionStr, _ := json.Marshal(dataVer)
memConfMux.Lock()
node.DataVersion = dataVer
node.DataVersionStr = string(dataVersionStr)
node.LastCheckUTC = utils.GetNowSecond()
memConfMux.Unlock()
updateNodeDataVersion(nil, node, dataVer)
}
return err
}
func NewApp(c *gin.Context) {
confWriteMux.Lock()
defer confWriteMux.Unlock()
var data struct {
Name string `json:"name" binding:"required"`
Type string `json:"type" binding:"required"`
AuxInfo string `json:"aux_info"`
}
if err := c.BindJSON(&data); err != nil {
Error(c, BAD_POST_DATA, err.Error())
return
}
if !models.IsValidAppType(data.Type) {
Error(c, BAD_REQUEST, "unknown app type: "+data.Type)
return
}
if memConfAppsByName[data.Name] != nil {
Error(c, BAD_REQUEST, "appname already exists: "+data.Name)
return
}
app := &models.App{
Key: utils.GenerateKey(),
Name: data.Name,
UserKey: getOpUserKey(c),
Type: data.Type,
AuxInfo: data.AuxInfo,
CreatedUTC: utils.GetNowSecond(),
}
if _, err := updateApp(app, nil, nil); err != nil {
Error(c, SERVER_ERROR, err.Error())
return
}
failedNodes := syncData2SlaveIfNeed(app, getOpUserKey(c))
if len(failedNodes) > 0 {
Success(c, map[string]interface{}{"failed_nodes": failedNodes})
} else {
Success(c, nil)
}
}
func handleSlaveCheckMaster(c *gin.Context, data string) {
if !conf.IsMasterNode() {
Error(c, BAD_REQUEST, "invalid req type for slave node: "+NODE_REQUEST_TYPE_CHECKMASTER)
return
}
node := &models.Node{}
if err := json.Unmarshal([]byte(data), node); err != nil {
Error(c, BAD_REQUEST, "bad req body format")
return
}
confWriteMux.Lock()
defer confWriteMux.Unlock()
oldNode := memConfNodes[node.URL]
node.LastCheckUTC = utils.GetNowSecond()
if oldNode == nil {
if err := models.InsertRow(nil, node); err != nil {
Error(c, SERVER_ERROR, err.Error())
return
}
} else {
if err := models.UpdateDBModel(nil, node); err != nil {
Error(c, SERVER_ERROR, err.Error())
return
}
}
memConfMux.Lock()
memConfNodes[node.URL] = node
bs, _ := json.Marshal(memConfDataVersion)
memConfMux.Unlock()
go masterSyncNodeToSlave(node)
Success(c, string(bs))
}
func TestPubuNotifaction(t *testing.T) {
pubuURL := `https://hooks.pubu.im/services/q2qp7wywyebgfqd`
hostname, _ := os.Hostname()
configHistory := &models.ConfigUpdateHistory{
Id: "",
ConfigKey: "",
K: "TestKey",
OldV: "",
OldVType: "",
NewV: "1.0",
NewVType: models.CONF_V_TYPE_INT,
Kind: models.CONFIG_UPDATE_KIND_NEW,
UserKey: "",
UserName: fmt.Sprintf("PubuTester@%s", hostname),
CreatedUTC: utils.GetNowSecond(),
}
app := &models.App{
Name: "TestApp",
}
sendNotificationToPubu(pubuURL, configHistory, app)
configHistory.UserName = fmt.Sprintf("SlackTester@%s", hostname)
sendNotificationToSlack(pubuURL, configHistory, app)
}
func updateConfig(config *models.Config, userKey string, newDataVersion *models.DataVersion, ms *models.Session) (*models.Config, error) {
var s *models.Session
if ms != nil {
s = ms
} else {
s = models.NewSession()
defer s.Close()
if err := s.Begin(); err != nil {
s.Rollback()
return nil, err
}
}
isSysConf := isSysConfType(config.AppKey)
node := *memConfNodes[conf.ClientAddr]
oldConfig := memConfRawConfigs[config.Key]
app, err := models.GetAppByKey(s, config.AppKey)
if err != nil {
return nil, err
}
if newDataVersion == nil {
newDataVersion = genNewDataVersion(memConfDataVersion)
}
if err := updateNodeDataVersion(s, &node, newDataVersion); err != nil {
if ms == nil {
s.Rollback()
}
return nil, err
}
var configHistory *models.ConfigUpdateHistory
if oldConfig == nil {
configHistory = &models.ConfigUpdateHistory{
Id: utils.GenerateKey(),
ConfigKey: config.Key,
K: config.K,
OldV: "",
OldVType: "",
NewV: config.V,
NewVType: config.VType,
Kind: models.CONFIG_UPDATE_KIND_NEW,
UserKey: userKey,
CreatedUTC: utils.GetNowSecond(),
}
if err := models.InsertRow(s, configHistory); err != nil {
if ms == nil {
s.Rollback()
}
return nil, err
}
config.LastUpdateId = configHistory.Id
if err := models.InsertRow(s, config); err != nil {
if ms == nil {
s.Rollback()
}
return nil, err
}
if !isSysConf {
app.KeyCount++
app.LastUpdateUTC = configHistory.CreatedUTC
app.LastUpdateId = configHistory.Id
app.UpdateTimes++
}
} else {
kind := models.CONFIG_UPDATE_KIND_UPDATE
if config.Status != oldConfig.Status {
if config.Status == models.CONF_STATUS_ACTIVE {
kind = models.CONFIG_UPDATE_KIND_RECOVER
} else {
kind = models.CONFIG_UPDATE_KIND_HIDE
}
}
configHistory = &models.ConfigUpdateHistory{
Id: utils.GenerateKey(),
ConfigKey: config.Key,
K: config.K,
OldV: oldConfig.V,
OldVType: oldConfig.VType,
NewV: config.V,
NewVType: config.VType,
Kind: kind,
UserKey: userKey,
CreatedUTC: utils.GetNowSecond(),
}
if err := models.InsertRow(s, configHistory); err != nil {
if ms == nil {
s.Rollback()
}
return nil, err
}
config.UpdateTimes++
config.LastUpdateId = configHistory.Id
if err := models.UpdateDBModel(s, config); err != nil {
if ms == nil {
s.Rollback()
}
return nil, err
}
if !isSysConf {
app.LastUpdateUTC = configHistory.CreatedUTC
app.LastUpdateId = configHistory.Id
app.UpdateTimes++
}
}
var toUpdateApps []*models.App
if !isSysConf {
newDataSign := utils.GenerateKey()
app.DataSign = newDataSign
if err := models.UpdateDBModel(s, app); err != nil {
if ms == nil {
s.Rollback()
}
return nil, err
}
if app.Type == models.APP_TYPE_TEMPLATE {
for _, _app := range memConfApps {
if _app.Key == config.AppKey {
continue
}
for _, _config := range memConfAppConfigs[_app.Key] {
if _config.VType == models.CONF_V_TYPE_TEMPLATE && _config.V == config.AppKey {
// this app has a config refer to this template app
toUpdateApps = append(toUpdateApps, _app)
break
}
}
}
}
for _, app := range toUpdateApps {
_app := *app
_app.DataSign = newDataSign
if err := models.UpdateDBModel(s, &_app); err != nil {
if ms == nil {
s.Rollback()
}
return nil, err
}
}
}
if ms == nil {
if err := s.Commit(); err != nil {
s.Rollback()
return nil, err
}
if !isSysConf {
go TriggerWebHooks(configHistory, app)
} else {
go TriggerWebHooks(configHistory, &models.App{Key: config.Key, Name: config.Key})
}
updateMemConf(config, newDataVersion, &node, toUpdateApps)
}
return config, nil
}
func ClientConf(c *gin.Context) {
clientData := &ClientData{
AppKey: c.Query("app_key"),
OSType: c.Query("os_type"),
OSVersion: c.Query("os_version"),
AppVersion: c.Query("app_version"),
Ip: c.Query("ip"),
Lang: c.Query("lang"),
DeviceId: c.Query("device_id"),
DataSign: c.Query("data_sign"),
TimeZone: c.Query("timezone"),
NetWork: c.Query("network"),
}
if clientData.AppKey == "" {
memConfMux.RLock()
app := memConfAppsByName[c.Query("app")]
memConfMux.RUnlock()
if app != nil {
clientData.AppKey = app.Key
clientData.OSType = "ios"
clientData.OSVersion = c.Query("osv")
clientData.AppVersion = c.Query("v")
clientData.DeviceId = c.Query("ida")
clientData.Ip = c.Request.RemoteAddr
}
clientData = uniformClientParams(clientData)
sendChanAsync(clientQueryParamCh, clientData)
setClientData(c, clientData)
c.JSON(http.StatusOK, getAppMatchConf(clientData.AppKey, clientData))
return
}
clientData = uniformClientParams(clientData)
sendChanAsync(clientQueryParamCh, clientData)
setClientData(c, clientData)
memConfMux.RLock()
if !conf.IsMasterNode() && conf.DataExpires > 0 {
if memConfNodes[conf.ClientAddr].LastCheckUTC < utils.GetNowSecond()-conf.DataExpires {
memConfMux.RUnlock()
Error(c, DATA_EXPIRED)
return
}
}
nodes := []string{}
nodes = make([]string, len(memConfNodes))
ix := 0
for _, node := range memConfNodes {
nodes[ix] = node.URL
ix++
}
needConf := memConfApps[clientData.AppKey] != nil && clientData.DataSign != memConfApps[clientData.AppKey].DataSign
memConfMux.RUnlock()
if needConf {
var dataSign string
configs := getAppMatchConf(clientData.AppKey, clientData)
if len(configs) > 0 {
memConfMux.RLock()
dataSign = memConfApps[clientData.AppKey].DataSign
memConfMux.RUnlock()
}
Success(c, map[string]interface{}{
"nodes": nodes,
"configs": configs,
"data_sign": dataSign,
})
} else {
Success(c, map[string]interface{}{
"data_sign": clientData.DataSign,
"nodes": nodes,
})
}
return
}
func slaveCheckMaster() error {
confWriteMux.Lock()
defer confWriteMux.Unlock()
localNode := *memConfNodes[conf.ClientAddr]
nodeString, _ := json.Marshal(localNode)
reqData := nodeRequestDataT{
Auth: nodeAuthString,
Data: string(nodeString),
}
data, err := nodeRequest(conf.MasterAddr, NODE_REQUEST_TYPE_CHECKMASTER, reqData)
if err != nil {
return err
}
masterVersion := &models.DataVersion{}
if err = json.Unmarshal([]byte(data.(string)), masterVersion); err != nil {
return fmt.Errorf("bad response data format: %s < %s >", err.Error(), data.(string))
}
if masterVersion.Version == memConfDataVersion.Version && masterVersion.Sign == memConfDataVersion.Sign {
localNode.LastCheckUTC = utils.GetNowSecond()
if err = models.UpdateDBModel(nil, &localNode); err != nil {
return err
}
memConfMux.Lock()
memConfNodes[conf.ClientAddr] = &localNode
memConfMux.Unlock()
return nil
}
reqData = nodeRequestDataT{
Auth: nodeAuthString,
Data: "",
}
// slave's data_version not equals master's data_version, slave sync all data from master
data, err = nodeRequest(conf.MasterAddr, NODE_REQUEST_TYPE_SYNCMASTER, reqData)
if err != nil {
return err
}
resData := &syncAllDataT{}
if err = json.Unmarshal([]byte(data.(string)), resData); err != nil {
return fmt.Errorf("bad response data format: %s < %s >", err.Error(), data.(string))
}
var users []*models.User
var apps []*models.App
var configs []*models.Config
var nodes []*models.Node
bs, _ := json.Marshal(resData.DataVersion)
localNode.DataVersion = resData.DataVersion
localNode.DataVersionStr = string(bs)
localNode.LastCheckUTC = utils.GetNowSecond()
// let sqlite use new db file
models.UpdateSqliteDBEngine()
s := models.NewSession()
defer s.Close()
if err = s.Begin(); err != nil {
s.Rollback()
return err
}
if err = models.ClearModeData(s); err != nil {
s.Rollback()
return err
}
toInsertModels := make([]interface{}, 0)
for _, node := range resData.Nodes {
if node.URL == conf.ClientAddr {
node.DataVersion = localNode.DataVersion
node.DataVersionStr = localNode.DataVersionStr
node.LastCheckUTC = localNode.LastCheckUTC
}
toInsertModels = append(toInsertModels, node)
nodes = append(nodes, node)
}
if err := models.InsertMultiRows(s, toInsertModels); err != nil {
s.Rollback()
return err
}
toInsertModels = make([]interface{}, 0)
for _, user := range resData.Users {
users = append(users, user)
toInsertModels = append(toInsertModels, user)
}
if err = models.InsertMultiRows(s, toInsertModels); err != nil {
s.Rollback()
return err
}
toInsertModels = make([]interface{}, 0)
for _, app := range resData.Apps {
toInsertModels = append(toInsertModels, app)
apps = append(apps, app)
}
if err = models.InsertMultiRows(s, toInsertModels); err != nil {
s.Rollback()
return err
}
toInsertModels = make([]interface{}, len(resData.WebHooks))
for ix, hook := range resData.WebHooks {
toInsertModels[ix] = hook
}
if err = models.InsertMultiRows(s, toInsertModels); err != nil {
s.Rollback()
return err
}
toInsertModels = make([]interface{}, 0)
for _, config := range resData.Configs {
toInsertModels = append(toInsertModels, config)
configs = append(configs, config)
}
if err = models.InsertMultiRows(s, toInsertModels); err != nil {
s.Rollback()
return err
}
toInsertModels = make([]interface{}, len(resData.ConfHistory))
for ix, history := range resData.ConfHistory {
toInsertModels[ix] = history
}
if err = models.InsertMultiRows(s, toInsertModels); err != nil {
s.Rollback()
return err
}
if err = models.UpdateDataVersion(s, resData.DataVersion); err != nil {
s.Rollback()
return err
}
if err = s.Commit(); err != nil {
s.Rollback()
return err
}
fillMemConfData(users, apps, resData.WebHooks, configs, nodes, resData.DataVersion)
nodeString, _ = json.Marshal(&localNode)
reqData = nodeRequestDataT{
Auth: nodeAuthString,
Data: string(nodeString),
}
nodeRequest(conf.MasterAddr, NODE_REQUEST_TYPE_CHECKMASTER, reqData)
return nil
}