func initNodeData() { if memConfNodes[conf.ClientAddr] == nil { bs, _ := json.Marshal(memConfDataVersion) node := &models.Node{ URL: conf.ClientAddr, NodeURL: conf.NodeAddr, Type: conf.NodeType, DataVersion: memConfDataVersion, DataVersionStr: string(bs), CreatedUTC: utils.GetNowSecond(), } if err := models.InsertRow(nil, node); err != nil { log.Panicf("Failed to init node data: %s", err.Error()) } memConfNodes[conf.ClientAddr] = node } node := memConfNodes[conf.ClientAddr] if node.Type != conf.NodeType { node.Type = conf.NodeType if err := models.UpdateDBModel(nil, node); err != nil { log.Panicf("Failed to update node data: %s", err.Error()) } } }
func updateApp(app *models.App, newDataVersion *models.DataVersion, ms *models.Session) (*models.App, error) { var s *models.Session if ms != nil { s = ms } else { s = models.NewSession() defer s.Close() if err := s.Begin(); err != nil { s.Rollback() return nil, err } } node := *memConfNodes[conf.ClientAddr] oldApp := memConfApps[app.Key] if newDataVersion == nil { newDataVersion = genNewDataVersion(memConfDataVersion) } if err := updateNodeDataVersion(s, &node, newDataVersion); err != nil { if ms == nil { s.Rollback() } return nil, err } if oldApp == nil { if err := models.InsertRow(s, app); err != nil { if ms == nil { s.Rollback() } return nil, err } } else { if err := models.UpdateDBModel(s, app); err != nil { if ms == nil { s.Rollback() } return nil, err } } if ms == nil { if err := s.Commit(); err != nil { s.Rollback() return nil, err } updateMemConf(app, newDataVersion, &node) } return app, nil }
func updateWebHook(hook *models.WebHook, newDataVersion *models.DataVersion) (*models.WebHook, error) { s := models.NewSession() defer s.Close() if err := s.Begin(); err != nil { s.Rollback() return nil, err } node := *memConfNodes[conf.ClientAddr] oldHookIdx := -1 var hooks []*models.WebHook if hook.Scope == models.WEBHOOK_SCOPE_GLOBAL { hooks = memConfGlobalWebHooks } else if hook.Scope == models.WEBHOOK_SCOPE_APP { hooks = memConfAppWebHooks[hook.AppKey] } for idx, oldHook := range hooks { if hook.Key == oldHook.Key { oldHookIdx = idx break } } if newDataVersion == nil { newDataVersion = genNewDataVersion(memConfDataVersion) } if err := updateNodeDataVersion(s, &node, newDataVersion); err != nil { s.Rollback() return nil, err } if oldHookIdx == -1 { if err := models.InsertRow(s, hook); err != nil { s.Rollback() return nil, err } } else { if err := models.UpdateDBModel(s, hook); err != nil { s.Rollback() return nil, err } } if err := s.Commit(); err != nil { s.Rollback() return nil, err } updateMemConf(hook, newDataVersion, &node, oldHookIdx) return hook, nil }
func UpdateMasterLastDataUpdateUTC(c *gin.Context) { if !getServiceStatus(c) || (c.Request.Method != http.MethodPost && c.Request.Method != http.MethodPut && c.Request.Method != http.MethodPatch && c.Request.Method != http.MethodDelete) { return } memConfMux.RLock() localNode := *memConfNodes[conf.ClientAddr] // must be master node memConfMux.RUnlock() // use master node's LastCheckUTC to store last update utc localNode.LastCheckUTC = utils.GetNowSecond() if models.UpdateDBModel(nil, &localNode) == nil { memConfMux.Lock() memConfNodes[localNode.URL] = &localNode memConfMux.Unlock() } }
func updateNodeDataVersion(s *models.Session, node *models.Node, ver *models.DataVersion) (err error) { var _s *models.Session var bs []byte if s == nil { _s = models.NewSession() defer _s.Close() if err = _s.Begin(); err != nil { goto ERROR } } else { _s = s } bs, _ = json.Marshal(ver) node.DataVersion = ver node.DataVersionStr = string(bs) if err = models.UpdateDBModel(_s, node); err != nil { goto ERROR } if node.URL == conf.ClientAddr { if err = models.UpdateDataVersion(_s, ver); err != nil { goto ERROR } } if s != nil { return } if err = _s.Commit(); err != nil { goto ERROR } return ERROR: if s == nil { _s.Rollback() } return }
func updateUser(user *models.User, newDataVersion *models.DataVersion) (*models.User, error) { s := models.NewSession() defer s.Close() if err := s.Begin(); err != nil { s.Rollback() return nil, err } node := *memConfNodes[conf.ClientAddr] oldUser := memConfUsers[user.Key] if newDataVersion == nil { newDataVersion = genNewDataVersion(memConfDataVersion) } if err := updateNodeDataVersion(s, &node, newDataVersion); err != nil { s.Rollback() return nil, err } if oldUser == nil { if err := models.InsertRow(s, user); err != nil { s.Rollback() return nil, err } } else { if err := models.UpdateDBModel(s, user); err != nil { s.Rollback() return nil, err } } if err := s.Commit(); err != nil { s.Rollback() return nil, err } updateMemConf(user, newDataVersion, &node) return user, nil }
func handleSlaveCheckMaster(c *gin.Context, data string) { if !conf.IsMasterNode() { Error(c, BAD_REQUEST, "invalid req type for slave node: "+NODE_REQUEST_TYPE_CHECKMASTER) return } node := &models.Node{} if err := json.Unmarshal([]byte(data), node); err != nil { Error(c, BAD_REQUEST, "bad req body format") return } confWriteMux.Lock() defer confWriteMux.Unlock() oldNode := memConfNodes[node.URL] node.LastCheckUTC = utils.GetNowSecond() if oldNode == nil { if err := models.InsertRow(nil, node); err != nil { Error(c, SERVER_ERROR, err.Error()) return } } else { if err := models.UpdateDBModel(nil, node); err != nil { Error(c, SERVER_ERROR, err.Error()) return } } memConfMux.Lock() memConfNodes[node.URL] = node bs, _ := json.Marshal(memConfDataVersion) memConfMux.Unlock() go masterSyncNodeToSlave(node) Success(c, string(bs)) }
func updateConfig(config *models.Config, userKey string, newDataVersion *models.DataVersion, ms *models.Session) (*models.Config, error) { var s *models.Session if ms != nil { s = ms } else { s = models.NewSession() defer s.Close() if err := s.Begin(); err != nil { s.Rollback() return nil, err } } isSysConf := isSysConfType(config.AppKey) node := *memConfNodes[conf.ClientAddr] oldConfig := memConfRawConfigs[config.Key] app, err := models.GetAppByKey(s, config.AppKey) if err != nil { return nil, err } if newDataVersion == nil { newDataVersion = genNewDataVersion(memConfDataVersion) } if err := updateNodeDataVersion(s, &node, newDataVersion); err != nil { if ms == nil { s.Rollback() } return nil, err } var configHistory *models.ConfigUpdateHistory if oldConfig == nil { configHistory = &models.ConfigUpdateHistory{ Id: utils.GenerateKey(), ConfigKey: config.Key, K: config.K, OldV: "", OldVType: "", NewV: config.V, NewVType: config.VType, Kind: models.CONFIG_UPDATE_KIND_NEW, UserKey: userKey, CreatedUTC: utils.GetNowSecond(), } if err := models.InsertRow(s, configHistory); err != nil { if ms == nil { s.Rollback() } return nil, err } config.LastUpdateId = configHistory.Id if err := models.InsertRow(s, config); err != nil { if ms == nil { s.Rollback() } return nil, err } if !isSysConf { app.KeyCount++ app.LastUpdateUTC = configHistory.CreatedUTC app.LastUpdateId = configHistory.Id app.UpdateTimes++ } } else { kind := models.CONFIG_UPDATE_KIND_UPDATE if config.Status != oldConfig.Status { if config.Status == models.CONF_STATUS_ACTIVE { kind = models.CONFIG_UPDATE_KIND_RECOVER } else { kind = models.CONFIG_UPDATE_KIND_HIDE } } configHistory = &models.ConfigUpdateHistory{ Id: utils.GenerateKey(), ConfigKey: config.Key, K: config.K, OldV: oldConfig.V, OldVType: oldConfig.VType, NewV: config.V, NewVType: config.VType, Kind: kind, UserKey: userKey, CreatedUTC: utils.GetNowSecond(), } if err := models.InsertRow(s, configHistory); err != nil { if ms == nil { s.Rollback() } return nil, err } config.UpdateTimes++ config.LastUpdateId = configHistory.Id if err := models.UpdateDBModel(s, config); err != nil { if ms == nil { s.Rollback() } return nil, err } if !isSysConf { app.LastUpdateUTC = configHistory.CreatedUTC app.LastUpdateId = configHistory.Id app.UpdateTimes++ } } var toUpdateApps []*models.App if !isSysConf { newDataSign := utils.GenerateKey() app.DataSign = newDataSign if err := models.UpdateDBModel(s, app); err != nil { if ms == nil { s.Rollback() } return nil, err } if app.Type == models.APP_TYPE_TEMPLATE { for _, _app := range memConfApps { if _app.Key == config.AppKey { continue } for _, _config := range memConfAppConfigs[_app.Key] { if _config.VType == models.CONF_V_TYPE_TEMPLATE && _config.V == config.AppKey { // this app has a config refer to this template app toUpdateApps = append(toUpdateApps, _app) break } } } } for _, app := range toUpdateApps { _app := *app _app.DataSign = newDataSign if err := models.UpdateDBModel(s, &_app); err != nil { if ms == nil { s.Rollback() } return nil, err } } } if ms == nil { if err := s.Commit(); err != nil { s.Rollback() return nil, err } if !isSysConf { go TriggerWebHooks(configHistory, app) } else { go TriggerWebHooks(configHistory, &models.App{Key: config.Key, Name: config.Key}) } updateMemConf(config, newDataVersion, &node, toUpdateApps) } return config, nil }
func handleSlaveSyncUpdateData(c *gin.Context, data string) { if conf.IsMasterNode() { Error(c, BAD_REQUEST, "invalid req type for master node: "+NODE_REQUEST_TYPE_SYNCSLAVE) return } syncData := &syncDataT{} err := json.Unmarshal([]byte(data), syncData) if err != nil { Error(c, BAD_REQUEST, "bad req body format") return } confWriteMux.Lock() defer confWriteMux.Unlock() if syncData.Kind != NODE_REQUEST_SYNC_TYPE_NODE { if memConfDataVersion.Version+1 != syncData.DataVersion.Version { Error(c, DATA_VERSION_ERROR, "slave node data version [%d] error for master data version [%d]", memConfDataVersion.Version, syncData.DataVersion.Version) return } if memConfDataVersion.Sign != syncData.DataVersion.OldSign { Error(c, DATA_VERSION_ERROR, "slave node's data sign [%s] not equal master node's old data sign [%s]", memConfDataVersion.Sign, syncData.DataVersion.OldSign) return } } switch syncData.Kind { case NODE_REQUEST_SYNC_TYPE_USER: user := &models.User{} if err = json.Unmarshal([]byte(syncData.Data), user); err != nil { Error(c, BAD_REQUEST, "bad data format for user model") return } if _, err = updateUser(user, syncData.DataVersion); err != nil { Error(c, SERVER_ERROR, err.Error()) return } case NODE_REQUEST_SYNC_TYPE_APP: app := &models.App{} if err = json.Unmarshal([]byte(syncData.Data), app); err != nil { Error(c, BAD_REQUEST, "bad data format for app model") return } if _, err = updateApp(app, syncData.DataVersion, nil); err != nil { Error(c, SERVER_ERROR, err.Error()) return } case NODE_REQUEST_SYNC_TYPE_WEBHOOK: hook := &models.WebHook{} if err = json.Unmarshal([]byte(syncData.Data), hook); err != nil { Error(c, BAD_REQUEST, "bad data format for webHook model") return } if _, err = updateWebHook(hook, syncData.DataVersion); err != nil { Error(c, SERVER_ERROR, err.Error()) return } case NODE_REQUEST_SYNC_TYPE_CONFIG: config := &models.Config{} if err = json.Unmarshal([]byte(syncData.Data), config); err != nil { Error(c, BAD_REQUEST, "bad data format for user model") return } if _, err = updateConfig(config, syncData.OpUserKey, syncData.DataVersion, nil); err != nil { Error(c, SERVER_ERROR, err.Error()) return } case NODE_REQUEST_SYNC_TYPE_NODE: node := &models.Node{} if err = json.Unmarshal([]byte(syncData.Data), node); err != nil { Error(c, BAD_REQUEST, "bad data format for node model") return } if memConfNodes[node.URL] == nil { if err := models.InsertRow(nil, node); err != nil { Error(c, SERVER_ERROR, err.Error()) return } } else { if err := models.UpdateDBModel(nil, node); err != nil { Error(c, SERVER_ERROR, err.Error()) return } } memConfMux.Lock() memConfNodes[node.URL] = node memConfMux.Unlock() Success(c, nil) return case NODE_REQUEST_SYNC_TYPE_CLONE: data := &cloneData{} if err := json.Unmarshal([]byte(syncData.Data), data); err != nil { Error(c, BAD_REQUEST, "bad data format for clone app") return } if err := cloneConfigs(data.App, data.Configs, syncData.OpUserKey); err != nil { Error(c, SERVER_ERROR, err.Error()) return } default: Error(c, BAD_REQUEST, "unknown node data sync type: "+syncData.Kind) return } masterNode := getMasterNode() masterNode.DataVersion = syncData.DataVersion bs, _ := json.Marshal(syncData.DataVersion) masterNode.DataVersionStr = string(bs) if err = models.UpdateDBModel(nil, &masterNode); err != nil { memConfMux.Lock() memConfNodes[masterNode.URL] = &masterNode memConfMux.Unlock() } Success(c, nil) }
func slaveCheckMaster() error { confWriteMux.Lock() defer confWriteMux.Unlock() localNode := *memConfNodes[conf.ClientAddr] nodeString, _ := json.Marshal(localNode) reqData := nodeRequestDataT{ Auth: nodeAuthString, Data: string(nodeString), } data, err := nodeRequest(conf.MasterAddr, NODE_REQUEST_TYPE_CHECKMASTER, reqData) if err != nil { return err } masterVersion := &models.DataVersion{} if err = json.Unmarshal([]byte(data.(string)), masterVersion); err != nil { return fmt.Errorf("bad response data format: %s < %s >", err.Error(), data.(string)) } if masterVersion.Version == memConfDataVersion.Version && masterVersion.Sign == memConfDataVersion.Sign { localNode.LastCheckUTC = utils.GetNowSecond() if err = models.UpdateDBModel(nil, &localNode); err != nil { return err } memConfMux.Lock() memConfNodes[conf.ClientAddr] = &localNode memConfMux.Unlock() return nil } reqData = nodeRequestDataT{ Auth: nodeAuthString, Data: "", } // slave's data_version not equals master's data_version, slave sync all data from master data, err = nodeRequest(conf.MasterAddr, NODE_REQUEST_TYPE_SYNCMASTER, reqData) if err != nil { return err } resData := &syncAllDataT{} if err = json.Unmarshal([]byte(data.(string)), resData); err != nil { return fmt.Errorf("bad response data format: %s < %s >", err.Error(), data.(string)) } var users []*models.User var apps []*models.App var configs []*models.Config var nodes []*models.Node bs, _ := json.Marshal(resData.DataVersion) localNode.DataVersion = resData.DataVersion localNode.DataVersionStr = string(bs) localNode.LastCheckUTC = utils.GetNowSecond() // let sqlite use new db file models.UpdateSqliteDBEngine() s := models.NewSession() defer s.Close() if err = s.Begin(); err != nil { s.Rollback() return err } if err = models.ClearModeData(s); err != nil { s.Rollback() return err } toInsertModels := make([]interface{}, 0) for _, node := range resData.Nodes { if node.URL == conf.ClientAddr { node.DataVersion = localNode.DataVersion node.DataVersionStr = localNode.DataVersionStr node.LastCheckUTC = localNode.LastCheckUTC } toInsertModels = append(toInsertModels, node) nodes = append(nodes, node) } if err := models.InsertMultiRows(s, toInsertModels); err != nil { s.Rollback() return err } toInsertModels = make([]interface{}, 0) for _, user := range resData.Users { users = append(users, user) toInsertModels = append(toInsertModels, user) } if err = models.InsertMultiRows(s, toInsertModels); err != nil { s.Rollback() return err } toInsertModels = make([]interface{}, 0) for _, app := range resData.Apps { toInsertModels = append(toInsertModels, app) apps = append(apps, app) } if err = models.InsertMultiRows(s, toInsertModels); err != nil { s.Rollback() return err } toInsertModels = make([]interface{}, len(resData.WebHooks)) for ix, hook := range resData.WebHooks { toInsertModels[ix] = hook } if err = models.InsertMultiRows(s, toInsertModels); err != nil { s.Rollback() return err } toInsertModels = make([]interface{}, 0) for _, config := range resData.Configs { toInsertModels = append(toInsertModels, config) configs = append(configs, config) } if err = models.InsertMultiRows(s, toInsertModels); err != nil { s.Rollback() return err } toInsertModels = make([]interface{}, len(resData.ConfHistory)) for ix, history := range resData.ConfHistory { toInsertModels[ix] = history } if err = models.InsertMultiRows(s, toInsertModels); err != nil { s.Rollback() return err } if err = models.UpdateDataVersion(s, resData.DataVersion); err != nil { s.Rollback() return err } if err = s.Commit(); err != nil { s.Rollback() return err } fillMemConfData(users, apps, resData.WebHooks, configs, nodes, resData.DataVersion) nodeString, _ = json.Marshal(&localNode) reqData = nodeRequestDataT{ Auth: nodeAuthString, Data: string(nodeString), } nodeRequest(conf.MasterAddr, NODE_REQUEST_TYPE_CHECKMASTER, reqData) return nil }