func (s *Sentinel) updateCluster(cd *cluster.ClusterData) (*cluster.ClusterData, error) { newcd := cd.DeepCopy() switch cd.Cluster.Status.Phase { case cluster.ClusterPhaseInitializing: switch cd.Cluster.Spec.InitMode { case cluster.ClusterInitModeNew: // Is there already a keeper choosed to be the new master? if cd.Cluster.Status.Master == "" { log.Info("trying to find initial master") k, err := s.findInitialKeeper(cd) if err != nil { return nil, fmt.Errorf("cannot choose initial master: %v", err) } log.Info("initializing cluster", zap.String("keeper", k.UID)) db := &cluster.DB{ UID: s.UIDFn(), Generation: cluster.InitialGeneration, ChangeTime: time.Now(), Spec: &cluster.DBSpec{ KeeperUID: k.UID, InitMode: cluster.DBInitModeNew, Role: common.RoleMaster, Followers: []string{}, IncludeConfig: *cd.Cluster.Spec.MergePgParameters, }, } newcd.DBs[db.UID] = db newcd.Cluster.Status.Master = db.UID log.Debug("newcd dump", zap.String("newcd", spew.Sdump(newcd))) } else { db, ok := cd.DBs[cd.Cluster.Status.Master] if !ok { panic(fmt.Errorf("db %q object doesn't exists. This shouldn't happen", cd.Cluster.Status.Master)) } // Check that the choosed db for being the master has correctly initialized switch s.dbConvergenceState(cd, db, cd.Cluster.Spec.InitTimeout.Duration) { case Converged: if db.Status.Healthy { log.Info("db initialized", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID)) // Set db initMode to none, not needed but just a security measure db.Spec.InitMode = cluster.DBInitModeNone // Don't include previous config anymore db.Spec.IncludeConfig = false // Replace reported pg parameters in cluster spec if *cd.Cluster.Spec.MergePgParameters { newcd.Cluster.Spec.PGParameters = db.Status.PGParameters } // Cluster initialized, switch to Normal state newcd.Cluster.Status.Phase = cluster.ClusterPhaseNormal } case Converging: log.Info("waiting for db", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID)) case ConvergenceFailed: log.Info("db failed to initialize", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID)) // Empty DBs newcd.DBs = cluster.DBs{} // Unset master so another keeper can be choosen newcd.Cluster.Status.Master = "" } } case cluster.ClusterInitModeExisting: if cd.Cluster.Status.Master == "" { wantedKeeper := cd.Cluster.Spec.ExistingConfig.KeeperUID log.Info("trying to use keeper as initial master", zap.String("keeper", wantedKeeper)) k, ok := cd.Keepers[wantedKeeper] if !ok { return nil, fmt.Errorf("keeper %q state not available", wantedKeeper) } log.Info("initializing cluster using selected keeper as master db owner", zap.String("keeper", k.UID)) db := &cluster.DB{ UID: s.UIDFn(), Generation: cluster.InitialGeneration, ChangeTime: time.Now(), Spec: &cluster.DBSpec{ KeeperUID: k.UID, InitMode: cluster.DBInitModeExisting, Role: common.RoleMaster, Followers: []string{}, IncludeConfig: *cd.Cluster.Spec.MergePgParameters, }, } newcd.DBs[db.UID] = db newcd.Cluster.Status.Master = db.UID log.Debug("newcd dump", zap.String("newcd", spew.Sdump(newcd))) } else { db, ok := newcd.DBs[cd.Cluster.Status.Master] if !ok { panic(fmt.Errorf("db %q object doesn't exists. This shouldn't happen", cd.Cluster.Status.Master)) } // Check that the choosed db for being the master has correctly initialized // TODO(sgotti) set a timeout (the max time for a noop operation, just a start/restart) if db.Status.Healthy && s.dbConvergenceState(cd, db, 0) == Converged { log.Info("db initialized", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID)) // Don't include previous config anymore db.Spec.IncludeConfig = false // Replace reported pg parameters in cluster spec if *cd.Cluster.Spec.MergePgParameters { newcd.Cluster.Spec.PGParameters = db.Status.PGParameters } // Cluster initialized, switch to Normal state newcd.Cluster.Status.Phase = cluster.ClusterPhaseNormal } } case cluster.ClusterInitModePITR: // Is there already a keeper choosed to be the new master? if cd.Cluster.Status.Master == "" { log.Info("trying to find initial master") k, err := s.findInitialKeeper(cd) if err != nil { return nil, fmt.Errorf("cannot choose initial master: %v", err) } log.Info("initializing cluster using selected keeper as master db owner", zap.String("keeper", k.UID)) db := &cluster.DB{ UID: s.UIDFn(), Generation: cluster.InitialGeneration, ChangeTime: time.Now(), Spec: &cluster.DBSpec{ KeeperUID: k.UID, InitMode: cluster.DBInitModePITR, PITRConfig: cd.Cluster.Spec.PITRConfig, Role: common.RoleMaster, Followers: []string{}, IncludeConfig: *cd.Cluster.Spec.MergePgParameters, }, } newcd.DBs[db.UID] = db newcd.Cluster.Status.Master = db.UID log.Debug("newcd dump", zap.String("newcd", spew.Sdump(newcd))) } else { db, ok := cd.DBs[cd.Cluster.Status.Master] if !ok { panic(fmt.Errorf("db %q object doesn't exists. This shouldn't happen", cd.Cluster.Status.Master)) } // Check that the choosed db for being the master has correctly initialized // TODO(sgotti) set a timeout (the max time for a restore operation) switch s.dbConvergenceState(cd, db, 0) { case Converged: if db.Status.Healthy { log.Info("db initialized", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID)) // Set db initMode to none, not needed but just a security measure db.Spec.InitMode = cluster.DBInitModeNone // Don't include previous config anymore db.Spec.IncludeConfig = false // Replace reported pg parameters in cluster spec if *cd.Cluster.Spec.MergePgParameters { newcd.Cluster.Spec.PGParameters = db.Status.PGParameters } // Cluster initialized, switch to Normal state newcd.Cluster.Status.Phase = cluster.ClusterPhaseNormal } case Converging: log.Info("waiting for db to converge", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID)) case ConvergenceFailed: log.Info("db failed to initialize", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID)) // Empty DBs newcd.DBs = cluster.DBs{} // Unset master so another keeper can be choosen newcd.Cluster.Status.Master = "" } } default: return nil, fmt.Errorf("unknown init mode %q", cd.Cluster.Spec.InitMode) } case cluster.ClusterPhaseNormal: // Add missing DBs for _, k := range cd.Keepers { if db := cd.FindDB(k); db == nil { db := &cluster.DB{ UID: s.UIDFn(), Generation: cluster.InitialGeneration, ChangeTime: time.Now(), Spec: &cluster.DBSpec{ KeeperUID: k.UID, InitMode: cluster.DBInitModeNone, Role: common.RoleUndefined, Followers: []string{}, }, } newcd.DBs[db.UID] = db } } // TODO(sgotti) When keeper removal is implemented, remove DBs for unexistent keepers // Calculate current master status curMasterDBUID := cd.Cluster.Status.Master wantedMasterDBUID := curMasterDBUID masterOK := true curMasterDB := cd.DBs[curMasterDBUID] if curMasterDB == nil { return nil, fmt.Errorf("db for keeper %q not available. This shouldn't happen!", curMasterDBUID) } log.Debug("db dump", zap.String("db", spew.Sdump(curMasterDB))) if !curMasterDB.Status.Healthy { log.Info("master db is failed", zap.String("db", curMasterDB.UID), zap.String("keeper", curMasterDB.Spec.KeeperUID)) masterOK = false } // Check that the wanted master is in master state (i.e. check that promotion from standby to master happened) if s.dbConvergenceState(cd, curMasterDB, newcd.Cluster.Spec.ConvergenceTimeout.Duration) == ConvergenceFailed { log.Info("db not converged", zap.String("db", curMasterDB.UID), zap.String("keeper", curMasterDB.Spec.KeeperUID)) masterOK = false } if !masterOK { log.Info("trying to find a standby to replace failed master") bestStandbyDB, err := s.findBestStandby(cd, curMasterDB) if err != nil { log.Error("error trying to find the best standby", zap.Error(err)) } else { log.Info("electing db as the new master", zap.String("db", bestStandbyDB.UID), zap.String("keeper", bestStandbyDB.Spec.KeeperUID)) wantedMasterDBUID = bestStandbyDB.UID } } // New master elected if curMasterDBUID != wantedMasterDBUID { // maintain the current role, remove followers oldMasterdb := newcd.DBs[curMasterDBUID] oldMasterdb.Spec.Followers = []string{} newcd.Cluster.Status.Master = wantedMasterDBUID newMasterDB := newcd.DBs[wantedMasterDBUID] newMasterDB.Spec.Role = common.RoleMaster newMasterDB.Spec.FollowConfig = nil // Tell proxy that there's currently no active master newcd.Proxy.Spec.MasterDBUID = "" newcd.Proxy.ChangeTime = time.Now() } // TODO(sgotti) Wait for the proxies being converged (closed connections to old master)? // Setup standbys, do this only when there's no master change if curMasterDBUID == wantedMasterDBUID { masterDB := newcd.DBs[curMasterDBUID] // Set standbys to follow master only if it's healthy and converged if masterDB.Status.Healthy && s.dbConvergenceState(newcd, masterDB, newcd.Cluster.Spec.ConvergenceTimeout.Duration) == Converged { // Tell proxy that there's a new active master newcd.Proxy.Spec.MasterDBUID = wantedMasterDBUID newcd.Proxy.ChangeTime = time.Now() // TODO(sgotti) do this only for the defined number of MaxStandbysPerSender (needs also to detect unhealthy standbys and switch to healthy one) for id, db := range newcd.DBs { if id == wantedMasterDBUID { continue } db.Spec.Role = common.RoleStandby // Remove followers db.Spec.Followers = []string{} db.Spec.FollowConfig = &cluster.FollowConfig{Type: cluster.FollowTypeInternal, DBUID: wantedMasterDBUID} } // Define followers for master DB masterDB.Spec.Followers = []string{} for _, db := range newcd.DBs { if masterDB.UID == db.UID { continue } fc := db.Spec.FollowConfig if fc != nil { if fc.Type == cluster.FollowTypeInternal && fc.DBUID == wantedMasterDBUID { masterDB.Spec.Followers = append(masterDB.Spec.Followers, db.UID) // Sort followers sort.Strings(masterDB.Spec.Followers) } } } } } // Update generation on DBs if they have changed for dbUID, db := range newcd.DBs { prevDB, ok := cd.DBs[dbUID] if !ok { continue } if !reflect.DeepEqual(db.Spec, prevDB.Spec) { log.Debug("db spec changed, updating generation", zap.String("prevDB", spew.Sdump(prevDB.Spec)), zap.String("db", spew.Sdump(db.Spec))) db.Generation++ db.ChangeTime = time.Now() } } default: return nil, fmt.Errorf("unknown cluster phase %s", cd.Cluster.Status.Phase) } // Copy the clusterSpec parameters to the dbSpec s.setDBSpecFromClusterSpec(newcd) return newcd, nil }