func (s *Sentinel) updateCluster(cd *cluster.ClusterData) (*cluster.ClusterData, error) {
newcd := cd.DeepCopy()
switch cd.Cluster.Status.Phase {
case cluster.ClusterPhaseInitializing:
switch cd.Cluster.Spec.InitMode {
case cluster.ClusterInitModeNew:
// Is there already a keeper choosed to be the new master?
if cd.Cluster.Status.Master == "" {
log.Info("trying to find initial master")
k, err := s.findInitialKeeper(cd)
if err != nil {
return nil, fmt.Errorf("cannot choose initial master: %v", err)
}
log.Info("initializing cluster", zap.String("keeper", k.UID))
db := &cluster.DB{
UID: s.UIDFn(),
Generation: cluster.InitialGeneration,
ChangeTime: time.Now(),
Spec: &cluster.DBSpec{
KeeperUID: k.UID,
InitMode: cluster.DBInitModeNew,
Role: common.RoleMaster,
Followers: []string{},
IncludeConfig: *cd.Cluster.Spec.MergePgParameters,
},
}
newcd.DBs[db.UID] = db
newcd.Cluster.Status.Master = db.UID
log.Debug("newcd dump", zap.String("newcd", spew.Sdump(newcd)))
} else {
db, ok := cd.DBs[cd.Cluster.Status.Master]
if !ok {
panic(fmt.Errorf("db %q object doesn't exists. This shouldn't happen", cd.Cluster.Status.Master))
}
// Check that the choosed db for being the master has correctly initialized
switch s.dbConvergenceState(cd, db, cd.Cluster.Spec.InitTimeout.Duration) {
case Converged:
if db.Status.Healthy {
log.Info("db initialized", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID))
// Set db initMode to none, not needed but just a security measure
db.Spec.InitMode = cluster.DBInitModeNone
// Don't include previous config anymore
db.Spec.IncludeConfig = false
// Replace reported pg parameters in cluster spec
if *cd.Cluster.Spec.MergePgParameters {
newcd.Cluster.Spec.PGParameters = db.Status.PGParameters
}
// Cluster initialized, switch to Normal state
newcd.Cluster.Status.Phase = cluster.ClusterPhaseNormal
}
case Converging:
log.Info("waiting for db", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID))
case ConvergenceFailed:
log.Info("db failed to initialize", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID))
// Empty DBs
newcd.DBs = cluster.DBs{}
// Unset master so another keeper can be choosen
newcd.Cluster.Status.Master = ""
}
}
case cluster.ClusterInitModeExisting:
if cd.Cluster.Status.Master == "" {
wantedKeeper := cd.Cluster.Spec.ExistingConfig.KeeperUID
log.Info("trying to use keeper as initial master", zap.String("keeper", wantedKeeper))
k, ok := cd.Keepers[wantedKeeper]
if !ok {
return nil, fmt.Errorf("keeper %q state not available", wantedKeeper)
}
log.Info("initializing cluster using selected keeper as master db owner", zap.String("keeper", k.UID))
db := &cluster.DB{
UID: s.UIDFn(),
Generation: cluster.InitialGeneration,
ChangeTime: time.Now(),
Spec: &cluster.DBSpec{
KeeperUID: k.UID,
InitMode: cluster.DBInitModeExisting,
Role: common.RoleMaster,
Followers: []string{},
IncludeConfig: *cd.Cluster.Spec.MergePgParameters,
},
}
newcd.DBs[db.UID] = db
newcd.Cluster.Status.Master = db.UID
log.Debug("newcd dump", zap.String("newcd", spew.Sdump(newcd)))
} else {
db, ok := newcd.DBs[cd.Cluster.Status.Master]
if !ok {
panic(fmt.Errorf("db %q object doesn't exists. This shouldn't happen", cd.Cluster.Status.Master))
}
// Check that the choosed db for being the master has correctly initialized
// TODO(sgotti) set a timeout (the max time for a noop operation, just a start/restart)
if db.Status.Healthy && s.dbConvergenceState(cd, db, 0) == Converged {
log.Info("db initialized", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID))
// Don't include previous config anymore
db.Spec.IncludeConfig = false
// Replace reported pg parameters in cluster spec
if *cd.Cluster.Spec.MergePgParameters {
newcd.Cluster.Spec.PGParameters = db.Status.PGParameters
}
// Cluster initialized, switch to Normal state
newcd.Cluster.Status.Phase = cluster.ClusterPhaseNormal
}
}
case cluster.ClusterInitModePITR:
// Is there already a keeper choosed to be the new master?
if cd.Cluster.Status.Master == "" {
log.Info("trying to find initial master")
k, err := s.findInitialKeeper(cd)
if err != nil {
return nil, fmt.Errorf("cannot choose initial master: %v", err)
}
log.Info("initializing cluster using selected keeper as master db owner", zap.String("keeper", k.UID))
db := &cluster.DB{
UID: s.UIDFn(),
Generation: cluster.InitialGeneration,
ChangeTime: time.Now(),
Spec: &cluster.DBSpec{
KeeperUID: k.UID,
InitMode: cluster.DBInitModePITR,
PITRConfig: cd.Cluster.Spec.PITRConfig,
Role: common.RoleMaster,
Followers: []string{},
IncludeConfig: *cd.Cluster.Spec.MergePgParameters,
},
}
newcd.DBs[db.UID] = db
newcd.Cluster.Status.Master = db.UID
log.Debug("newcd dump", zap.String("newcd", spew.Sdump(newcd)))
} else {
db, ok := cd.DBs[cd.Cluster.Status.Master]
if !ok {
panic(fmt.Errorf("db %q object doesn't exists. This shouldn't happen", cd.Cluster.Status.Master))
}
// Check that the choosed db for being the master has correctly initialized
// TODO(sgotti) set a timeout (the max time for a restore operation)
switch s.dbConvergenceState(cd, db, 0) {
case Converged:
if db.Status.Healthy {
log.Info("db initialized", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID))
// Set db initMode to none, not needed but just a security measure
db.Spec.InitMode = cluster.DBInitModeNone
// Don't include previous config anymore
db.Spec.IncludeConfig = false
// Replace reported pg parameters in cluster spec
if *cd.Cluster.Spec.MergePgParameters {
newcd.Cluster.Spec.PGParameters = db.Status.PGParameters
}
// Cluster initialized, switch to Normal state
newcd.Cluster.Status.Phase = cluster.ClusterPhaseNormal
}
case Converging:
log.Info("waiting for db to converge", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID))
case ConvergenceFailed:
log.Info("db failed to initialize", zap.String("db", db.UID), zap.String("keeper", db.Spec.KeeperUID))
// Empty DBs
newcd.DBs = cluster.DBs{}
// Unset master so another keeper can be choosen
newcd.Cluster.Status.Master = ""
}
}
default:
return nil, fmt.Errorf("unknown init mode %q", cd.Cluster.Spec.InitMode)
}
case cluster.ClusterPhaseNormal:
// Add missing DBs
for _, k := range cd.Keepers {
if db := cd.FindDB(k); db == nil {
db := &cluster.DB{
UID: s.UIDFn(),
Generation: cluster.InitialGeneration,
ChangeTime: time.Now(),
Spec: &cluster.DBSpec{
KeeperUID: k.UID,
InitMode: cluster.DBInitModeNone,
Role: common.RoleUndefined,
Followers: []string{},
},
}
newcd.DBs[db.UID] = db
}
}
// TODO(sgotti) When keeper removal is implemented, remove DBs for unexistent keepers
// Calculate current master status
curMasterDBUID := cd.Cluster.Status.Master
wantedMasterDBUID := curMasterDBUID
masterOK := true
curMasterDB := cd.DBs[curMasterDBUID]
if curMasterDB == nil {
return nil, fmt.Errorf("db for keeper %q not available. This shouldn't happen!", curMasterDBUID)
}
log.Debug("db dump", zap.String("db", spew.Sdump(curMasterDB)))
if !curMasterDB.Status.Healthy {
log.Info("master db is failed", zap.String("db", curMasterDB.UID), zap.String("keeper", curMasterDB.Spec.KeeperUID))
masterOK = false
}
// Check that the wanted master is in master state (i.e. check that promotion from standby to master happened)
if s.dbConvergenceState(cd, curMasterDB, newcd.Cluster.Spec.ConvergenceTimeout.Duration) == ConvergenceFailed {
log.Info("db not converged", zap.String("db", curMasterDB.UID), zap.String("keeper", curMasterDB.Spec.KeeperUID))
masterOK = false
}
if !masterOK {
log.Info("trying to find a standby to replace failed master")
bestStandbyDB, err := s.findBestStandby(cd, curMasterDB)
if err != nil {
log.Error("error trying to find the best standby", zap.Error(err))
} else {
log.Info("electing db as the new master", zap.String("db", bestStandbyDB.UID), zap.String("keeper", bestStandbyDB.Spec.KeeperUID))
wantedMasterDBUID = bestStandbyDB.UID
}
}
// New master elected
if curMasterDBUID != wantedMasterDBUID {
// maintain the current role, remove followers
oldMasterdb := newcd.DBs[curMasterDBUID]
oldMasterdb.Spec.Followers = []string{}
newcd.Cluster.Status.Master = wantedMasterDBUID
newMasterDB := newcd.DBs[wantedMasterDBUID]
newMasterDB.Spec.Role = common.RoleMaster
newMasterDB.Spec.FollowConfig = nil
// Tell proxy that there's currently no active master
newcd.Proxy.Spec.MasterDBUID = ""
newcd.Proxy.ChangeTime = time.Now()
}
// TODO(sgotti) Wait for the proxies being converged (closed connections to old master)?
// Setup standbys, do this only when there's no master change
if curMasterDBUID == wantedMasterDBUID {
masterDB := newcd.DBs[curMasterDBUID]
// Set standbys to follow master only if it's healthy and converged
if masterDB.Status.Healthy && s.dbConvergenceState(newcd, masterDB, newcd.Cluster.Spec.ConvergenceTimeout.Duration) == Converged {
// Tell proxy that there's a new active master
newcd.Proxy.Spec.MasterDBUID = wantedMasterDBUID
newcd.Proxy.ChangeTime = time.Now()
// TODO(sgotti) do this only for the defined number of MaxStandbysPerSender (needs also to detect unhealthy standbys and switch to healthy one)
for id, db := range newcd.DBs {
if id == wantedMasterDBUID {
continue
}
db.Spec.Role = common.RoleStandby
// Remove followers
db.Spec.Followers = []string{}
db.Spec.FollowConfig = &cluster.FollowConfig{Type: cluster.FollowTypeInternal, DBUID: wantedMasterDBUID}
}
// Define followers for master DB
masterDB.Spec.Followers = []string{}
for _, db := range newcd.DBs {
if masterDB.UID == db.UID {
continue
}
fc := db.Spec.FollowConfig
if fc != nil {
if fc.Type == cluster.FollowTypeInternal && fc.DBUID == wantedMasterDBUID {
masterDB.Spec.Followers = append(masterDB.Spec.Followers, db.UID)
// Sort followers
sort.Strings(masterDB.Spec.Followers)
}
}
}
}
}
// Update generation on DBs if they have changed
for dbUID, db := range newcd.DBs {
prevDB, ok := cd.DBs[dbUID]
if !ok {
continue
}
if !reflect.DeepEqual(db.Spec, prevDB.Spec) {
log.Debug("db spec changed, updating generation", zap.String("prevDB", spew.Sdump(prevDB.Spec)), zap.String("db", spew.Sdump(db.Spec)))
db.Generation++
db.ChangeTime = time.Now()
}
}
default:
return nil, fmt.Errorf("unknown cluster phase %s", cd.Cluster.Status.Phase)
}
// Copy the clusterSpec parameters to the dbSpec
s.setDBSpecFromClusterSpec(newcd)
return newcd, nil
}
