// isScalingEvent checks whether the provided deployment has been updated with a scaling event // by looking at the desired-replicas annotation in the active replica sets of the deployment. func (dc *DeploymentController) isScalingEvent(d *extensions.Deployment) (bool, error) { newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(d, false) if err != nil { return false, err } // If there is no new replica set matching this deployment and the deployment isn't paused // then there is a new rollout that waits to happen if newRS == nil && !d.Spec.Paused { // Update all active replicas sets to the new deployment size. SetReplicasAnnotations makes // sure that we will update only replica sets that don't have the current size of the deployment. maxSurge := deploymentutil.MaxSurge(*d) for _, rs := range controller.FilterActiveReplicaSets(oldRSs) { if updated := deploymentutil.SetReplicasAnnotations(rs, d.Spec.Replicas, d.Spec.Replicas+maxSurge); updated { if _, err := dc.client.Extensions().ReplicaSets(rs.Namespace).Update(rs); err != nil { glog.Infof("Cannot update annotations for replica set %q: %v", rs.Name, err) return false, err } } } return false, nil } allRSs := append(oldRSs, newRS) for _, rs := range controller.FilterActiveReplicaSets(allRSs) { desired, ok := deploymentutil.GetDesiredReplicasAnnotation(rs) if !ok { continue } if desired != d.Spec.Replicas { return true, nil } } return false, nil }
func (dc *DeploymentController) syncRollingUpdateDeployment(deployment extensions.Deployment) error { newRS, oldRSs, err := dc.getAllReplicaSets(deployment, true) if err != nil { return err } allRSs := append(controller.FilterActiveReplicaSets(oldRSs), newRS) // Scale up, if we can. scaledUp, err := dc.reconcileNewReplicaSet(allRSs, newRS, deployment) if err != nil { return err } if scaledUp { // Update DeploymentStatus return dc.updateDeploymentStatus(allRSs, newRS, deployment) } // Scale down, if we can. scaledDown, err := dc.reconcileOldReplicaSets(allRSs, controller.FilterActiveReplicaSets(oldRSs), newRS, deployment, true) if err != nil { return err } if scaledDown { // Update DeploymentStatus return dc.updateDeploymentStatus(allRSs, newRS, deployment) } if deployment.Spec.RevisionHistoryLimit != nil { // Cleanup old replicas sets dc.cleanupOldReplicaSets(oldRSs, deployment) } // Sync deployment status return dc.syncDeploymentStatus(allRSs, newRS, deployment) }
func (dc *DeploymentController) syncRecreateDeployment(deployment extensions.Deployment) error { newRS, oldRSs, err := dc.getAllReplicaSets(deployment) if err != nil { return err } allRSs := append(controller.FilterActiveReplicaSets(oldRSs), newRS) // scale down old replica sets scaledDown, err := dc.scaleDownOldReplicaSetsForRecreate(controller.FilterActiveReplicaSets(oldRSs), deployment) if err != nil { return err } if scaledDown { // Update DeploymentStatus return dc.updateDeploymentStatus(allRSs, newRS, deployment) } // scale up new replica set scaledUp, err := dc.scaleUpNewReplicaSetForRecreate(newRS, deployment) if err != nil { return err } if scaledUp { // Update DeploymentStatus return dc.updateDeploymentStatus(allRSs, newRS, deployment) } if deployment.Spec.RevisionHistoryLimit != nil { // Cleanup old replica sets dc.cleanupOldReplicaSets(oldRSs, deployment) } // Sync deployment status return dc.syncDeploymentStatus(allRSs, newRS, deployment) }
// rolloutRolling implements the logic for rolling a new replica set. func (dc *DeploymentController) rolloutRolling(deployment *extensions.Deployment) error { newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(deployment, true) if err != nil { return err } allRSs := append(oldRSs, newRS) // Scale up, if we can. scaledUp, err := dc.reconcileNewReplicaSet(allRSs, newRS, deployment) if err != nil { return err } if scaledUp { // Update DeploymentStatus return dc.syncRolloutStatus(allRSs, newRS, deployment) } // Scale down, if we can. scaledDown, err := dc.reconcileOldReplicaSets(allRSs, controller.FilterActiveReplicaSets(oldRSs), newRS, deployment) if err != nil { return err } if scaledDown { // Update DeploymentStatus return dc.syncRolloutStatus(allRSs, newRS, deployment) } // Sync deployment status return dc.syncRolloutStatus(allRSs, newRS, deployment) }
func (dc *DeploymentController) syncRecreateDeployment(deployment extensions.Deployment) error { // Don't create a new RS if not already existed, so that we avoid scaling up before scaling down newRS, oldRSs, err := dc.getAllReplicaSets(deployment, false) if err != nil { return err } allRSs := append(controller.FilterActiveReplicaSets(oldRSs), newRS) // scale down old replica sets scaledDown, err := dc.scaleDownOldReplicaSetsForRecreate(controller.FilterActiveReplicaSets(oldRSs), deployment) if err != nil { return err } if scaledDown { // Update DeploymentStatus return dc.updateDeploymentStatus(allRSs, newRS, deployment) } // If we need to create a new RS, create it now // TODO: Create a new RS without re-listing all RSs. if newRS == nil { newRS, oldRSs, err = dc.getAllReplicaSets(deployment, true) if err != nil { return err } allRSs = append(oldRSs, newRS) } // scale up new replica set scaledUp, err := dc.scaleUpNewReplicaSetForRecreate(newRS, deployment) if err != nil { return err } if scaledUp { // Update DeploymentStatus return dc.updateDeploymentStatus(allRSs, newRS, deployment) } if deployment.Spec.RevisionHistoryLimit != nil { // Cleanup old replica sets dc.cleanupOldReplicaSets(oldRSs, deployment) } // Sync deployment status return dc.syncDeploymentStatus(allRSs, newRS, deployment) }
// rolloutRecreate implements the logic for recreating a replica set. func (dc *DeploymentController) rolloutRecreate(deployment *extensions.Deployment) error { // Don't create a new RS if not already existed, so that we avoid scaling up before scaling down newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(deployment, false) if err != nil { return err } allRSs := append(oldRSs, newRS) activeOldRSs := controller.FilterActiveReplicaSets(oldRSs) // scale down old replica sets scaledDown, err := dc.scaleDownOldReplicaSetsForRecreate(activeOldRSs, deployment) if err != nil { return err } if scaledDown { // Update DeploymentStatus return dc.syncRolloutStatus(allRSs, newRS, deployment) } newStatus := calculateStatus(allRSs, newRS, deployment) // Do not process a deployment when it has old pods running. if newStatus.UpdatedReplicas == 0 { podList, err := dc.listPods(deployment) if err != nil { return err } if len(podList.Items) > 0 { return dc.syncRolloutStatus(allRSs, newRS, deployment) } } // If we need to create a new RS, create it now // TODO: Create a new RS without re-listing all RSs. if newRS == nil { newRS, oldRSs, err = dc.getAllReplicaSetsAndSyncRevision(deployment, true) if err != nil { return err } allRSs = append(oldRSs, newRS) } // scale up new replica set scaledUp, err := dc.scaleUpNewReplicaSetForRecreate(newRS, deployment) if err != nil { return err } if scaledUp { // Update DeploymentStatus return dc.syncRolloutStatus(allRSs, newRS, deployment) } dc.cleanupDeployment(oldRSs, deployment) // Sync deployment status return dc.syncRolloutStatus(allRSs, newRS, deployment) }
// Updates the status of a paused deployment func (dc *DeploymentController) syncPausedDeploymentStatus(deployment *extensions.Deployment) error { newRS, oldRSs, err := dc.getAllReplicaSets(deployment, false) if err != nil { return err } allRSs := append(controller.FilterActiveReplicaSets(oldRSs), newRS) // Sync deployment status return dc.syncDeploymentStatus(allRSs, newRS, deployment) }
// rolloutRecreate implements the logic for recreating a replica set. func (dc *DeploymentController) rolloutRecreate(deployment *extensions.Deployment) error { // Don't create a new RS if not already existed, so that we avoid scaling up before scaling down newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(deployment, false) if err != nil { return err } allRSs := append(oldRSs, newRS) activeOldRSs := controller.FilterActiveReplicaSets(oldRSs) // scale down old replica sets scaledDown, err := dc.scaleDownOldReplicaSetsForRecreate(activeOldRSs, deployment) if err != nil { return err } if scaledDown { // Update DeploymentStatus return dc.syncDeploymentStatus(allRSs, newRS, deployment) } // Wait for all old replica set to scale down to zero. if err := dc.waitForInactiveReplicaSets(activeOldRSs); err != nil { return err } // If we need to create a new RS, create it now // TODO: Create a new RS without re-listing all RSs. if newRS == nil { newRS, oldRSs, err = dc.getAllReplicaSetsAndSyncRevision(deployment, true) if err != nil { return err } allRSs = append(oldRSs, newRS) } // scale up new replica set scaledUp, err := dc.scaleUpNewReplicaSetForRecreate(newRS, deployment) if err != nil { return err } if scaledUp { // Update DeploymentStatus return dc.syncDeploymentStatus(allRSs, newRS, deployment) } dc.cleanupDeployment(oldRSs, deployment) // Sync deployment status return dc.syncDeploymentStatus(allRSs, newRS, deployment) }
// isScalingEvent checks whether the provided deployment has been updated with a scaling event // by looking at the desired-replicas annotation in the active replica sets of the deployment. func (dc *DeploymentController) isScalingEvent(d *extensions.Deployment) (bool, error) { newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(d, false) if err != nil { return false, err } allRSs := append(oldRSs, newRS) for _, rs := range controller.FilterActiveReplicaSets(allRSs) { desired, ok := deploymentutil.GetDesiredReplicasAnnotation(rs) if !ok { continue } if desired != *(d.Spec.Replicas) { return true, nil } } return false, nil }
// FindActiveOrLatest returns the only active or the latest replica set in case there is at most one active // replica set. If there are more active replica sets, then we should proportionally scale them. func FindActiveOrLatest(newRS *extensions.ReplicaSet, oldRSs []*extensions.ReplicaSet) *extensions.ReplicaSet { if newRS == nil && len(oldRSs) == 0 { return nil } sort.Sort(sort.Reverse(controller.ReplicaSetsByCreationTimestamp(oldRSs))) allRSs := controller.FilterActiveReplicaSets(append(oldRSs, newRS)) switch len(allRSs) { case 0: // If there is no active replica set then we should return the newest. if newRS != nil { return newRS } return oldRSs[0] case 1: return allRSs[0] default: return nil } }
// isScalingEvent checks whether the provided deployment has been updated with a scaling event // by looking at the desired-replicas annotation in the active replica sets of the deployment. func (dc *DeploymentController) isScalingEvent(d *extensions.Deployment) bool { newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(d, false) if err != nil { return false } // If there is no new replica set matching this deployment and the deployment isn't paused // then there is a new rollout that waits to happen if newRS == nil && !d.Spec.Paused { return false } allRSs := append(oldRSs, newRS) for _, rs := range controller.FilterActiveReplicaSets(allRSs) { desired, ok := getDesiredReplicasAnnotation(rs) if !ok { continue } if desired != d.Spec.Replicas { return true } } return false }
// scale scales proportionally in order to mitigate risk. Otherwise, scaling up can increase the size // of the new replica set and scaling down can decrease the sizes of the old ones, both of which would // have the effect of hastening the rollout progress, which could produce a higher proportion of unavailable // replicas in the event of a problem with the rolled out template. Should run only on scaling events or // when a deployment is paused and not during the normal rollout process. func (dc *DeploymentController) scale(deployment *extensions.Deployment, newRS *extensions.ReplicaSet, oldRSs []*extensions.ReplicaSet) error { // If there is only one active replica set then we should scale that up to the full count of the // deployment. If there is no active replica set, then we should scale up the newest replica set. if activeOrLatest := deploymentutil.FindActiveOrLatest(newRS, oldRSs); activeOrLatest != nil { if *(activeOrLatest.Spec.Replicas) == *(deployment.Spec.Replicas) { return nil } _, _, err := dc.scaleReplicaSetAndRecordEvent(activeOrLatest, *(deployment.Spec.Replicas), deployment) return err } // If the new replica set is saturated, old replica sets should be fully scaled down. // This case handles replica set adoption during a saturated new replica set. if deploymentutil.IsSaturated(deployment, newRS) { for _, old := range controller.FilterActiveReplicaSets(oldRSs) { if _, _, err := dc.scaleReplicaSetAndRecordEvent(old, 0, deployment); err != nil { return err } } return nil } // There are old replica sets with pods and the new replica set is not saturated. // We need to proportionally scale all replica sets (new and old) in case of a // rolling deployment. if deploymentutil.IsRollingUpdate(deployment) { allRSs := controller.FilterActiveReplicaSets(append(oldRSs, newRS)) allRSsReplicas := deploymentutil.GetReplicaCountForReplicaSets(allRSs) allowedSize := int32(0) if *(deployment.Spec.Replicas) > 0 { allowedSize = *(deployment.Spec.Replicas) + deploymentutil.MaxSurge(*deployment) } // Number of additional replicas that can be either added or removed from the total // replicas count. These replicas should be distributed proportionally to the active // replica sets. deploymentReplicasToAdd := allowedSize - allRSsReplicas // The additional replicas should be distributed proportionally amongst the active // replica sets from the larger to the smaller in size replica set. Scaling direction // drives what happens in case we are trying to scale replica sets of the same size. // In such a case when scaling up, we should scale up newer replica sets first, and // when scaling down, we should scale down older replica sets first. var scalingOperation string switch { case deploymentReplicasToAdd > 0: sort.Sort(controller.ReplicaSetsBySizeNewer(allRSs)) scalingOperation = "up" case deploymentReplicasToAdd < 0: sort.Sort(controller.ReplicaSetsBySizeOlder(allRSs)) scalingOperation = "down" } // Iterate over all active replica sets and estimate proportions for each of them. // The absolute value of deploymentReplicasAdded should never exceed the absolute // value of deploymentReplicasToAdd. deploymentReplicasAdded := int32(0) nameToSize := make(map[string]int32) for i := range allRSs { rs := allRSs[i] // Estimate proportions if we have replicas to add, otherwise simply populate // nameToSize with the current sizes for each replica set. if deploymentReplicasToAdd != 0 { proportion := deploymentutil.GetProportion(rs, *deployment, deploymentReplicasToAdd, deploymentReplicasAdded) nameToSize[rs.Name] = *(rs.Spec.Replicas) + proportion deploymentReplicasAdded += proportion } else { nameToSize[rs.Name] = *(rs.Spec.Replicas) } } // Update all replica sets for i := range allRSs { rs := allRSs[i] // Add/remove any leftovers to the largest replica set. if i == 0 && deploymentReplicasToAdd != 0 { leftover := deploymentReplicasToAdd - deploymentReplicasAdded nameToSize[rs.Name] = nameToSize[rs.Name] + leftover if nameToSize[rs.Name] < 0 { nameToSize[rs.Name] = 0 } } // TODO: Use transactions when we have them. if _, err := dc.scaleReplicaSet(rs, nameToSize[rs.Name], deployment, scalingOperation); err != nil { // Return as soon as we fail, the deployment is requeued return err } } } return nil }