// handleOverlap relists all deployment in the same namespace for overlaps, and avoid syncing
// the newer overlapping ones (only sync the oldest one). New/old is determined by when the
// deployment's selector is last updated.
func (dc *DeploymentController) handleOverlap(d *extensions.Deployment) error {
selector, err := unversioned.LabelSelectorAsSelector(d.Spec.Selector)
if err != nil {
return fmt.Errorf("deployment %s/%s has invalid label selector: %v", d.Namespace, d.Name, err)
}
deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything())
if err != nil {
return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err)
}
overlapping := false
for _, other := range deployments {
if !selector.Empty() && selector.Matches(labels.Set(other.Spec.Template.Labels)) && d.UID != other.UID {
deploymentCopy, err := util.DeploymentDeepCopy(other)
if err != nil {
return err
}
overlapping = true
// Skip syncing this one if older overlapping one is found.
if util.SelectorUpdatedBefore(deploymentCopy, d) {
// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
dc.markDeploymentOverlap(d, deploymentCopy.Name)
dc.clearDeploymentOverlap(deploymentCopy)
return fmt.Errorf("found deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, deploymentCopy.Namespace, deploymentCopy.Name)
}
dc.markDeploymentOverlap(deploymentCopy, d.Name)
d, _ = dc.clearDeploymentOverlap(d)
}
}
if !overlapping {
// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
d, _ = dc.clearDeploymentOverlap(d)
}
return nil
}
// handleOverlap relists all deployment in the same namespace for overlaps, and avoid syncing
// the newer overlapping ones (only sync the oldest one). New/old is determined by when the
// deployment's selector is last updated.
func (dc *DeploymentController) handleOverlap(d *extensions.Deployment) error {
deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything())
if err != nil {
return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err)
}
overlapping := false
for _, other := range deployments {
foundOverlaps, err := util.OverlapsWith(d, other)
if err != nil {
return err
}
if foundOverlaps {
deploymentCopy, err := util.DeploymentDeepCopy(other)
if err != nil {
return err
}
overlapping = true
// Skip syncing this one if older overlapping one is found.
if util.SelectorUpdatedBefore(deploymentCopy, d) {
// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
dc.markDeploymentOverlap(d, deploymentCopy.Name)
dc.clearDeploymentOverlap(deploymentCopy)
return fmt.Errorf("found deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, deploymentCopy.Namespace, deploymentCopy.Name)
}
dc.markDeploymentOverlap(deploymentCopy, d.Name)
d, _ = dc.clearDeploymentOverlap(d)
}
}
if !overlapping {
// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
d, _ = dc.clearDeploymentOverlap(d)
}
return nil
}
// syncDeployment will sync the deployment with the given key.
// This function is not meant to be invoked concurrently with the same key.
func (dc *DeploymentController) syncDeployment(key string) error {
startTime := time.Now()
defer func() {
glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime))
}()
obj, exists, err := dc.dStore.Store.GetByKey(key)
if err != nil {
glog.Infof("Unable to retrieve deployment %v from store: %v", key, err)
return err
}
if !exists {
glog.Infof("Deployment has been deleted %v", key)
return nil
}
deployment := obj.(*extensions.Deployment)
everything := unversioned.LabelSelector{}
if reflect.DeepEqual(deployment.Spec.Selector, &everything) {
dc.eventRecorder.Eventf(deployment, api.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.")
return nil
}
// Deep-copy otherwise we are mutating our cache.
// TODO: Deep-copy only when needed.
d, err := util.DeploymentDeepCopy(deployment)
if err != nil {
return err
}
if d.DeletionTimestamp != nil {
return dc.syncStatusOnly(d)
}
if d.Spec.Paused {
return dc.sync(d)
}
if d.Spec.RollbackTo != nil {
revision := d.Spec.RollbackTo.Revision
if _, err = dc.rollback(d, &revision); err != nil {
return err
}
}
if dc.isScalingEvent(d) {
return dc.sync(d)
}
switch d.Spec.Strategy.Type {
case extensions.RecreateDeploymentStrategyType:
return dc.rolloutRecreate(d)
case extensions.RollingUpdateDeploymentStrategyType:
return dc.rolloutRolling(d)
}
return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
}
// handleOverlap relists all deployment in the same namespace for overlaps, and avoid syncing
// the newer overlapping ones (only sync the oldest one). New/old is determined by when the
// deployment's selector is last updated.
func (dc *DeploymentController) handleOverlap(d *extensions.Deployment) error {
selector, err := unversioned.LabelSelectorAsSelector(d.Spec.Selector)
if err != nil {
return fmt.Errorf("deployment %s/%s has invalid label selector: %v", d.Namespace, d.Name, err)
}
deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything())
if err != nil {
return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err)
}
overlapping := false
for _, other := range deployments {
if !selector.Empty() && selector.Matches(labels.Set(other.Spec.Template.Labels)) && d.UID != other.UID {
deploymentCopy, err := util.DeploymentDeepCopy(other)
if err != nil {
return err
}
overlapping = true
// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
d, _ = dc.markDeploymentOverlap(d, other.Name)
deploymentCopy, _ = dc.markDeploymentOverlap(deploymentCopy, d.Name)
// Skip syncing this one if older overlapping one is found
// TODO: figure out a better way to determine which deployment to skip,
// either with controller reference, or with validation.
// Using oldest active replica set to determine which deployment to skip wouldn't make much difference,
// since new replica set hasn't been created after selector update
if util.SelectorUpdatedBefore(deploymentCopy, d) {
return fmt.Errorf("found deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, other.Namespace, other.Name)
}
}
}
if !overlapping {
// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
d, _ = dc.clearDeploymentOverlap(d)
}
return nil
}
// checkForProgress checks the progress for the provided deployment. Meant to be called
// by the progressWorker and work on items synced in a secondary queue.
func (dc *DeploymentController) checkForProgress(key string) (bool, error) {
obj, exists, err := dc.dLister.Indexer.GetByKey(key)
if err != nil {
glog.V(2).Infof("Cannot retrieve deployment %q found in the secondary queue: %#v", key, err)
return false, err
}
if !exists {
return false, nil
}
deployment := obj.(*extensions.Deployment)
cond := util.GetDeploymentCondition(deployment.Status, extensions.DeploymentProgressing)
// Already marked with a terminal reason - no need to add it back to the main queue.
if cond != nil && (cond.Reason == util.TimedOutReason || cond.Reason == util.NewRSAvailableReason) {
return false, nil
}
// Deep-copy otherwise we may mutate our cache.
// TODO: Remove deep-copying from here. This worker does not need to sync the annotations
// in the deployment.
d, err := util.DeploymentDeepCopy(deployment)
if err != nil {
return false, err
}
return dc.hasFailed(d)
}
// handleOverlap will avoid syncing the newer overlapping ones (only sync the oldest one). New/old is
// determined by when the deployment's selector is last updated.
func (dc *DeploymentController) handleOverlap(d *extensions.Deployment, deployments []*extensions.Deployment) (bool, error) {
overlapping := false
var errs []error
for i := range deployments {
otherD := deployments[i]
if d.Name == otherD.Name {
continue
}
// Error is already checked during validation
foundOverlaps, _ := util.OverlapsWith(d, otherD)
// If the otherD deployment overlaps with the current we need to identify which one
// holds the set longer and mark the other as overlapping. Requeue the overlapping
// deployments if this one has been marked deleted, we only update its status as long
// as it is not actually deleted.
if foundOverlaps && d.DeletionTimestamp == nil {
overlapping = true
// Look at the overlapping annotation in both deployments. If one of them has it and points
// to the other one then we don't need to compare their timestamps.
otherOverlapsWith := otherD.Annotations[util.OverlapAnnotation]
currentOverlapsWith := d.Annotations[util.OverlapAnnotation]
// The other deployment is already marked as overlapping with the current one.
if otherOverlapsWith == d.Name {
var err error
if d, err = dc.clearDeploymentOverlap(d, otherD.Name); err != nil {
errs = append(errs, err)
}
continue
}
otherCopy, err := util.DeploymentDeepCopy(otherD)
if err != nil {
return false, err
}
// Skip syncing this one if older overlapping one is found.
if currentOverlapsWith == otherCopy.Name || util.SelectorUpdatedBefore(otherCopy, d) {
if _, err = dc.markDeploymentOverlap(d, otherCopy.Name); err != nil {
return false, err
}
if _, err = dc.clearDeploymentOverlap(otherCopy, d.Name); err != nil {
return false, err
}
return true, fmt.Errorf("deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, otherCopy.Namespace, otherCopy.Name)
}
// TODO: We need to support annotations in deployments that overlap with multiple other
// deployments.
if _, err = dc.markDeploymentOverlap(otherCopy, d.Name); err != nil {
errs = append(errs, err)
}
// This is going to get some deployments into update hotlooping if we remove the overlapping
// annotation unconditionally.
//
// Scenario:
// --> Deployment foo with label selector A=A is created.
// --> Deployment bar with label selector A=A,B=B is created. Marked as overlapping since it
// overlaps with foo.
// --> Deployment baz with label selector B=B is created. Marked as overlapping, since it
// overlaps with bar, bar overlapping annotation is cleaned up. Next sync loop marks bar
// as overlapping and it gets in an update hotloop.
if d, err = dc.clearDeploymentOverlap(d, otherCopy.Name); err != nil {
errs = append(errs, err)
}
continue
}
// If the otherD deployment does not overlap with the current deployment *anymore*
// we need to cleanup otherD from the overlapping annotation so it can be synced by
// the deployment controller.
dName, hasOverlappingAnnotation := otherD.Annotations[util.OverlapAnnotation]
if hasOverlappingAnnotation && dName == d.Name {
otherCopy, err := util.DeploymentDeepCopy(otherD)
if err != nil {
return false, err
}
if _, err = dc.clearDeploymentOverlap(otherCopy, d.Name); err != nil {
errs = append(errs, err)
}
}
}
if !overlapping {
var err error
if d, err = dc.clearDeploymentOverlap(d, ""); err != nil {
errs = append(errs, err)
}
}
return false, utilerrors.NewAggregate(errs)
}
// syncDeployment will sync the deployment with the given key.
// This function is not meant to be invoked concurrently with the same key.
func (dc *DeploymentController) syncDeployment(key string) error {
startTime := time.Now()
glog.V(4).Infof("Started syncing deployment %q (%v)", key, startTime)
defer func() {
glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime))
}()
obj, exists, err := dc.dLister.Indexer.GetByKey(key)
if err != nil {
utilruntime.HandleError(fmt.Errorf("Unable to retrieve deployment %v from store: %v", key, err))
return err
}
if !exists {
glog.Infof("Deployment has been deleted %v", key)
return nil
}
deployment := obj.(*extensions.Deployment)
// Deep-copy otherwise we are mutating our cache.
// TODO: Deep-copy only when needed.
d, err := util.DeploymentDeepCopy(deployment)
if err != nil {
return err
}
everything := metav1.LabelSelector{}
if reflect.DeepEqual(d.Spec.Selector, &everything) {
dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.")
if d.Status.ObservedGeneration < d.Generation {
d.Status.ObservedGeneration = d.Generation
dc.client.Extensions().Deployments(d.Namespace).UpdateStatus(d)
}
return nil
}
deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything())
if err != nil {
return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err)
}
// Handle overlapping deployments by deterministically avoid syncing deployments that fight over ReplicaSets.
overlaps, err := dc.handleOverlap(d, deployments)
if err != nil {
if overlaps {
// Emit an event and return a nil error for overlapping deployments so we won't resync them again.
dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectorOverlap", err.Error())
return nil
}
// For any other failure, we should retry the deployment.
return err
}
if d.DeletionTimestamp != nil {
return dc.syncStatusOnly(d)
}
// Why run the cleanup policy only when there is no rollback request?
// The thing with the cleanup policy currently is that it is far from smart because it takes into account
// the latest replica sets while it should instead retain the latest *working* replica sets. This means that
// you can have a cleanup policy of 1 but your last known working replica set may be 2 or 3 versions back
// in the history.
// Eventually we will want to find a way to recognize replica sets that have worked at some point in time
// (and chances are higher that they will work again as opposed to others that didn't) for candidates to
// automatically roll back to (#23211) and the cleanup policy should help.
if d.Spec.RollbackTo == nil {
_, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(d, false)
if err != nil {
return err
}
// So far the cleanup policy was executed once a deployment was paused, scaled up/down, or it
// succesfully completed deploying a replica set. Decouple it from the strategies and have it
// run almost unconditionally - cleanupDeployment is safe by default.
dc.cleanupDeployment(oldRSs, d)
}
err = dc.classifyReplicaSets(d)
if err != nil {
return err
}
// Update deployment conditions with an Unknown condition when pausing/resuming
// a deployment. In this way, we can be sure that we won't timeout when a user
// resumes a Deployment with a set progressDeadlineSeconds.
if err = dc.checkPausedConditions(d); err != nil {
return err
}
_, err = dc.hasFailed(d)
if err != nil {
return err
}
// TODO: Automatically rollback here if we failed above. Locate the last complete
// revision and populate the rollback spec with it.
// See https://github.com/kubernetes/kubernetes/issues/23211.
if d.Spec.Paused {
return dc.sync(d)
}
if d.Spec.RollbackTo != nil {
revision := d.Spec.RollbackTo.Revision
if d, err = dc.rollback(d, &revision); err != nil {
return err
}
}
scalingEvent, err := dc.isScalingEvent(d)
if err != nil {
return err
}
if scalingEvent {
return dc.sync(d)
}
switch d.Spec.Strategy.Type {
case extensions.RecreateDeploymentStrategyType:
return dc.rolloutRecreate(d)
case extensions.RollingUpdateDeploymentStrategyType:
return dc.rolloutRolling(d)
}
return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
}
// syncDeployment will sync the deployment with the given key.
// This function is not meant to be invoked concurrently with the same key.
func (dc *DeploymentController) syncDeployment(key string) error {
startTime := time.Now()
defer func() {
glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime))
}()
obj, exists, err := dc.dLister.Indexer.GetByKey(key)
if err != nil {
glog.Errorf("Unable to retrieve deployment %v from store: %v", key, err)
return err
}
if !exists {
glog.Infof("Deployment has been deleted %v", key)
return nil
}
deployment := obj.(*extensions.Deployment)
// Deep-copy otherwise we are mutating our cache.
// TODO: Deep-copy only when needed.
d, err := util.DeploymentDeepCopy(deployment)
if err != nil {
return err
}
everything := unversioned.LabelSelector{}
if reflect.DeepEqual(d.Spec.Selector, &everything) {
dc.eventRecorder.Eventf(d, api.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.")
if d.Status.ObservedGeneration < d.Generation {
d.Status.ObservedGeneration = d.Generation
dc.client.Extensions().Deployments(d.Namespace).UpdateStatus(d)
}
return nil
}
if d.DeletionTimestamp != nil {
return dc.syncStatusOnly(d)
}
// Handle overlapping deployments by deterministically avoid syncing deployments that fight over ReplicaSets.
if err = dc.handleOverlap(d); err != nil {
dc.eventRecorder.Eventf(d, api.EventTypeWarning, "SelectorOverlap", err.Error())
return nil
}
if d.Spec.Paused {
return dc.sync(d)
}
if d.Spec.RollbackTo != nil {
revision := d.Spec.RollbackTo.Revision
if d, err = dc.rollback(d, &revision); err != nil {
return err
}
}
scalingEvent, err := dc.isScalingEvent(d)
if err != nil {
return err
}
if scalingEvent {
return dc.sync(d)
}
switch d.Spec.Strategy.Type {
case extensions.RecreateDeploymentStrategyType:
return dc.rolloutRecreate(d)
case extensions.RollingUpdateDeploymentStrategyType:
return dc.rolloutRolling(d)
}
return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
}
// syncDeployment will sync the deployment with the given key.
// This function is not meant to be invoked concurrently with the same key.
func (dc *DeploymentController) syncDeployment(key string) error {
startTime := time.Now()
defer func() {
glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime))
}()
obj, exists, err := dc.dLister.Indexer.GetByKey(key)
if err != nil {
glog.Errorf("Unable to retrieve deployment %v from store: %v", key, err)
return err
}
if !exists {
glog.Infof("Deployment has been deleted %v", key)
return nil
}
deployment := obj.(*extensions.Deployment)
// Deep-copy otherwise we are mutating our cache.
// TODO: Deep-copy only when needed.
d, err := util.DeploymentDeepCopy(deployment)
if err != nil {
return err
}
everything := metav1.LabelSelector{}
if reflect.DeepEqual(d.Spec.Selector, &everything) {
dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.")
if d.Status.ObservedGeneration < d.Generation {
d.Status.ObservedGeneration = d.Generation
dc.client.Extensions().Deployments(d.Namespace).UpdateStatus(d)
}
return nil
}
if d.DeletionTimestamp != nil {
return dc.syncStatusOnly(d)
}
// Handle overlapping deployments by deterministically avoid syncing deployments that fight over ReplicaSets.
if err = dc.handleOverlap(d); err != nil {
dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectorOverlap", err.Error())
return nil
}
// Update deployment conditions with an Unknown condition when pausing/resuming
// a deployment. In this way, we can be sure that we won't timeout when a user
// resumes a Deployment with a set progressDeadlineSeconds.
if err = dc.checkPausedConditions(d); err != nil {
return err
}
_, err = dc.hasFailed(d)
if err != nil {
return err
}
// TODO: Automatically rollback here if we failed above. Locate the last complete
// revision and populate the rollback spec with it.
// See https://github.com/kubernetes/kubernetes/issues/23211.
if d.Spec.Paused {
return dc.sync(d)
}
if d.Spec.RollbackTo != nil {
revision := d.Spec.RollbackTo.Revision
if d, err = dc.rollback(d, &revision); err != nil {
return err
}
}
scalingEvent, err := dc.isScalingEvent(d)
if err != nil {
return err
}
if scalingEvent {
return dc.sync(d)
}
switch d.Spec.Strategy.Type {
case extensions.RecreateDeploymentStrategyType:
return dc.rolloutRecreate(d)
case extensions.RollingUpdateDeploymentStrategyType:
return dc.rolloutRolling(d)
}
return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
}