// handleOverlap relists all deployment in the same namespace for overlaps, and avoid syncing // the newer overlapping ones (only sync the oldest one). New/old is determined by when the // deployment's selector is last updated. func (dc *DeploymentController) handleOverlap(d *extensions.Deployment) error { selector, err := unversioned.LabelSelectorAsSelector(d.Spec.Selector) if err != nil { return fmt.Errorf("deployment %s/%s has invalid label selector: %v", d.Namespace, d.Name, err) } deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything()) if err != nil { return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err) } overlapping := false for _, other := range deployments { if !selector.Empty() && selector.Matches(labels.Set(other.Spec.Template.Labels)) && d.UID != other.UID { deploymentCopy, err := util.DeploymentDeepCopy(other) if err != nil { return err } overlapping = true // Skip syncing this one if older overlapping one is found. if util.SelectorUpdatedBefore(deploymentCopy, d) { // We don't care if the overlapping annotation update failed or not (we don't make decision on it) dc.markDeploymentOverlap(d, deploymentCopy.Name) dc.clearDeploymentOverlap(deploymentCopy) return fmt.Errorf("found deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, deploymentCopy.Namespace, deploymentCopy.Name) } dc.markDeploymentOverlap(deploymentCopy, d.Name) d, _ = dc.clearDeploymentOverlap(d) } } if !overlapping { // We don't care if the overlapping annotation update failed or not (we don't make decision on it) d, _ = dc.clearDeploymentOverlap(d) } return nil }
// handleOverlap relists all deployment in the same namespace for overlaps, and avoid syncing // the newer overlapping ones (only sync the oldest one). New/old is determined by when the // deployment's selector is last updated. func (dc *DeploymentController) handleOverlap(d *extensions.Deployment) error { deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything()) if err != nil { return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err) } overlapping := false for _, other := range deployments { foundOverlaps, err := util.OverlapsWith(d, other) if err != nil { return err } if foundOverlaps { deploymentCopy, err := util.DeploymentDeepCopy(other) if err != nil { return err } overlapping = true // Skip syncing this one if older overlapping one is found. if util.SelectorUpdatedBefore(deploymentCopy, d) { // We don't care if the overlapping annotation update failed or not (we don't make decision on it) dc.markDeploymentOverlap(d, deploymentCopy.Name) dc.clearDeploymentOverlap(deploymentCopy) return fmt.Errorf("found deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, deploymentCopy.Namespace, deploymentCopy.Name) } dc.markDeploymentOverlap(deploymentCopy, d.Name) d, _ = dc.clearDeploymentOverlap(d) } } if !overlapping { // We don't care if the overlapping annotation update failed or not (we don't make decision on it) d, _ = dc.clearDeploymentOverlap(d) } return nil }
// syncDeployment will sync the deployment with the given key. // This function is not meant to be invoked concurrently with the same key. func (dc *DeploymentController) syncDeployment(key string) error { startTime := time.Now() defer func() { glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime)) }() obj, exists, err := dc.dStore.Store.GetByKey(key) if err != nil { glog.Infof("Unable to retrieve deployment %v from store: %v", key, err) return err } if !exists { glog.Infof("Deployment has been deleted %v", key) return nil } deployment := obj.(*extensions.Deployment) everything := unversioned.LabelSelector{} if reflect.DeepEqual(deployment.Spec.Selector, &everything) { dc.eventRecorder.Eventf(deployment, api.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.") return nil } // Deep-copy otherwise we are mutating our cache. // TODO: Deep-copy only when needed. d, err := util.DeploymentDeepCopy(deployment) if err != nil { return err } if d.DeletionTimestamp != nil { return dc.syncStatusOnly(d) } if d.Spec.Paused { return dc.sync(d) } if d.Spec.RollbackTo != nil { revision := d.Spec.RollbackTo.Revision if _, err = dc.rollback(d, &revision); err != nil { return err } } if dc.isScalingEvent(d) { return dc.sync(d) } switch d.Spec.Strategy.Type { case extensions.RecreateDeploymentStrategyType: return dc.rolloutRecreate(d) case extensions.RollingUpdateDeploymentStrategyType: return dc.rolloutRolling(d) } return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type) }
// handleOverlap relists all deployment in the same namespace for overlaps, and avoid syncing // the newer overlapping ones (only sync the oldest one). New/old is determined by when the // deployment's selector is last updated. func (dc *DeploymentController) handleOverlap(d *extensions.Deployment) error { selector, err := unversioned.LabelSelectorAsSelector(d.Spec.Selector) if err != nil { return fmt.Errorf("deployment %s/%s has invalid label selector: %v", d.Namespace, d.Name, err) } deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything()) if err != nil { return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err) } overlapping := false for _, other := range deployments { if !selector.Empty() && selector.Matches(labels.Set(other.Spec.Template.Labels)) && d.UID != other.UID { deploymentCopy, err := util.DeploymentDeepCopy(other) if err != nil { return err } overlapping = true // We don't care if the overlapping annotation update failed or not (we don't make decision on it) d, _ = dc.markDeploymentOverlap(d, other.Name) deploymentCopy, _ = dc.markDeploymentOverlap(deploymentCopy, d.Name) // Skip syncing this one if older overlapping one is found // TODO: figure out a better way to determine which deployment to skip, // either with controller reference, or with validation. // Using oldest active replica set to determine which deployment to skip wouldn't make much difference, // since new replica set hasn't been created after selector update if util.SelectorUpdatedBefore(deploymentCopy, d) { return fmt.Errorf("found deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, other.Namespace, other.Name) } } } if !overlapping { // We don't care if the overlapping annotation update failed or not (we don't make decision on it) d, _ = dc.clearDeploymentOverlap(d) } return nil }
// checkForProgress checks the progress for the provided deployment. Meant to be called // by the progressWorker and work on items synced in a secondary queue. func (dc *DeploymentController) checkForProgress(key string) (bool, error) { obj, exists, err := dc.dLister.Indexer.GetByKey(key) if err != nil { glog.V(2).Infof("Cannot retrieve deployment %q found in the secondary queue: %#v", key, err) return false, err } if !exists { return false, nil } deployment := obj.(*extensions.Deployment) cond := util.GetDeploymentCondition(deployment.Status, extensions.DeploymentProgressing) // Already marked with a terminal reason - no need to add it back to the main queue. if cond != nil && (cond.Reason == util.TimedOutReason || cond.Reason == util.NewRSAvailableReason) { return false, nil } // Deep-copy otherwise we may mutate our cache. // TODO: Remove deep-copying from here. This worker does not need to sync the annotations // in the deployment. d, err := util.DeploymentDeepCopy(deployment) if err != nil { return false, err } return dc.hasFailed(d) }
// handleOverlap will avoid syncing the newer overlapping ones (only sync the oldest one). New/old is // determined by when the deployment's selector is last updated. func (dc *DeploymentController) handleOverlap(d *extensions.Deployment, deployments []*extensions.Deployment) (bool, error) { overlapping := false var errs []error for i := range deployments { otherD := deployments[i] if d.Name == otherD.Name { continue } // Error is already checked during validation foundOverlaps, _ := util.OverlapsWith(d, otherD) // If the otherD deployment overlaps with the current we need to identify which one // holds the set longer and mark the other as overlapping. Requeue the overlapping // deployments if this one has been marked deleted, we only update its status as long // as it is not actually deleted. if foundOverlaps && d.DeletionTimestamp == nil { overlapping = true // Look at the overlapping annotation in both deployments. If one of them has it and points // to the other one then we don't need to compare their timestamps. otherOverlapsWith := otherD.Annotations[util.OverlapAnnotation] currentOverlapsWith := d.Annotations[util.OverlapAnnotation] // The other deployment is already marked as overlapping with the current one. if otherOverlapsWith == d.Name { var err error if d, err = dc.clearDeploymentOverlap(d, otherD.Name); err != nil { errs = append(errs, err) } continue } otherCopy, err := util.DeploymentDeepCopy(otherD) if err != nil { return false, err } // Skip syncing this one if older overlapping one is found. if currentOverlapsWith == otherCopy.Name || util.SelectorUpdatedBefore(otherCopy, d) { if _, err = dc.markDeploymentOverlap(d, otherCopy.Name); err != nil { return false, err } if _, err = dc.clearDeploymentOverlap(otherCopy, d.Name); err != nil { return false, err } return true, fmt.Errorf("deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, otherCopy.Namespace, otherCopy.Name) } // TODO: We need to support annotations in deployments that overlap with multiple other // deployments. if _, err = dc.markDeploymentOverlap(otherCopy, d.Name); err != nil { errs = append(errs, err) } // This is going to get some deployments into update hotlooping if we remove the overlapping // annotation unconditionally. // // Scenario: // --> Deployment foo with label selector A=A is created. // --> Deployment bar with label selector A=A,B=B is created. Marked as overlapping since it // overlaps with foo. // --> Deployment baz with label selector B=B is created. Marked as overlapping, since it // overlaps with bar, bar overlapping annotation is cleaned up. Next sync loop marks bar // as overlapping and it gets in an update hotloop. if d, err = dc.clearDeploymentOverlap(d, otherCopy.Name); err != nil { errs = append(errs, err) } continue } // If the otherD deployment does not overlap with the current deployment *anymore* // we need to cleanup otherD from the overlapping annotation so it can be synced by // the deployment controller. dName, hasOverlappingAnnotation := otherD.Annotations[util.OverlapAnnotation] if hasOverlappingAnnotation && dName == d.Name { otherCopy, err := util.DeploymentDeepCopy(otherD) if err != nil { return false, err } if _, err = dc.clearDeploymentOverlap(otherCopy, d.Name); err != nil { errs = append(errs, err) } } } if !overlapping { var err error if d, err = dc.clearDeploymentOverlap(d, ""); err != nil { errs = append(errs, err) } } return false, utilerrors.NewAggregate(errs) }
// syncDeployment will sync the deployment with the given key. // This function is not meant to be invoked concurrently with the same key. func (dc *DeploymentController) syncDeployment(key string) error { startTime := time.Now() glog.V(4).Infof("Started syncing deployment %q (%v)", key, startTime) defer func() { glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime)) }() obj, exists, err := dc.dLister.Indexer.GetByKey(key) if err != nil { utilruntime.HandleError(fmt.Errorf("Unable to retrieve deployment %v from store: %v", key, err)) return err } if !exists { glog.Infof("Deployment has been deleted %v", key) return nil } deployment := obj.(*extensions.Deployment) // Deep-copy otherwise we are mutating our cache. // TODO: Deep-copy only when needed. d, err := util.DeploymentDeepCopy(deployment) if err != nil { return err } everything := metav1.LabelSelector{} if reflect.DeepEqual(d.Spec.Selector, &everything) { dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.") if d.Status.ObservedGeneration < d.Generation { d.Status.ObservedGeneration = d.Generation dc.client.Extensions().Deployments(d.Namespace).UpdateStatus(d) } return nil } deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything()) if err != nil { return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err) } // Handle overlapping deployments by deterministically avoid syncing deployments that fight over ReplicaSets. overlaps, err := dc.handleOverlap(d, deployments) if err != nil { if overlaps { // Emit an event and return a nil error for overlapping deployments so we won't resync them again. dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectorOverlap", err.Error()) return nil } // For any other failure, we should retry the deployment. return err } if d.DeletionTimestamp != nil { return dc.syncStatusOnly(d) } // Why run the cleanup policy only when there is no rollback request? // The thing with the cleanup policy currently is that it is far from smart because it takes into account // the latest replica sets while it should instead retain the latest *working* replica sets. This means that // you can have a cleanup policy of 1 but your last known working replica set may be 2 or 3 versions back // in the history. // Eventually we will want to find a way to recognize replica sets that have worked at some point in time // (and chances are higher that they will work again as opposed to others that didn't) for candidates to // automatically roll back to (#23211) and the cleanup policy should help. if d.Spec.RollbackTo == nil { _, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(d, false) if err != nil { return err } // So far the cleanup policy was executed once a deployment was paused, scaled up/down, or it // succesfully completed deploying a replica set. Decouple it from the strategies and have it // run almost unconditionally - cleanupDeployment is safe by default. dc.cleanupDeployment(oldRSs, d) } err = dc.classifyReplicaSets(d) if err != nil { return err } // Update deployment conditions with an Unknown condition when pausing/resuming // a deployment. In this way, we can be sure that we won't timeout when a user // resumes a Deployment with a set progressDeadlineSeconds. if err = dc.checkPausedConditions(d); err != nil { return err } _, err = dc.hasFailed(d) if err != nil { return err } // TODO: Automatically rollback here if we failed above. Locate the last complete // revision and populate the rollback spec with it. // See https://github.com/kubernetes/kubernetes/issues/23211. if d.Spec.Paused { return dc.sync(d) } if d.Spec.RollbackTo != nil { revision := d.Spec.RollbackTo.Revision if d, err = dc.rollback(d, &revision); err != nil { return err } } scalingEvent, err := dc.isScalingEvent(d) if err != nil { return err } if scalingEvent { return dc.sync(d) } switch d.Spec.Strategy.Type { case extensions.RecreateDeploymentStrategyType: return dc.rolloutRecreate(d) case extensions.RollingUpdateDeploymentStrategyType: return dc.rolloutRolling(d) } return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type) }
// syncDeployment will sync the deployment with the given key. // This function is not meant to be invoked concurrently with the same key. func (dc *DeploymentController) syncDeployment(key string) error { startTime := time.Now() defer func() { glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime)) }() obj, exists, err := dc.dLister.Indexer.GetByKey(key) if err != nil { glog.Errorf("Unable to retrieve deployment %v from store: %v", key, err) return err } if !exists { glog.Infof("Deployment has been deleted %v", key) return nil } deployment := obj.(*extensions.Deployment) // Deep-copy otherwise we are mutating our cache. // TODO: Deep-copy only when needed. d, err := util.DeploymentDeepCopy(deployment) if err != nil { return err } everything := unversioned.LabelSelector{} if reflect.DeepEqual(d.Spec.Selector, &everything) { dc.eventRecorder.Eventf(d, api.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.") if d.Status.ObservedGeneration < d.Generation { d.Status.ObservedGeneration = d.Generation dc.client.Extensions().Deployments(d.Namespace).UpdateStatus(d) } return nil } if d.DeletionTimestamp != nil { return dc.syncStatusOnly(d) } // Handle overlapping deployments by deterministically avoid syncing deployments that fight over ReplicaSets. if err = dc.handleOverlap(d); err != nil { dc.eventRecorder.Eventf(d, api.EventTypeWarning, "SelectorOverlap", err.Error()) return nil } if d.Spec.Paused { return dc.sync(d) } if d.Spec.RollbackTo != nil { revision := d.Spec.RollbackTo.Revision if d, err = dc.rollback(d, &revision); err != nil { return err } } scalingEvent, err := dc.isScalingEvent(d) if err != nil { return err } if scalingEvent { return dc.sync(d) } switch d.Spec.Strategy.Type { case extensions.RecreateDeploymentStrategyType: return dc.rolloutRecreate(d) case extensions.RollingUpdateDeploymentStrategyType: return dc.rolloutRolling(d) } return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type) }
// syncDeployment will sync the deployment with the given key. // This function is not meant to be invoked concurrently with the same key. func (dc *DeploymentController) syncDeployment(key string) error { startTime := time.Now() defer func() { glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime)) }() obj, exists, err := dc.dLister.Indexer.GetByKey(key) if err != nil { glog.Errorf("Unable to retrieve deployment %v from store: %v", key, err) return err } if !exists { glog.Infof("Deployment has been deleted %v", key) return nil } deployment := obj.(*extensions.Deployment) // Deep-copy otherwise we are mutating our cache. // TODO: Deep-copy only when needed. d, err := util.DeploymentDeepCopy(deployment) if err != nil { return err } everything := metav1.LabelSelector{} if reflect.DeepEqual(d.Spec.Selector, &everything) { dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.") if d.Status.ObservedGeneration < d.Generation { d.Status.ObservedGeneration = d.Generation dc.client.Extensions().Deployments(d.Namespace).UpdateStatus(d) } return nil } if d.DeletionTimestamp != nil { return dc.syncStatusOnly(d) } // Handle overlapping deployments by deterministically avoid syncing deployments that fight over ReplicaSets. if err = dc.handleOverlap(d); err != nil { dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectorOverlap", err.Error()) return nil } // Update deployment conditions with an Unknown condition when pausing/resuming // a deployment. In this way, we can be sure that we won't timeout when a user // resumes a Deployment with a set progressDeadlineSeconds. if err = dc.checkPausedConditions(d); err != nil { return err } _, err = dc.hasFailed(d) if err != nil { return err } // TODO: Automatically rollback here if we failed above. Locate the last complete // revision and populate the rollback spec with it. // See https://github.com/kubernetes/kubernetes/issues/23211. if d.Spec.Paused { return dc.sync(d) } if d.Spec.RollbackTo != nil { revision := d.Spec.RollbackTo.Revision if d, err = dc.rollback(d, &revision); err != nil { return err } } scalingEvent, err := dc.isScalingEvent(d) if err != nil { return err } if scalingEvent { return dc.sync(d) } switch d.Spec.Strategy.Type { case extensions.RecreateDeploymentStrategyType: return dc.rolloutRecreate(d) case extensions.RollingUpdateDeploymentStrategyType: return dc.rolloutRolling(d) } return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type) }