// handleOverlap relists all deployment in the same namespace for overlaps, and avoid syncing
// the newer overlapping ones (only sync the oldest one). New/old is determined by when the
// deployment's selector is last updated.
func (dc *DeploymentController) handleOverlap(d *extensions.Deployment) error {
	selector, err := unversioned.LabelSelectorAsSelector(d.Spec.Selector)
	if err != nil {
		return fmt.Errorf("deployment %s/%s has invalid label selector: %v", d.Namespace, d.Name, err)
	}
	deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything())
	if err != nil {
		return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err)
	}
	overlapping := false
	for _, other := range deployments {
		if !selector.Empty() && selector.Matches(labels.Set(other.Spec.Template.Labels)) && d.UID != other.UID {
			deploymentCopy, err := util.DeploymentDeepCopy(other)
			if err != nil {
				return err
			}
			overlapping = true
			// Skip syncing this one if older overlapping one is found.
			if util.SelectorUpdatedBefore(deploymentCopy, d) {
				// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
				dc.markDeploymentOverlap(d, deploymentCopy.Name)
				dc.clearDeploymentOverlap(deploymentCopy)
				return fmt.Errorf("found deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, deploymentCopy.Namespace, deploymentCopy.Name)
			}
			dc.markDeploymentOverlap(deploymentCopy, d.Name)
			d, _ = dc.clearDeploymentOverlap(d)
		}
	}
	if !overlapping {
		// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
		d, _ = dc.clearDeploymentOverlap(d)
	}
	return nil
}
Example #2
0
// handleOverlap relists all deployment in the same namespace for overlaps, and avoid syncing
// the newer overlapping ones (only sync the oldest one). New/old is determined by when the
// deployment's selector is last updated.
func (dc *DeploymentController) handleOverlap(d *extensions.Deployment) error {
	deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything())
	if err != nil {
		return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err)
	}
	overlapping := false
	for _, other := range deployments {
		foundOverlaps, err := util.OverlapsWith(d, other)
		if err != nil {
			return err
		}
		if foundOverlaps {
			deploymentCopy, err := util.DeploymentDeepCopy(other)
			if err != nil {
				return err
			}
			overlapping = true
			// Skip syncing this one if older overlapping one is found.
			if util.SelectorUpdatedBefore(deploymentCopy, d) {
				// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
				dc.markDeploymentOverlap(d, deploymentCopy.Name)
				dc.clearDeploymentOverlap(deploymentCopy)
				return fmt.Errorf("found deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, deploymentCopy.Namespace, deploymentCopy.Name)
			}
			dc.markDeploymentOverlap(deploymentCopy, d.Name)
			d, _ = dc.clearDeploymentOverlap(d)
		}
	}
	if !overlapping {
		// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
		d, _ = dc.clearDeploymentOverlap(d)
	}
	return nil
}
// syncDeployment will sync the deployment with the given key.
// This function is not meant to be invoked concurrently with the same key.
func (dc *DeploymentController) syncDeployment(key string) error {
	startTime := time.Now()
	defer func() {
		glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime))
	}()

	obj, exists, err := dc.dStore.Store.GetByKey(key)
	if err != nil {
		glog.Infof("Unable to retrieve deployment %v from store: %v", key, err)
		return err
	}
	if !exists {
		glog.Infof("Deployment has been deleted %v", key)
		return nil
	}

	deployment := obj.(*extensions.Deployment)
	everything := unversioned.LabelSelector{}
	if reflect.DeepEqual(deployment.Spec.Selector, &everything) {
		dc.eventRecorder.Eventf(deployment, api.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.")
		return nil
	}

	// Deep-copy otherwise we are mutating our cache.
	// TODO: Deep-copy only when needed.
	d, err := util.DeploymentDeepCopy(deployment)
	if err != nil {
		return err
	}

	if d.DeletionTimestamp != nil {
		return dc.syncStatusOnly(d)
	}

	if d.Spec.Paused {
		return dc.sync(d)
	}

	if d.Spec.RollbackTo != nil {
		revision := d.Spec.RollbackTo.Revision
		if _, err = dc.rollback(d, &revision); err != nil {
			return err
		}
	}

	if dc.isScalingEvent(d) {
		return dc.sync(d)
	}

	switch d.Spec.Strategy.Type {
	case extensions.RecreateDeploymentStrategyType:
		return dc.rolloutRecreate(d)
	case extensions.RollingUpdateDeploymentStrategyType:
		return dc.rolloutRolling(d)
	}
	return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
}
// handleOverlap relists all deployment in the same namespace for overlaps, and avoid syncing
// the newer overlapping ones (only sync the oldest one). New/old is determined by when the
// deployment's selector is last updated.
func (dc *DeploymentController) handleOverlap(d *extensions.Deployment) error {
	selector, err := unversioned.LabelSelectorAsSelector(d.Spec.Selector)
	if err != nil {
		return fmt.Errorf("deployment %s/%s has invalid label selector: %v", d.Namespace, d.Name, err)
	}
	deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything())
	if err != nil {
		return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err)
	}
	overlapping := false
	for _, other := range deployments {
		if !selector.Empty() && selector.Matches(labels.Set(other.Spec.Template.Labels)) && d.UID != other.UID {
			deploymentCopy, err := util.DeploymentDeepCopy(other)
			if err != nil {
				return err
			}
			overlapping = true
			// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
			d, _ = dc.markDeploymentOverlap(d, other.Name)
			deploymentCopy, _ = dc.markDeploymentOverlap(deploymentCopy, d.Name)
			// Skip syncing this one if older overlapping one is found
			// TODO: figure out a better way to determine which deployment to skip,
			// either with controller reference, or with validation.
			// Using oldest active replica set to determine which deployment to skip wouldn't make much difference,
			// since new replica set hasn't been created after selector update
			if util.SelectorUpdatedBefore(deploymentCopy, d) {
				return fmt.Errorf("found deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, other.Namespace, other.Name)
			}
		}
	}
	if !overlapping {
		// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
		d, _ = dc.clearDeploymentOverlap(d)
	}
	return nil
}
// checkForProgress checks the progress for the provided deployment. Meant to be called
// by the progressWorker and work on items synced in a secondary queue.
func (dc *DeploymentController) checkForProgress(key string) (bool, error) {
	obj, exists, err := dc.dLister.Indexer.GetByKey(key)
	if err != nil {
		glog.V(2).Infof("Cannot retrieve deployment %q found in the secondary queue: %#v", key, err)
		return false, err
	}
	if !exists {
		return false, nil
	}
	deployment := obj.(*extensions.Deployment)
	cond := util.GetDeploymentCondition(deployment.Status, extensions.DeploymentProgressing)
	// Already marked with a terminal reason - no need to add it back to the main queue.
	if cond != nil && (cond.Reason == util.TimedOutReason || cond.Reason == util.NewRSAvailableReason) {
		return false, nil
	}
	// Deep-copy otherwise we may mutate our cache.
	// TODO: Remove deep-copying from here. This worker does not need to sync the annotations
	// in the deployment.
	d, err := util.DeploymentDeepCopy(deployment)
	if err != nil {
		return false, err
	}
	return dc.hasFailed(d)
}
// handleOverlap will avoid syncing the newer overlapping ones (only sync the oldest one). New/old is
// determined by when the deployment's selector is last updated.
func (dc *DeploymentController) handleOverlap(d *extensions.Deployment, deployments []*extensions.Deployment) (bool, error) {
	overlapping := false
	var errs []error

	for i := range deployments {
		otherD := deployments[i]

		if d.Name == otherD.Name {
			continue
		}

		// Error is already checked during validation
		foundOverlaps, _ := util.OverlapsWith(d, otherD)

		// If the otherD deployment overlaps with the current we need to identify which one
		// holds the set longer and mark the other as overlapping. Requeue the overlapping
		// deployments if this one has been marked deleted, we only update its status as long
		// as it is not actually deleted.
		if foundOverlaps && d.DeletionTimestamp == nil {
			overlapping = true
			// Look at the overlapping annotation in both deployments. If one of them has it and points
			// to the other one then we don't need to compare their timestamps.
			otherOverlapsWith := otherD.Annotations[util.OverlapAnnotation]
			currentOverlapsWith := d.Annotations[util.OverlapAnnotation]
			// The other deployment is already marked as overlapping with the current one.
			if otherOverlapsWith == d.Name {
				var err error
				if d, err = dc.clearDeploymentOverlap(d, otherD.Name); err != nil {
					errs = append(errs, err)
				}
				continue
			}

			otherCopy, err := util.DeploymentDeepCopy(otherD)
			if err != nil {
				return false, err
			}

			// Skip syncing this one if older overlapping one is found.
			if currentOverlapsWith == otherCopy.Name || util.SelectorUpdatedBefore(otherCopy, d) {
				if _, err = dc.markDeploymentOverlap(d, otherCopy.Name); err != nil {
					return false, err
				}
				if _, err = dc.clearDeploymentOverlap(otherCopy, d.Name); err != nil {
					return false, err
				}
				return true, fmt.Errorf("deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, otherCopy.Namespace, otherCopy.Name)
			}

			// TODO: We need to support annotations in deployments that overlap with multiple other
			// deployments.
			if _, err = dc.markDeploymentOverlap(otherCopy, d.Name); err != nil {
				errs = append(errs, err)
			}
			// This is going to get some deployments into update hotlooping if we remove the overlapping
			// annotation unconditionally.
			//
			// Scenario:
			// --> Deployment foo with label selector A=A is created.
			// --> Deployment bar with label selector A=A,B=B is created. Marked as overlapping since it
			//     overlaps with foo.
			// --> Deployment baz with label selector B=B is created. Marked as overlapping, since it
			//     overlaps with bar, bar overlapping annotation is cleaned up. Next sync loop marks bar
			//     as overlapping and it gets in an update hotloop.
			if d, err = dc.clearDeploymentOverlap(d, otherCopy.Name); err != nil {
				errs = append(errs, err)
			}
			continue
		}

		// If the otherD deployment does not overlap with the current deployment *anymore*
		// we need to cleanup otherD from the overlapping annotation so it can be synced by
		// the deployment controller.
		dName, hasOverlappingAnnotation := otherD.Annotations[util.OverlapAnnotation]
		if hasOverlappingAnnotation && dName == d.Name {
			otherCopy, err := util.DeploymentDeepCopy(otherD)
			if err != nil {
				return false, err
			}
			if _, err = dc.clearDeploymentOverlap(otherCopy, d.Name); err != nil {
				errs = append(errs, err)
			}
		}
	}

	if !overlapping {
		var err error
		if d, err = dc.clearDeploymentOverlap(d, ""); err != nil {
			errs = append(errs, err)
		}
	}

	return false, utilerrors.NewAggregate(errs)
}
// syncDeployment will sync the deployment with the given key.
// This function is not meant to be invoked concurrently with the same key.
func (dc *DeploymentController) syncDeployment(key string) error {
	startTime := time.Now()
	glog.V(4).Infof("Started syncing deployment %q (%v)", key, startTime)
	defer func() {
		glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime))
	}()

	obj, exists, err := dc.dLister.Indexer.GetByKey(key)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("Unable to retrieve deployment %v from store: %v", key, err))
		return err
	}
	if !exists {
		glog.Infof("Deployment has been deleted %v", key)
		return nil
	}

	deployment := obj.(*extensions.Deployment)
	// Deep-copy otherwise we are mutating our cache.
	// TODO: Deep-copy only when needed.
	d, err := util.DeploymentDeepCopy(deployment)
	if err != nil {
		return err
	}

	everything := metav1.LabelSelector{}
	if reflect.DeepEqual(d.Spec.Selector, &everything) {
		dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.")
		if d.Status.ObservedGeneration < d.Generation {
			d.Status.ObservedGeneration = d.Generation
			dc.client.Extensions().Deployments(d.Namespace).UpdateStatus(d)
		}
		return nil
	}

	deployments, err := dc.dLister.Deployments(d.Namespace).List(labels.Everything())
	if err != nil {
		return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err)
	}

	// Handle overlapping deployments by deterministically avoid syncing deployments that fight over ReplicaSets.
	overlaps, err := dc.handleOverlap(d, deployments)
	if err != nil {
		if overlaps {
			// Emit an event and return a nil error for overlapping deployments so we won't resync them again.
			dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectorOverlap", err.Error())
			return nil
		}
		// For any other failure, we should retry the deployment.
		return err
	}

	if d.DeletionTimestamp != nil {
		return dc.syncStatusOnly(d)
	}

	// Why run the cleanup policy only when there is no rollback request?
	// The thing with the cleanup policy currently is that it is far from smart because it takes into account
	// the latest replica sets while it should instead retain the latest *working* replica sets. This means that
	// you can have a cleanup policy of 1 but your last known working replica set may be 2 or 3 versions back
	// in the history.
	// Eventually we will want to find a way to recognize replica sets that have worked at some point in time
	// (and chances are higher that they will work again as opposed to others that didn't) for candidates to
	// automatically roll back to (#23211) and the cleanup policy should help.
	if d.Spec.RollbackTo == nil {
		_, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(d, false)
		if err != nil {
			return err
		}
		// So far the cleanup policy was executed once a deployment was paused, scaled up/down, or it
		// succesfully completed deploying a replica set. Decouple it from the strategies and have it
		// run almost unconditionally - cleanupDeployment is safe by default.
		dc.cleanupDeployment(oldRSs, d)
	}

	err = dc.classifyReplicaSets(d)
	if err != nil {
		return err
	}

	// Update deployment conditions with an Unknown condition when pausing/resuming
	// a deployment. In this way, we can be sure that we won't timeout when a user
	// resumes a Deployment with a set progressDeadlineSeconds.
	if err = dc.checkPausedConditions(d); err != nil {
		return err
	}

	_, err = dc.hasFailed(d)
	if err != nil {
		return err
	}
	// TODO: Automatically rollback here if we failed above. Locate the last complete
	// revision and populate the rollback spec with it.
	// See https://github.com/kubernetes/kubernetes/issues/23211.

	if d.Spec.Paused {
		return dc.sync(d)
	}

	if d.Spec.RollbackTo != nil {
		revision := d.Spec.RollbackTo.Revision
		if d, err = dc.rollback(d, &revision); err != nil {
			return err
		}
	}

	scalingEvent, err := dc.isScalingEvent(d)
	if err != nil {
		return err
	}
	if scalingEvent {
		return dc.sync(d)
	}

	switch d.Spec.Strategy.Type {
	case extensions.RecreateDeploymentStrategyType:
		return dc.rolloutRecreate(d)
	case extensions.RollingUpdateDeploymentStrategyType:
		return dc.rolloutRolling(d)
	}
	return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
}
// syncDeployment will sync the deployment with the given key.
// This function is not meant to be invoked concurrently with the same key.
func (dc *DeploymentController) syncDeployment(key string) error {
	startTime := time.Now()
	defer func() {
		glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime))
	}()

	obj, exists, err := dc.dLister.Indexer.GetByKey(key)
	if err != nil {
		glog.Errorf("Unable to retrieve deployment %v from store: %v", key, err)
		return err
	}
	if !exists {
		glog.Infof("Deployment has been deleted %v", key)
		return nil
	}

	deployment := obj.(*extensions.Deployment)
	// Deep-copy otherwise we are mutating our cache.
	// TODO: Deep-copy only when needed.
	d, err := util.DeploymentDeepCopy(deployment)
	if err != nil {
		return err
	}

	everything := unversioned.LabelSelector{}
	if reflect.DeepEqual(d.Spec.Selector, &everything) {
		dc.eventRecorder.Eventf(d, api.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.")
		if d.Status.ObservedGeneration < d.Generation {
			d.Status.ObservedGeneration = d.Generation
			dc.client.Extensions().Deployments(d.Namespace).UpdateStatus(d)
		}
		return nil
	}

	if d.DeletionTimestamp != nil {
		return dc.syncStatusOnly(d)
	}

	// Handle overlapping deployments by deterministically avoid syncing deployments that fight over ReplicaSets.
	if err = dc.handleOverlap(d); err != nil {
		dc.eventRecorder.Eventf(d, api.EventTypeWarning, "SelectorOverlap", err.Error())
		return nil
	}

	if d.Spec.Paused {
		return dc.sync(d)
	}

	if d.Spec.RollbackTo != nil {
		revision := d.Spec.RollbackTo.Revision
		if d, err = dc.rollback(d, &revision); err != nil {
			return err
		}
	}

	scalingEvent, err := dc.isScalingEvent(d)
	if err != nil {
		return err
	}
	if scalingEvent {
		return dc.sync(d)
	}

	switch d.Spec.Strategy.Type {
	case extensions.RecreateDeploymentStrategyType:
		return dc.rolloutRecreate(d)
	case extensions.RollingUpdateDeploymentStrategyType:
		return dc.rolloutRolling(d)
	}
	return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
}
Example #9
0
// syncDeployment will sync the deployment with the given key.
// This function is not meant to be invoked concurrently with the same key.
func (dc *DeploymentController) syncDeployment(key string) error {
	startTime := time.Now()
	defer func() {
		glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime))
	}()

	obj, exists, err := dc.dLister.Indexer.GetByKey(key)
	if err != nil {
		glog.Errorf("Unable to retrieve deployment %v from store: %v", key, err)
		return err
	}
	if !exists {
		glog.Infof("Deployment has been deleted %v", key)
		return nil
	}

	deployment := obj.(*extensions.Deployment)
	// Deep-copy otherwise we are mutating our cache.
	// TODO: Deep-copy only when needed.
	d, err := util.DeploymentDeepCopy(deployment)
	if err != nil {
		return err
	}

	everything := metav1.LabelSelector{}
	if reflect.DeepEqual(d.Spec.Selector, &everything) {
		dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.")
		if d.Status.ObservedGeneration < d.Generation {
			d.Status.ObservedGeneration = d.Generation
			dc.client.Extensions().Deployments(d.Namespace).UpdateStatus(d)
		}
		return nil
	}

	if d.DeletionTimestamp != nil {
		return dc.syncStatusOnly(d)
	}

	// Handle overlapping deployments by deterministically avoid syncing deployments that fight over ReplicaSets.
	if err = dc.handleOverlap(d); err != nil {
		dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectorOverlap", err.Error())
		return nil
	}

	// Update deployment conditions with an Unknown condition when pausing/resuming
	// a deployment. In this way, we can be sure that we won't timeout when a user
	// resumes a Deployment with a set progressDeadlineSeconds.
	if err = dc.checkPausedConditions(d); err != nil {
		return err
	}

	_, err = dc.hasFailed(d)
	if err != nil {
		return err
	}
	// TODO: Automatically rollback here if we failed above. Locate the last complete
	// revision and populate the rollback spec with it.
	// See https://github.com/kubernetes/kubernetes/issues/23211.

	if d.Spec.Paused {
		return dc.sync(d)
	}

	if d.Spec.RollbackTo != nil {
		revision := d.Spec.RollbackTo.Revision
		if d, err = dc.rollback(d, &revision); err != nil {
			return err
		}
	}

	scalingEvent, err := dc.isScalingEvent(d)
	if err != nil {
		return err
	}
	if scalingEvent {
		return dc.sync(d)
	}

	switch d.Spec.Strategy.Type {
	case extensions.RecreateDeploymentStrategyType:
		return dc.rolloutRecreate(d)
	case extensions.RollingUpdateDeploymentStrategyType:
		return dc.rolloutRolling(d)
	}
	return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
}