func (c *DeploymentConfigController) updateStatus(config *deployapi.DeploymentConfig, deployments []kapi.ReplicationController) error {
newStatus, err := c.calculateStatus(*config, deployments)
if err != nil {
glog.V(2).Infof("Cannot calculate the status for %q: %v", deployutil.LabelForDeploymentConfig(config), err)
return err
}
latestExists, latestRC := deployutil.LatestDeploymentInfo(config, deployments)
if !latestExists {
latestRC = nil
}
updateConditions(config, &newStatus, latestRC)
// NOTE: We should update the status of the deployment config only if we need to, otherwise
// we hotloop between updates.
if reflect.DeepEqual(newStatus, config.Status) {
return nil
}
copied, err := deployutil.DeploymentConfigDeepCopy(config)
if err != nil {
return err
}
copied.Status = newStatus
if _, err := c.dn.DeploymentConfigs(copied.Namespace).UpdateStatus(copied); err != nil {
glog.V(2).Infof("Cannot update the status for %q: %v", deployutil.LabelForDeploymentConfig(copied), err)
return err
}
glog.V(4).Infof("Updated the status for %q (observed generation: %d)", deployutil.LabelForDeploymentConfig(copied), copied.Status.ObservedGeneration)
return nil
}
// Handle processes image change triggers associated with imageRepo.
func (c *ImageChangeController) Handle(imageRepo *imageapi.ImageStream) error {
configs, err := c.deploymentConfigClient.listDeploymentConfigs()
if err != nil {
return fmt.Errorf("couldn't get list of DeploymentConfig while handling ImageStream %s: %v", labelForRepo(imageRepo), err)
}
// Find any configs which should be updated based on the new image state
configsToUpdate := map[string]*deployapi.DeploymentConfig{}
for _, config := range configs {
glog.V(4).Infof("Detecting changed images for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config))
for _, trigger := range config.Triggers {
params := trigger.ImageChangeParams
// Only automatic image change triggers should fire
if trigger.Type != deployapi.DeploymentTriggerOnImageChange || !params.Automatic {
continue
}
// Check if the image repo matches the trigger
if !triggerMatchesImage(config, params, imageRepo) {
continue
}
// Find the latest tag event for the trigger tag
latestEvent := imageapi.LatestTaggedImage(imageRepo, params.Tag)
if latestEvent == nil {
glog.V(2).Infof("Couldn't find latest tag event for tag %s in ImageStream %s", params.Tag, labelForRepo(imageRepo))
continue
}
// Ensure a change occurred
if len(latestEvent.DockerImageReference) > 0 &&
latestEvent.DockerImageReference != params.LastTriggeredImage {
// Mark the config for regeneration
configsToUpdate[config.Name] = config
}
}
}
// Attempt to regenerate all configs which may contain image updates
anyFailed := false
for _, config := range configsToUpdate {
err := c.regenerate(config)
if err != nil {
anyFailed = true
glog.V(2).Infof("Couldn't regenerate DeploymentConfig %s: %s", deployutil.LabelForDeploymentConfig(config), err)
continue
}
glog.V(4).Infof("Regenerated DeploymentConfig %s in response to image change trigger", deployutil.LabelForDeploymentConfig(config))
}
if anyFailed {
return fatalError(fmt.Sprintf("couldn't update some DeploymentConfig for trigger on ImageStream %s", labelForRepo(imageRepo)))
}
glog.V(4).Infof("Updated all DeploymentConfigs for trigger on ImageStream %s", labelForRepo(imageRepo))
return nil
}
// Handle processes change triggers for config.
func (c *DeploymentConfigChangeController) Handle(config *deployapi.DeploymentConfig) error {
if !deployutil.HasChangeTrigger(config) {
glog.V(5).Infof("Ignoring DeploymentConfig %s; no change triggers detected", deployutil.LabelForDeploymentConfig(config))
return nil
}
if config.Status.LatestVersion == 0 {
_, _, abort, err := c.generateDeployment(config)
if err != nil {
if kerrors.IsConflict(err) {
return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err))
}
glog.V(4).Infof("Couldn't create initial deployment for deploymentConfig %q: %v", deployutil.LabelForDeploymentConfig(config), err)
return nil
}
if !abort {
glog.V(4).Infof("Created initial deployment for deploymentConfig %q", deployutil.LabelForDeploymentConfig(config))
}
return nil
}
latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config)
deployment, err := c.changeStrategy.getDeployment(config.Namespace, latestDeploymentName)
if err != nil {
// If there's no deployment for the latest config, we have no basis of
// comparison. It's the responsibility of the deployment config controller
// to make the deployment for the config, so return early.
if kerrors.IsNotFound(err) {
glog.V(5).Infof("Ignoring change for DeploymentConfig %s; no existing Deployment found", deployutil.LabelForDeploymentConfig(config))
return nil
}
return fmt.Errorf("couldn't retrieve Deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
deployedConfig, err := c.decodeConfig(deployment)
if err != nil {
return fatalError(fmt.Sprintf("error decoding DeploymentConfig from Deployment %s for DeploymentConfig %s: %v", deployutil.LabelForDeployment(deployment), deployutil.LabelForDeploymentConfig(config), err))
}
// Detect template diffs, and return early if there aren't any changes.
if kapi.Semantic.DeepEqual(config.Spec.Template, deployedConfig.Spec.Template) {
glog.V(5).Infof("Ignoring DeploymentConfig change for %s (latestVersion=%d); same as Deployment %s", deployutil.LabelForDeploymentConfig(config), config.Status.LatestVersion, deployutil.LabelForDeployment(deployment))
return nil
}
// There was a template diff, so generate a new config version.
fromVersion, toVersion, abort, err := c.generateDeployment(config)
if err != nil {
if kerrors.IsConflict(err) {
return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err))
}
return fmt.Errorf("couldn't generate deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
if !abort {
glog.V(4).Infof("Updated DeploymentConfig %s from version %d to %d for existing deployment %s", deployutil.LabelForDeploymentConfig(config), fromVersion, toVersion, deployutil.LabelForDeployment(deployment))
}
return nil
}
func (c *DeploymentConfigController) updateStatus(config *deployapi.DeploymentConfig) error {
if config.Generation > config.Status.ObservedGeneration {
config.Status.ObservedGeneration = config.Generation
}
if _, err := c.osClient.DeploymentConfigs(config.Namespace).UpdateStatus(config); err != nil {
glog.V(2).Infof("Cannot update the status for %q: %v", deployutil.LabelForDeploymentConfig(config), err)
return transientError(fmt.Sprintf("cannot update the status for %q - requeuing", deployutil.LabelForDeploymentConfig(config)))
}
glog.V(4).Infof("Updated the status for %q (observed generation: %d)", deployutil.LabelForDeploymentConfig(config), config.Status.ObservedGeneration)
return nil
}
func (c *DeploymentConfigController) updateStatus(config *deployapi.DeploymentConfig) error {
// NOTE: We should update the status of the deployment config only if we need to, otherwise
// we hotloop between updates.
if !needsUpdate(config) {
return nil
}
config.Status.ObservedGeneration = config.Generation
if _, err := c.dn.DeploymentConfigs(config.Namespace).UpdateStatus(config); err != nil {
glog.V(2).Infof("Cannot update the status for %q: %v", deployutil.LabelForDeploymentConfig(config), err)
return err
}
glog.V(4).Infof("Updated the status for %q (observed generation: %d)", deployutil.LabelForDeploymentConfig(config), config.Status.ObservedGeneration)
return nil
}
// Handle processes change triggers for config.
func (c *DeploymentConfigChangeController) Handle(config *deployapi.DeploymentConfig) error {
if !deployutil.HasChangeTrigger(config) {
glog.V(5).Infof("Ignoring deployment config %q; no change triggers detected", deployutil.LabelForDeploymentConfig(config))
return nil
}
// Try to decode this deployment config from the encoded annotation found in
// its latest deployment.
decoded, err := c.decodeFromLatest(config)
if err != nil {
return err
}
// If this is the initial deployment, then wait for any images that need to be resolved, otherwise
// automatically start a new deployment.
if config.Status.LatestVersion == 0 {
canTrigger, causes := canTrigger(config, decoded)
if !canTrigger {
// If we cannot trigger then we need to wait for the image change controller.
glog.V(5).Infof("Ignoring deployment config %q; template image needs to be resolved by the image change controller", deployutil.LabelForDeploymentConfig(config))
return nil
}
return c.updateStatus(config, causes)
}
// If this is not the initial deployment, check if there is any template difference between
// this and the decoded deploymentconfig.
if kapi.Semantic.DeepEqual(config.Spec.Template, decoded.Spec.Template) {
return nil
}
_, causes := canTrigger(config, decoded)
return c.updateStatus(config, causes)
}
// regenerate calls the generator to get a new config. If the newly generated
// config's version is newer, update the old config to be the new config.
// Otherwise do nothing.
func (c *ImageChangeController) regenerate(config *deployapi.DeploymentConfig) error {
// Get a regenerated config which includes the new image repo references
newConfig, err := c.deploymentConfigClient.generateDeploymentConfig(config.Namespace, config.Name)
if err != nil {
return fmt.Errorf("error generating new version of DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
// No update occurred
if config.LatestVersion == newConfig.LatestVersion {
glog.V(4).Infof("No version difference for generated DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config))
return nil
}
// Persist the new config
_, err = c.deploymentConfigClient.updateDeploymentConfig(newConfig.Namespace, newConfig)
if err != nil {
return err
}
glog.Infof("Regenerated DeploymentConfig %s for image updates", deployutil.LabelForDeploymentConfig(config))
return nil
}
func (g *DeploymentConfigGenerator) findImageStream(config *deployapi.DeploymentConfig, params *deployapi.DeploymentTriggerImageChangeParams) (*imageapi.ImageStream, error) {
if len(params.From.Name) > 0 {
namespace := params.From.Namespace
if len(namespace) == 0 {
namespace = config.Namespace
}
name, _, ok := imageapi.SplitImageStreamTag(params.From.Name)
if !ok {
return nil, fmt.Errorf("invalid ImageStreamTag: %s", params.From.Name)
}
return g.Client.GetImageStream(kapi.WithNamespace(kapi.NewContext(), namespace), name)
}
return nil, fmt.Errorf("couldn't find image stream for config %s trigger params", deployutil.LabelForDeploymentConfig(config))
}
func (g *DeploymentConfigGenerator) findImageStream(config *deployapi.DeploymentConfig, params *deployapi.DeploymentTriggerImageChangeParams) (*imageapi.ImageStream, error) {
// Try to find the repo by ObjectReference
if len(params.From.Name) > 0 {
namespace := params.From.Namespace
if len(namespace) == 0 {
namespace = config.Namespace
}
return g.Client.GetImageStream(kapi.WithNamespace(kapi.NewContext(), namespace), params.From.Name)
}
// Fall back to a list based lookup on RepositoryName
repos, err := g.Client.ListImageStreams(kapi.WithNamespace(kapi.NewContext(), config.Namespace))
if err != nil {
return nil, err
}
for _, repo := range repos.Items {
if len(repo.Status.DockerImageRepository) > 0 &&
params.RepositoryName == repo.Status.DockerImageRepository {
return &repo, nil
}
}
return nil, fmt.Errorf("couldn't find image stream for config %s trigger params", deployutil.LabelForDeploymentConfig(config))
}
// Handle implements the loop that processes deployment configs. Since this controller started
// using caches, the provided config MUST be deep-copied beforehand (see work() in factory.go).
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error {
// There's nothing to reconcile until the version is nonzero.
if config.Status.LatestVersion == 0 {
return c.updateStatus(config, []kapi.ReplicationController{})
}
// Find all deployments owned by the deployment config.
selector := deployutil.ConfigSelector(config.Name)
existingDeployments, err := c.rcStore.ReplicationControllers(config.Namespace).List(selector)
if err != nil {
return err
}
// In case the deployment config has been marked for deletion, merely update its status with
// the latest available information. Some deletions make take some time to complete so there
// is value in doing this.
if config.DeletionTimestamp != nil {
return c.updateStatus(config, existingDeployments)
}
latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
// If the latest deployment doesn't exist yet, cancel any running
// deployments to allow them to be superceded by the new config version.
awaitingCancellations := false
if !latestIsDeployed {
for i := range existingDeployments {
deployment := existingDeployments[i]
// Skip deployments with an outcome.
if deployutil.IsTerminatedDeployment(&deployment) {
continue
}
// Cancel running deployments.
awaitingCancellations = true
if !deployutil.IsDeploymentCancelled(&deployment) {
// Retry faster on conflicts
var updatedDeployment *kapi.ReplicationController
if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error {
rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name)
if kapierrors.IsNotFound(err) {
return nil
}
if err != nil {
return err
}
copied, err := deployutil.DeploymentDeepCopy(rc)
if err != nil {
return err
}
copied.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
copied.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists
updatedDeployment, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied)
return err
}); err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err)
} else {
if updatedDeployment != nil {
// replace the current deployment with the updated copy so that a future update has a chance at working
existingDeployments[i] = *updatedDeployment
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion)
}
}
}
}
}
// Wait for deployment cancellations before reconciling or creating a new
// deployment to avoid competing with existing deployment processes.
if awaitingCancellations {
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion)
return fmt.Errorf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))
}
// If the latest deployment already exists, reconcile existing deployments
// and return early.
if latestIsDeployed {
// If the latest deployment is still running, try again later. We don't
// want to compete with the deployer.
if !deployutil.IsTerminatedDeployment(latestDeployment) {
return c.updateStatus(config, existingDeployments)
}
return c.reconcileDeployments(existingDeployments, config)
}
// If the config is paused we shouldn't create new deployments for it.
if config.Spec.Paused {
// in order for revision history limit cleanup to work for paused
// deployments, we need to trigger it here
if err := c.cleanupOldDeployments(existingDeployments, config); err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err)
}
return c.updateStatus(config, existingDeployments)
}
// No deployments are running and the latest deployment doesn't exist, so
// create the new deployment.
deployment, err := deployutil.MakeDeployment(config, c.codec)
if err != nil {
return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err))
}
created, err := c.rn.ReplicationControllers(config.Namespace).Create(deployment)
if err != nil {
// If the deployment was already created, just move on. The cache could be
// stale, or another process could have already handled this update.
if kapierrors.IsAlreadyExists(err) {
return c.updateStatus(config, existingDeployments)
}
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err)
return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion)
// As we've just created a new deployment, we need to make sure to clean
// up old deployments if we have reached our deployment history quota
existingDeployments = append(existingDeployments, *created)
if err := c.cleanupOldDeployments(existingDeployments, config); err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err)
}
return c.updateStatus(config, existingDeployments)
}
// Handle processes change triggers for config.
func (c *DeploymentConfigChangeController) Handle(config *deployapi.DeploymentConfig) error {
hasChangeTrigger := false
for _, trigger := range config.Triggers {
if trigger.Type == deployapi.DeploymentTriggerOnConfigChange {
hasChangeTrigger = true
break
}
}
if !hasChangeTrigger {
glog.V(4).Infof("Ignoring DeploymentConfig %s; no change triggers detected", deployutil.LabelForDeploymentConfig(config))
return nil
}
if config.LatestVersion == 0 {
_, _, err := c.generateDeployment(config)
if err != nil {
if kerrors.IsConflict(err) {
return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err))
}
// TODO: This needs handled by setting some state within the API so
// users know why the trigger isn't doing anything.
// https://github.com/openshift/origin/issues/3526
return nil
}
glog.V(4).Infof("Created initial Deployment for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config))
return nil
}
latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config)
deployment, err := c.changeStrategy.getDeployment(config.Namespace, latestDeploymentName)
if err != nil {
// If there's no deployment for the latest config, we have no basis of
// comparison. It's the responsibility of the deployment config controller
// to make the deployment for the config, so return early.
if kerrors.IsNotFound(err) {
glog.V(2).Infof("Ignoring change for DeploymentConfig %s; no existing Deployment found", deployutil.LabelForDeploymentConfig(config))
return nil
}
return fmt.Errorf("couldn't retrieve Deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
deployedConfig, err := c.decodeConfig(deployment)
if err != nil {
return fatalError(fmt.Sprintf("error decoding DeploymentConfig from Deployment %s for DeploymentConfig %s: %v", deployutil.LabelForDeployment(deployment), deployutil.LabelForDeploymentConfig(config), err))
}
newSpec, oldSpec := config.Template.ControllerTemplate.Template.Spec, deployedConfig.Template.ControllerTemplate.Template.Spec
if kapi.Semantic.DeepEqual(oldSpec, newSpec) {
glog.V(2).Infof("Ignoring DeploymentConfig change for %s (latestVersion=%d); same as Deployment %s", deployutil.LabelForDeploymentConfig(config), config.LatestVersion, deployutil.LabelForDeployment(deployment))
return nil
}
fromVersion, toVersion, err := c.generateDeployment(config)
if err != nil {
if kerrors.IsConflict(err) {
return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err))
}
return fmt.Errorf("couldn't generate deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
glog.V(4).Infof("Updated DeploymentConfig %s from version %d to %d for existing deployment %s", deployutil.LabelForDeploymentConfig(config), fromVersion, toVersion, deployutil.LabelForDeployment(deployment))
return nil
}
// reconcileDeployments reconciles existing deployment replica counts which
// could have diverged outside the deployment process (e.g. due to auto or
// manual scaling, or partial deployments). The active deployment is the last
// successful deployment, not necessarily the latest in terms of the config
// version. The active deployment replica count should follow the config, and
// all other deployments should be scaled to zero.
//
// Previously, scaling behavior was that the config replica count was used
// only for initial deployments and the active deployment had to be scaled up
// directly. To continue supporting that old behavior we must detect when the
// deployment has been directly manipulated, and if so, preserve the directly
// updated value and sync the config with the deployment.
func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []kapi.ReplicationController, config *deployapi.DeploymentConfig) error {
latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
if !latestIsDeployed {
// We shouldn't be reconciling if the latest deployment hasn't been
// created; this is enforced on the calling side, but double checking
// can't hurt.
return c.updateStatus(config, existingDeployments)
}
activeDeployment := deployutil.ActiveDeployment(existingDeployments)
// Compute the replica count for the active deployment (even if the active
// deployment doesn't exist). The active replica count is the value that
// should be assigned to the config, to allow the replica propagation to
// flow downward from the config.
//
// By default we'll assume the config replicas should be used to update the
// active deployment except in special cases (like first sync or externally
// updated deployments.)
activeReplicas := config.Spec.Replicas
source := "the deploymentConfig itself (no change)"
activeDeploymentExists := activeDeployment != nil
activeDeploymentIsLatest := activeDeploymentExists && activeDeployment.Name == latestDeployment.Name
latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment)
switch {
case activeDeploymentExists && activeDeploymentIsLatest:
// The active/latest deployment follows the config unless this is its first
// sync or if an external change to the deployment replicas is detected.
lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment)
if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas {
activeReplicas = activeDeployment.Spec.Replicas
source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment))
}
case activeDeploymentExists && !activeDeploymentIsLatest:
// The active/non-latest deployment follows the config if it was
// previously synced; if this is the first sync, infer what the config
// value should be based on either the latest desired or whatever the
// deployment is currently scaled to.
_, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment)
if hasLastActiveReplicas {
break
}
if latestHasDesiredReplicas {
activeReplicas = latestDesiredReplicas
source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment))
} else if activeDeployment.Spec.Replicas > 0 {
activeReplicas = activeDeployment.Spec.Replicas
source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment))
}
case !activeDeploymentExists && latestHasDesiredReplicas:
// If there's no active deployment, use the latest desired, if available.
activeReplicas = latestDesiredReplicas
source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment))
}
// Bring the config in sync with the deployment. Once we know the config
// accurately represents the desired replica count of the active deployment,
// we can safely reconcile deployments.
//
// If the deployment config is test, never update the deployment config based
// on deployments, since test behavior overrides user scaling.
switch {
case config.Spec.Replicas == activeReplicas:
case config.Spec.Test:
glog.V(4).Infof("Detected changed replicas for test deploymentConfig %q, ignoring that change", deployutil.LabelForDeploymentConfig(config))
default:
copied, err := deployutil.DeploymentConfigDeepCopy(config)
if err != nil {
return err
}
oldReplicas := copied.Spec.Replicas
copied.Spec.Replicas = activeReplicas
config, err = c.dn.DeploymentConfigs(copied.Namespace).Update(copied)
if err != nil {
return err
}
glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source)
}
// Reconcile deployments. The active deployment follows the config, and all
// other deployments should be scaled to zero.
var updatedDeployments []kapi.ReplicationController
for i := range existingDeployments {
deployment := existingDeployments[i]
toAppend := deployment
isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name
oldReplicaCount := deployment.Spec.Replicas
newReplicaCount := int32(0)
if isActiveDeployment {
newReplicaCount = activeReplicas
}
if config.Spec.Test {
glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment))
newReplicaCount = 0
}
lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment)
// Only update if necessary.
var copied *kapi.ReplicationController
if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount {
if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error {
// refresh the replication controller version
rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name)
if err != nil {
return err
}
copied, err = deployutil.DeploymentDeepCopy(rc)
if err != nil {
glog.V(2).Infof("Deep copy of deployment %q failed: %v", rc.Name, err)
return err
}
copied.Spec.Replicas = newReplicaCount
copied.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(int(newReplicaCount))
_, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied)
return err
}); err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentScaleFailed",
"Failed to scale deployment %q from %d to %d: %v", deployment.Name, oldReplicaCount, newReplicaCount, err)
return err
}
// Only report scaling events if we changed the replica count.
if oldReplicaCount != newReplicaCount {
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentScaled",
"Scaled deployment %q from %d to %d", copied.Name, oldReplicaCount, newReplicaCount)
} else {
glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(copied), newReplicaCount)
}
toAppend = *copied
}
updatedDeployments = append(updatedDeployments, toAppend)
}
// As the deployment configuration has changed, we need to make sure to clean
// up old deployments if we have now reached our deployment history quota
if err := c.cleanupOldDeployments(existingDeployments, config); err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err)
}
return c.updateStatus(config, updatedDeployments)
}
func (c *DeploymentTriggerController) updateDeploymentConfig(old, cur interface{}) {
newDc := cur.(*deployapi.DeploymentConfig)
oldDc := old.(*deployapi.DeploymentConfig)
// A periodic relist will send update events for all known deployment configs.
if newDc.ResourceVersion == oldDc.ResourceVersion {
return
}
// No need to enqueue deployment configs that have no triggers or are paused.
if len(newDc.Spec.Triggers) == 0 || newDc.Spec.Paused {
return
}
// We don't want to compete with the main deployment config controller. Let's process this
// config once it's synced. Note that this does not eliminate conflicts between the two
// controllers because the main controller is constantly updating deployment configs as
// owning replication controllers and pods are updated.
if !deployutil.HasSynced(newDc, newDc.Generation) {
return
}
// Enqueue the deployment config if it hasn't been deployed yet.
if newDc.Status.LatestVersion == 0 {
c.enqueueDeploymentConfig(newDc)
return
}
// Compare deployment config templates before enqueueing. This reduces the amount of times
// we will try to instantiate a deployment config at the expense of duplicating some of the
// work that the instantiate endpoint is already doing but I think this is fine.
shouldInstantiate := true
latestRc, err := c.rcLister.ReplicationControllers(newDc.Namespace).Get(deployutil.LatestDeploymentNameForConfig(newDc))
if err != nil {
// If we get an error here it may be due to the rc cache lagging behind. In such a case
// just defer to the api server (instantiate REST) where we will retry this.
glog.V(2).Infof("Cannot get latest rc for dc %s:%d (%v) - will defer to instantiate", deployutil.LabelForDeploymentConfig(newDc), newDc.Status.LatestVersion, err)
} else {
initial, err := deployutil.DecodeDeploymentConfig(latestRc, c.codec)
if err != nil {
glog.V(2).Infof("Cannot decode dc from replication controller %s: %v", deployutil.LabelForDeployment(latestRc), err)
return
}
shouldInstantiate = !reflect.DeepEqual(newDc.Spec.Template, initial.Spec.Template)
}
if !shouldInstantiate {
return
}
c.enqueueDeploymentConfig(newDc)
}
// addDeploymentConfig is used for making sure that new deployment configs with triggers pointing
// to existing images will be deployed as soon as they are created.
func (c *ImageChangeController) addDeploymentConfig(obj interface{}) {
dc := obj.(*deployapi.DeploymentConfig)
for _, stream := range c.streamLister.GetStreamsForConfig(dc) {
glog.V(4).Infof("Reconciling stream %q for config %q\n", imageapi.LabelForStream(stream), deployutil.LabelForDeploymentConfig(dc))
c.enqueueImageStream(stream)
}
}
// decodeFromLatest will try to return the decoded version of the current deploymentconfig found
// in the annotations of its latest deployment. If there is no previous deploymentconfig (ie.
// latestVersion == 0), the returned deploymentconfig will be the same.
func (c *DeploymentTriggerController) decodeFromLatest(config *deployapi.DeploymentConfig) (*deployapi.DeploymentConfig, error) {
if config.Status.LatestVersion == 0 {
return config, nil
}
latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config)
deployment, err := c.rn.ReplicationControllers(config.Namespace).Get(latestDeploymentName)
if err != nil {
// If there's no deployment for the latest config, we have no basis of
// comparison. It's the responsibility of the deployment config controller
// to make the deployment for the config, so return early.
return nil, fmt.Errorf("couldn't retrieve deployment for deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err)
}
latest, err := deployutil.DecodeDeploymentConfig(deployment, c.codec)
if err != nil {
return nil, fatalError(err.Error())
}
return latest, nil
}
// Create generates a new DeploymentConfig representing a rollback.
func (r *REST) Create(ctx kapi.Context, obj runtime.Object) (runtime.Object, error) {
namespace, ok := kapi.NamespaceFrom(ctx)
if !ok {
return nil, kerrors.NewBadRequest("namespace parameter required.")
}
rollback, ok := obj.(*deployapi.DeploymentConfigRollback)
if !ok {
return nil, kerrors.NewBadRequest(fmt.Sprintf("not a rollback spec: %#v", obj))
}
if errs := validation.ValidateDeploymentConfigRollback(rollback); len(errs) > 0 {
return nil, kerrors.NewInvalid(deployapi.Kind("DeploymentConfigRollback"), rollback.Name, errs)
}
from, err := r.dn.DeploymentConfigs(namespace).Get(rollback.Name)
if err != nil {
return nil, newInvalidError(rollback, fmt.Sprintf("cannot get deployment config %q: %v", rollback.Name, err))
}
switch from.Status.LatestVersion {
case 0:
return nil, newInvalidError(rollback, "cannot rollback an undeployed config")
case 1:
return nil, newInvalidError(rollback, fmt.Sprintf("no previous deployment exists for %q", deployutil.LabelForDeploymentConfig(from)))
}
revision := from.Status.LatestVersion - 1
if rollback.Spec.Revision > 0 {
revision = rollback.Spec.Revision
}
// Find the target deployment and decode its config.
name := deployutil.DeploymentNameForConfigVersion(from.Name, revision)
targetDeployment, err := r.rn.ReplicationControllers(namespace).Get(name)
if err != nil {
return nil, newInvalidError(rollback, err.Error())
}
to, err := deployutil.DecodeDeploymentConfig(targetDeployment, r.codec)
if err != nil {
return nil, newInvalidError(rollback, fmt.Sprintf("couldn't decode deployment config from deployment: %v", err))
}
if from.Annotations == nil && len(rollback.UpdatedAnnotations) > 0 {
from.Annotations = make(map[string]string)
}
for key, value := range rollback.UpdatedAnnotations {
from.Annotations[key] = value
}
return r.generator.GenerateRollback(from, to, &rollback.Spec)
}
// Handle processes image change triggers associated with imagestream.
func (c *ImageChangeController) Handle(stream *imageapi.ImageStream) error {
configs, err := c.listDeploymentConfigs()
if err != nil {
return fmt.Errorf("couldn't get list of deployment configs while handling image stream %q: %v", imageapi.LabelForStream(stream), err)
}
// Find any configs which should be updated based on the new image state
configsToUpdate := []*deployapi.DeploymentConfig{}
for _, config := range configs {
glog.V(4).Infof("Detecting image changes for deployment config %q", deployutil.LabelForDeploymentConfig(config))
hasImageChange := false
for _, trigger := range config.Spec.Triggers {
params := trigger.ImageChangeParams
// Only automatic image change triggers should fire
if trigger.Type != deployapi.DeploymentTriggerOnImageChange {
continue
}
// All initial deployments (latestVersion == 0) should have their images resolved in order
// to be able to work and not try to pull non-existent images from DockerHub.
// Deployments with automatic set to false that have been deployed at least once (latestVersion > 0)
// shouldn't have their images updated.
if !params.Automatic && config.Status.LatestVersion != 0 {
continue
}
// Check if the image stream matches the trigger
if !triggerMatchesImage(config, params, stream) {
continue
}
_, tag, ok := imageapi.SplitImageStreamTag(params.From.Name)
if !ok {
glog.Warningf("Invalid image stream tag %q in %q", params.From.Name, deployutil.LabelForDeploymentConfig(config))
continue
}
// Find the latest tag event for the trigger tag
latestEvent := imageapi.LatestTaggedImage(stream, tag)
if latestEvent == nil {
glog.V(5).Infof("Couldn't find latest tag event for tag %q in image stream %q", tag, imageapi.LabelForStream(stream))
continue
}
// Ensure a change occurred
if len(latestEvent.DockerImageReference) == 0 || latestEvent.DockerImageReference == params.LastTriggeredImage {
glog.V(4).Infof("No image changes for deployment config %q were detected", deployutil.LabelForDeploymentConfig(config))
continue
}
names := sets.NewString(params.ContainerNames...)
for i := range config.Spec.Template.Spec.Containers {
container := &config.Spec.Template.Spec.Containers[i]
if !names.Has(container.Name) {
continue
}
// Update the image
container.Image = latestEvent.DockerImageReference
// Log the last triggered image ID
params.LastTriggeredImage = latestEvent.DockerImageReference
hasImageChange = true
}
}
if hasImageChange {
configsToUpdate = append(configsToUpdate, config)
}
}
// Attempt to regenerate all configs which may contain image updates
anyFailed := false
for _, config := range configsToUpdate {
if _, err := c.client.DeploymentConfigs(config.Namespace).Update(config); err != nil {
anyFailed = true
glog.V(2).Infof("Couldn't update deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err)
}
}
if anyFailed {
return fatalError(fmt.Sprintf("couldn't update some deployment configs for trigger on image stream %q", imageapi.LabelForStream(stream)))
}
glog.V(5).Infof("Updated all deployment configs for trigger on image stream %q", imageapi.LabelForStream(stream))
return nil
}
// updateDeploymentConfig is used for making sure that deployment configs with new triggers
// pointing to existing images will be deployed as soon as they are updated.
func (c *ImageChangeController) updateDeploymentConfig(old, cur interface{}) {
// A periodic relist will send update events for all known configs.
newDc := cur.(*deployapi.DeploymentConfig)
oldDc := old.(*deployapi.DeploymentConfig)
if newDc.ResourceVersion == oldDc.ResourceVersion {
return
}
for _, stream := range c.streamLister.GetStreamsForConfig(newDc) {
glog.V(4).Infof("Reconciling stream %q for config %q\n", imageapi.LabelForStream(stream), deployutil.LabelForDeploymentConfig(newDc))
c.enqueueImageStream(stream)
}
}
// Handle processes config and creates a new deployment if necessary.
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error {
// Inspect a deployment configuration every time the controller reconciles it
details, existingDeployments, latestDeploymentExists, err := c.findDetails(config)
if err != nil {
return err
}
config, err = c.updateDetails(config, details)
if err != nil {
return transientError(err.Error())
}
// Only deploy when the version has advanced past 0.
if config.LatestVersion == 0 {
glog.V(5).Infof("Waiting for first version of %s", deployutil.LabelForDeploymentConfig(config))
return nil
}
var inflightDeployment *kapi.ReplicationController
for _, deployment := range existingDeployments.Items {
deploymentStatus := deployutil.DeploymentStatusFor(&deployment)
switch deploymentStatus {
case deployapi.DeploymentStatusFailed,
deployapi.DeploymentStatusComplete:
// Previous deployment in terminal state - can ignore
// Ignoring specific deployment states so that any newly introduced
// deployment state will not be ignored
default:
if inflightDeployment == nil {
inflightDeployment = &deployment
continue
}
var deploymentForCancellation *kapi.ReplicationController
if deployutil.DeploymentVersionFor(inflightDeployment) < deployutil.DeploymentVersionFor(&deployment) {
deploymentForCancellation, inflightDeployment = inflightDeployment, &deployment
} else {
deploymentForCancellation = &deployment
}
deploymentForCancellation.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
deploymentForCancellation.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists
if _, err := c.deploymentClient.updateDeployment(deploymentForCancellation.Namespace, deploymentForCancellation); err != nil {
util.HandleError(fmt.Errorf("couldn't cancel deployment %s: %v", deployutil.LabelForDeployment(deploymentForCancellation), err))
}
glog.V(4).Infof("Cancelled deployment %s for deployment config %s", deployutil.LabelForDeployment(deploymentForCancellation), deployutil.LabelForDeploymentConfig(config))
}
}
// if the latest deployment exists then nothing else needs to be done
if latestDeploymentExists {
return nil
}
// check to see if there are inflight deployments
if inflightDeployment != nil {
// raise a transientError so that the deployment config can be re-queued
glog.V(4).Infof("Found previous inflight deployment for %s - will requeue", deployutil.LabelForDeploymentConfig(config))
return transientError(fmt.Sprintf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config)))
}
// Try and build a deployment for the config.
deployment, err := c.makeDeployment(config)
if err != nil {
return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err))
}
// Compute the desired replicas for the deployment. Use the last completed
// deployment's current replica count, or the config template if there is no
// prior completed deployment available.
desiredReplicas := config.Template.ControllerTemplate.Replicas
if len(existingDeployments.Items) > 0 {
sort.Sort(deployutil.ByLatestVersionDesc(existingDeployments.Items))
for _, existing := range existingDeployments.Items {
if deployutil.DeploymentStatusFor(&existing) == deployapi.DeploymentStatusComplete {
desiredReplicas = existing.Spec.Replicas
glog.V(4).Infof("Desired replicas for %s set to %d based on prior completed deployment %s", deployutil.LabelForDeploymentConfig(config), desiredReplicas, existing.Name)
break
}
}
}
deployment.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(desiredReplicas)
// Create the deployment.
if _, err := c.deploymentClient.createDeployment(config.Namespace, deployment); err == nil {
glog.V(4).Infof("Created deployment for deployment config %s", deployutil.LabelForDeploymentConfig(config))
return nil
} else {
// If the deployment was already created, just move on. The cache could be stale, or another
// process could have already handled this update.
if errors.IsAlreadyExists(err) {
glog.V(4).Infof("Deployment already exists for deployment config %s", deployutil.LabelForDeploymentConfig(config))
return nil
}
glog.Warningf("Cannot create latest deployment for deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err)
return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
}
// findDetails inspects the given deployment configuration for any failure causes
// and returns any found details about it
func (c *DeploymentConfigController) findDetails(config *deployapi.DeploymentConfig) (string, *kapi.ReplicationControllerList, bool, error) {
// Check if any existing inflight deployments (any non-terminal state).
existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(config.Namespace, config.Name)
if err != nil {
return "", nil, false, fmt.Errorf("couldn't list deployments for deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err)
}
// check if the latest deployment exists
// we'll return after we've dealt with the multiple-active-deployments case
latestDeploymentExists, latestDeploymentStatus := deployutil.LatestDeploymentInfo(config, existingDeployments)
if latestDeploymentExists && latestDeploymentStatus != deployapi.DeploymentStatusFailed {
// If the latest deployment exists and is not failed, clear the dc message
return "", existingDeployments, latestDeploymentExists, nil
}
// Rest of the code will handle non-existing or failed latest deployment causes
// TODO: Inspect pod logs in case of failed latest
details := ""
allDetails := []string{}
if config.Details != nil && len(config.Details.Message) > 0 {
// Populate details with the previous message so that in case we stumble upon
// an unexpected client error, the message won't be overwritten
details = config.Details.Message
}
// Look into image change triggers and find out possible deployment failures such as
// missing image stream tags with or without build configurations pointing at them
for _, trigger := range config.Triggers {
if trigger.Type != deployapi.DeploymentTriggerOnImageChange || trigger.ImageChangeParams == nil || !trigger.ImageChangeParams.Automatic {
continue
}
name := trigger.ImageChangeParams.From.Name
tag := trigger.ImageChangeParams.Tag
istag := imageapi.JoinImageStreamTag(name, tag)
// Check if the image stream tag pointed by the trigger exists
if _, err := c.osClient.ImageStreamTags(config.Namespace).Get(name, tag); err != nil {
if !errors.IsNotFound(err) {
glog.V(2).Infof("Error while trying to get image stream tag %q: %v", istag, err)
return details, existingDeployments, latestDeploymentExists, nil
}
// In case the image stream tag was not found, then it either doesn't exist or doesn't exist yet
// (a build configuration output points to it so it's going to be populated at some point in the
// future)
details = fmt.Sprintf("The image trigger for image stream tag %q will have no effect because image stream tag %q does not exist.", istag, istag)
bcList, err := c.osClient.BuildConfigs(kapi.NamespaceAll).List(labels.Everything(), fields.Everything())
if err != nil {
glog.V(2).Infof("Error while trying to list build configs: %v", err)
return details, existingDeployments, latestDeploymentExists, nil
}
for _, bc := range bcList.Items {
if bc.Spec.Output.To != nil && bc.Spec.Output.To.Kind == "ImageStreamTag" {
parts := strings.Split(bc.Spec.Output.To.Name, ":")
if len(parts) != 2 {
glog.V(2).Infof("Invalid image stream tag: %q", bc.Spec.Output.To.Name)
return details, existingDeployments, latestDeploymentExists, nil
}
if parts[0] == name && parts[1] == tag {
details = fmt.Sprintf("The image trigger for image stream tag %q will have no effect because image stream tag %q does not exist.\n\tIf image stream tag %q is expected, check build config %q which produces image stream tag %q.", istag, istag, istag, bc.Name, istag)
break
}
}
}
// Try to see if the image stream exists, if not then the build will never be able to update the
// tag in question
if _, err := c.osClient.ImageStreams(config.Namespace).Get(name); err != nil {
glog.V(2).Infof("Error while trying to get image stream %q: %v", name, err)
if errors.IsNotFound(err) {
details = fmt.Sprintf("The image trigger for image stream tag %q will have no effect because image stream %q does not exist.", istag, name)
}
}
}
allDetails = append(allDetails, details)
}
if len(allDetails) > 1 {
for i := range allDetails {
allDetails[i] = fmt.Sprintf("\t* %s", allDetails[i])
}
// Prepend multiple errors warning
multipleErrWarning := fmt.Sprintf("Deployment config %q blocked by multiple errors:\n", config.Name)
allDetails = append([]string{multipleErrWarning}, allDetails...)
}
return strings.Join(allDetails, "\n"), existingDeployments, latestDeploymentExists, nil
}
// reconcileDeployments reconciles existing deployment replica counts which
// could have diverged outside the deployment process (e.g. due to auto or
// manual scaling, or partial deployments). The active deployment is the last
// successful deployment, not necessarily the latest in terms of the config
// version. The active deployment replica count should follow the config, and
// all other deployments should be scaled to zero.
//
// Previously, scaling behavior was that the config replica count was used
// only for initial deployments and the active deployment had to be scaled up
// directly. To continue supporting that old behavior we must detect when the
// deployment has been directly manipulated, and if so, preserve the directly
// updated value and sync the config with the deployment.
func (c *DeploymentConfigController) reconcileDeployments(existingDeployments *kapi.ReplicationControllerList, config *deployapi.DeploymentConfig) error {
latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
if !latestIsDeployed {
// We shouldn't be reconciling if the latest deployment hasn't been
// created; this is enforced on the calling side, but double checking
// can't hurt.
return nil
}
activeDeployment := deployutil.ActiveDeployment(config, existingDeployments)
// Compute the replica count for the active deployment (even if the active
// deployment doesn't exist). The active replica count is the value that
// should be assigned to the config, to allow the replica propagation to
// flow downward from the config.
//
// This takes into account resources predating the propagation behavior
// change, as well as external modifications to the deployments (e.g.
// scalers).
activeReplicas := config.Template.ControllerTemplate.Replicas
source := "the deploymentConfig itself (no change)"
if activeDeployment != nil {
activeDeploymentIsLatest := activeDeployment.Name == latestDeployment.Name
lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment)
if activeDeploymentIsLatest {
if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas {
activeReplicas = activeDeployment.Spec.Replicas
source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment))
}
} else {
if hasLastActiveReplicas {
if lastActiveReplicas != activeDeployment.Spec.Replicas && activeDeployment.Spec.Replicas > 0 {
activeReplicas = activeDeployment.Spec.Replicas
source = fmt.Sprintf("the active deployment %q which was scaled directly", deployutil.LabelForDeployment(activeDeployment))
}
} else {
if activeDeployment.Spec.Replicas > 0 {
activeReplicas = activeDeployment.Spec.Replicas
source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment))
} else {
latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment)
if latestHasDesiredReplicas {
activeReplicas = latestDesiredReplicas
source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment))
}
}
}
}
} else {
latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment)
if latestHasDesiredReplicas {
activeReplicas = latestDesiredReplicas
source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment))
}
}
// Bring the config in sync with the deployment. Once we know the config
// accurately represents the desired replica count of the active deployment,
// we can safely reconcile deployments.
if config.Template.ControllerTemplate.Replicas != activeReplicas {
oldReplicas := config.Template.ControllerTemplate.Replicas
config.Template.ControllerTemplate.Replicas = activeReplicas
_, err := c.osClient.DeploymentConfigs(config.Namespace).Update(config)
if err != nil {
return err
}
glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source)
}
// Reconcile deployments. The active deployment follows the config, and all
// other deployments should be scaled to zero.
for _, deployment := range existingDeployments.Items {
isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name
oldReplicaCount := deployment.Spec.Replicas
newReplicaCount := 0
if isActiveDeployment {
newReplicaCount = activeReplicas
}
lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment)
// Only update if necessary.
if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount {
deployment.Spec.Replicas = newReplicaCount
deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(newReplicaCount)
_, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment)
if err != nil {
c.recorder.Eventf(config, "DeploymentScaleFailed",
"Failed to scale deployment %q from %d to %d: %s", deployment.Name, oldReplicaCount, newReplicaCount, err)
return err
}
// Only report scaling events if we changed the replica count.
if oldReplicaCount != newReplicaCount {
c.recorder.Eventf(config, "DeploymentScaled",
"Scaled deployment %q from %d to %d", deployment.Name, oldReplicaCount, newReplicaCount)
} else {
glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(&deployment), newReplicaCount)
}
}
}
return nil
}
// reconcileDeployments reconciles existing deployment replica counts which
// could have diverged outside the deployment process (e.g. due to auto or
// manual scaling, or partial deployments). The active deployment is the last
// successful deployment, not necessarily the latest in terms of the config
// version. The active deployment replica count should follow the config, and
// all other deployments should be scaled to zero.
//
// Previously, scaling behavior was that the config replica count was used
// only for initial deployments and the active deployment had to be scaled up
// directly. To continue supporting that old behavior we must detect when the
// deployment has been directly manipulated, and if so, preserve the directly
// updated value and sync the config with the deployment.
func (c *DeploymentConfigController) reconcileDeployments(existingDeployments *kapi.ReplicationControllerList, config *deployapi.DeploymentConfig) error {
latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
if !latestIsDeployed {
// We shouldn't be reconciling if the latest deployment hasn't been
// created; this is enforced on the calling side, but double checking
// can't hurt.
return nil
}
activeDeployment := deployutil.ActiveDeployment(config, existingDeployments)
// Compute the replica count for the active deployment (even if the active
// deployment doesn't exist). The active replica count is the value that
// should be assigned to the config, to allow the replica propagation to
// flow downward from the config.
//
// By default we'll assume the config replicas should be used to update the
// active deployment except in special cases (like first sync or externally
// updated deployments.)
activeReplicas := config.Spec.Replicas
source := "the deploymentConfig itself (no change)"
activeDeploymentExists := activeDeployment != nil
activeDeploymentIsLatest := activeDeploymentExists && activeDeployment.Name == latestDeployment.Name
latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment)
switch {
case activeDeploymentExists && activeDeploymentIsLatest:
// The active/latest deployment follows the config unless this is its first
// sync or if an external change to the deployment replicas is detected.
lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment)
if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas {
activeReplicas = activeDeployment.Spec.Replicas
source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment))
}
case activeDeploymentExists && !activeDeploymentIsLatest:
// The active/non-latest deployment follows the config if it was
// previously synced; if this is the first sync, infer what the config
// value should be based on either the latest desired or whatever the
// deployment is currently scaled to.
_, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment)
if hasLastActiveReplicas {
break
}
if latestHasDesiredReplicas {
activeReplicas = latestDesiredReplicas
source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment))
} else if activeDeployment.Spec.Replicas > 0 {
activeReplicas = activeDeployment.Spec.Replicas
source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment))
}
case !activeDeploymentExists && latestHasDesiredReplicas:
// If there's no active deployment, use the latest desired, if available.
activeReplicas = latestDesiredReplicas
source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment))
}
// Bring the config in sync with the deployment. Once we know the config
// accurately represents the desired replica count of the active deployment,
// we can safely reconcile deployments.
//
// If the deployment config is test, never update the deployment config based
// on deployments, since test behavior overrides user scaling.
switch {
case config.Spec.Replicas == activeReplicas:
case config.Spec.Test:
glog.V(4).Infof("Detected changed replicas for test deploymentConfig %q, ignoring that change", deployutil.LabelForDeploymentConfig(config))
default:
oldReplicas := config.Spec.Replicas
config.Spec.Replicas = activeReplicas
var err error
config, err = c.osClient.DeploymentConfigs(config.Namespace).Update(config)
if err != nil {
return err
}
glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source)
}
// Reconcile deployments. The active deployment follows the config, and all
// other deployments should be scaled to zero.
for _, deployment := range existingDeployments.Items {
isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name
oldReplicaCount := deployment.Spec.Replicas
newReplicaCount := 0
if isActiveDeployment {
newReplicaCount = activeReplicas
}
if config.Spec.Test {
glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment))
newReplicaCount = 0
}
lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment)
// Only update if necessary.
if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount {
deployment.Spec.Replicas = newReplicaCount
deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(newReplicaCount)
_, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment)
if err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentScaleFailed",
"Failed to scale deployment %q from %d to %d: %s", deployment.Name, oldReplicaCount, newReplicaCount, err)
return err
}
// Only report scaling events if we changed the replica count.
if oldReplicaCount != newReplicaCount {
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentScaled",
"Scaled deployment %q from %d to %d", deployment.Name, oldReplicaCount, newReplicaCount)
} else {
glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(&deployment), newReplicaCount)
}
}
}
return c.updateStatus(config)
}
// reconcileDeployments reconciles existing deployment replica counts which
// could have diverged outside the deployment process (e.g. due to auto or
// manual scaling, or partial deployments). The active deployment is the last
// successful deployment, not necessarily the latest in terms of the config
// version. The active deployment replica count should follow the config, and
// all other deployments should be scaled to zero.
func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []kapi.ReplicationController, config *deployapi.DeploymentConfig) error {
activeDeployment := deployutil.ActiveDeployment(existingDeployments)
// Reconcile deployments. The active deployment follows the config, and all
// other deployments should be scaled to zero.
var updatedDeployments []kapi.ReplicationController
for i := range existingDeployments {
deployment := existingDeployments[i]
toAppend := deployment
isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name
oldReplicaCount := deployment.Spec.Replicas
newReplicaCount := int32(0)
if isActiveDeployment {
newReplicaCount = config.Spec.Replicas
}
if config.Spec.Test {
glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment))
newReplicaCount = 0
}
// Only update if necessary.
var copied *kapi.ReplicationController
if newReplicaCount != oldReplicaCount {
if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error {
// refresh the replication controller version
rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name)
if err != nil {
return err
}
copied, err = deployutil.DeploymentDeepCopy(rc)
if err != nil {
glog.V(2).Infof("Deep copy of deployment %q failed: %v", rc.Name, err)
return err
}
copied.Spec.Replicas = newReplicaCount
copied, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied)
return err
}); err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "ReplicationControllerScaleFailed",
"Failed to scale replication controler %q from %d to %d: %v", deployment.Name, oldReplicaCount, newReplicaCount, err)
return err
}
c.recorder.Eventf(config, kapi.EventTypeNormal, "ReplicationControllerScaled", "Scaled replication controller %q from %d to %d", copied.Name, oldReplicaCount, newReplicaCount)
toAppend = *copied
}
updatedDeployments = append(updatedDeployments, toAppend)
}
// As the deployment configuration has changed, we need to make sure to clean
// up old deployments if we have now reached our deployment history quota
if err := c.cleanupOldDeployments(updatedDeployments, config); err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "ReplicationControllerCleanupFailed", "Couldn't clean up replication controllers: %v", err)
}
return c.updateStatus(config, updatedDeployments)
}
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error {
// There's nothing to reconcile until the version is nonzero.
if config.Status.LatestVersion == 0 {
glog.V(5).Infof("Waiting for first version of %q", deployutil.LabelForDeploymentConfig(config))
return c.updateStatus(config)
}
// Find all deployments owned by the deploymentConfig.
sel := deployutil.ConfigSelector(config.Name)
existingDeployments, err := c.kubeClient.ReplicationControllers(config.Namespace).List(kapi.ListOptions{LabelSelector: sel})
if err != nil {
return err
}
latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
// If the latest deployment doesn't exist yet, cancel any running
// deployments to allow them to be superceded by the new config version.
awaitingCancellations := false
if !latestIsDeployed {
for _, deployment := range existingDeployments.Items {
// Skip deployments with an outcome.
if deployutil.IsTerminatedDeployment(&deployment) {
continue
}
// Cancel running deployments.
awaitingCancellations = true
if !deployutil.IsDeploymentCancelled(&deployment) {
deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists
_, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment)
if err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err)
} else {
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion)
}
}
}
}
// Wait for deployment cancellations before reconciling or creating a new
// deployment to avoid competing with existing deployment processes.
if awaitingCancellations {
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion)
// raise a transientError so that the deployment config can be re-queued
return transientError(fmt.Sprintf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config)))
}
// If the latest deployment already exists, reconcile existing deployments
// and return early.
if latestIsDeployed {
// If the latest deployment is still running, try again later. We don't
// want to compete with the deployer.
if !deployutil.IsTerminatedDeployment(latestDeployment) {
return c.updateStatus(config)
}
return c.reconcileDeployments(existingDeployments, config)
}
// No deployments are running and the latest deployment doesn't exist, so
// create the new deployment.
deployment, err := deployutil.MakeDeployment(config, c.codec)
if err != nil {
return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err))
}
created, err := c.kubeClient.ReplicationControllers(config.Namespace).Create(deployment)
if err != nil {
// If the deployment was already created, just move on. The cache could be
// stale, or another process could have already handled this update.
if errors.IsAlreadyExists(err) {
return c.updateStatus(config)
}
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err)
return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion)
return c.updateStatus(config)
}
// Handle processes image change triggers associated with imagestream.
func (c *ImageChangeController) Handle(stream *imageapi.ImageStream) error {
configs, err := c.dcLister.GetConfigsForImageStream(stream)
if err != nil {
return fmt.Errorf("couldn't get list of deployment configs while handling image stream %q: %v", imageapi.LabelForStream(stream), err)
}
// Find any configs which should be updated based on the new image state
var configsToUpdate []*deployapi.DeploymentConfig
for n, config := range configs {
glog.V(4).Infof("Detecting image changes for deployment config %q", deployutil.LabelForDeploymentConfig(config))
hasImageChange := false
for j := range config.Spec.Triggers {
// because config can be copied during this loop, make sure we load from config for subsequent loops
trigger := config.Spec.Triggers[j]
params := trigger.ImageChangeParams
// Only automatic image change triggers should fire
if trigger.Type != deployapi.DeploymentTriggerOnImageChange {
continue
}
// All initial deployments should have their images resolved in order to
// be able to work and not try to pull non-existent images from DockerHub.
// Deployments with automatic set to false that have been deployed at least
// once shouldn't have their images updated.
if (!params.Automatic || config.Spec.Paused) && len(params.LastTriggeredImage) > 0 {
continue
}
// Check if the image stream matches the trigger
if !triggerMatchesImage(config, params, stream) {
continue
}
_, tag, ok := imageapi.SplitImageStreamTag(params.From.Name)
if !ok {
glog.Warningf("Invalid image stream tag %q in %q", params.From.Name, deployutil.LabelForDeploymentConfig(config))
continue
}
// Find the latest tag event for the trigger tag
latestEvent := imageapi.LatestTaggedImage(stream, tag)
if latestEvent == nil {
glog.V(5).Infof("Couldn't find latest tag event for tag %q in image stream %q", tag, imageapi.LabelForStream(stream))
continue
}
// Ensure a change occurred
if len(latestEvent.DockerImageReference) == 0 || latestEvent.DockerImageReference == params.LastTriggeredImage {
glog.V(4).Infof("No image changes for deployment config %q were detected", deployutil.LabelForDeploymentConfig(config))
continue
}
names := sets.NewString(params.ContainerNames...)
for i := range config.Spec.Template.Spec.Containers {
container := &config.Spec.Template.Spec.Containers[i]
if !names.Has(container.Name) {
continue
}
if !hasImageChange {
// create a copy prior to mutation
result, err := deployutil.DeploymentConfigDeepCopy(configs[n])
if err != nil {
utilruntime.HandleError(err)
continue
}
configs[n] = result
container = &configs[n].Spec.Template.Spec.Containers[i]
params = configs[n].Spec.Triggers[j].ImageChangeParams
}
// Update the image
container.Image = latestEvent.DockerImageReference
// Log the last triggered image ID
params.LastTriggeredImage = latestEvent.DockerImageReference
hasImageChange = true
}
}
if hasImageChange {
configsToUpdate = append(configsToUpdate, configs[n])
}
}
// Attempt to regenerate all configs which may contain image updates
anyFailed := false
for _, config := range configsToUpdate {
if _, err := c.dn.DeploymentConfigs(config.Namespace).Update(config); err != nil {
utilruntime.HandleError(err)
anyFailed = true
} else {
glog.V(4).Infof("Updated deployment config %q for trigger on image stream %q",
deployutil.LabelForDeploymentConfig(config), imageapi.LabelForStream(stream))
}
}
if anyFailed {
return fmt.Errorf("couldn't update some deployment configs for trigger on image stream %q", imageapi.LabelForStream(stream))
}
return nil
}