func (c *DeploymentConfigController) updateStatus(config *deployapi.DeploymentConfig, deployments []kapi.ReplicationController) error { newStatus, err := c.calculateStatus(*config, deployments) if err != nil { glog.V(2).Infof("Cannot calculate the status for %q: %v", deployutil.LabelForDeploymentConfig(config), err) return err } latestExists, latestRC := deployutil.LatestDeploymentInfo(config, deployments) if !latestExists { latestRC = nil } updateConditions(config, &newStatus, latestRC) // NOTE: We should update the status of the deployment config only if we need to, otherwise // we hotloop between updates. if reflect.DeepEqual(newStatus, config.Status) { return nil } copied, err := deployutil.DeploymentConfigDeepCopy(config) if err != nil { return err } copied.Status = newStatus if _, err := c.dn.DeploymentConfigs(copied.Namespace).UpdateStatus(copied); err != nil { glog.V(2).Infof("Cannot update the status for %q: %v", deployutil.LabelForDeploymentConfig(copied), err) return err } glog.V(4).Infof("Updated the status for %q (observed generation: %d)", deployutil.LabelForDeploymentConfig(copied), copied.Status.ObservedGeneration) return nil }
// Handle processes image change triggers associated with imageRepo. func (c *ImageChangeController) Handle(imageRepo *imageapi.ImageStream) error { configs, err := c.deploymentConfigClient.listDeploymentConfigs() if err != nil { return fmt.Errorf("couldn't get list of DeploymentConfig while handling ImageStream %s: %v", labelForRepo(imageRepo), err) } // Find any configs which should be updated based on the new image state configsToUpdate := map[string]*deployapi.DeploymentConfig{} for _, config := range configs { glog.V(4).Infof("Detecting changed images for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config)) for _, trigger := range config.Triggers { params := trigger.ImageChangeParams // Only automatic image change triggers should fire if trigger.Type != deployapi.DeploymentTriggerOnImageChange || !params.Automatic { continue } // Check if the image repo matches the trigger if !triggerMatchesImage(config, params, imageRepo) { continue } // Find the latest tag event for the trigger tag latestEvent := imageapi.LatestTaggedImage(imageRepo, params.Tag) if latestEvent == nil { glog.V(2).Infof("Couldn't find latest tag event for tag %s in ImageStream %s", params.Tag, labelForRepo(imageRepo)) continue } // Ensure a change occurred if len(latestEvent.DockerImageReference) > 0 && latestEvent.DockerImageReference != params.LastTriggeredImage { // Mark the config for regeneration configsToUpdate[config.Name] = config } } } // Attempt to regenerate all configs which may contain image updates anyFailed := false for _, config := range configsToUpdate { err := c.regenerate(config) if err != nil { anyFailed = true glog.V(2).Infof("Couldn't regenerate DeploymentConfig %s: %s", deployutil.LabelForDeploymentConfig(config), err) continue } glog.V(4).Infof("Regenerated DeploymentConfig %s in response to image change trigger", deployutil.LabelForDeploymentConfig(config)) } if anyFailed { return fatalError(fmt.Sprintf("couldn't update some DeploymentConfig for trigger on ImageStream %s", labelForRepo(imageRepo))) } glog.V(4).Infof("Updated all DeploymentConfigs for trigger on ImageStream %s", labelForRepo(imageRepo)) return nil }
// Handle processes change triggers for config. func (c *DeploymentConfigChangeController) Handle(config *deployapi.DeploymentConfig) error { if !deployutil.HasChangeTrigger(config) { glog.V(5).Infof("Ignoring DeploymentConfig %s; no change triggers detected", deployutil.LabelForDeploymentConfig(config)) return nil } if config.Status.LatestVersion == 0 { _, _, abort, err := c.generateDeployment(config) if err != nil { if kerrors.IsConflict(err) { return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err)) } glog.V(4).Infof("Couldn't create initial deployment for deploymentConfig %q: %v", deployutil.LabelForDeploymentConfig(config), err) return nil } if !abort { glog.V(4).Infof("Created initial deployment for deploymentConfig %q", deployutil.LabelForDeploymentConfig(config)) } return nil } latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config) deployment, err := c.changeStrategy.getDeployment(config.Namespace, latestDeploymentName) if err != nil { // If there's no deployment for the latest config, we have no basis of // comparison. It's the responsibility of the deployment config controller // to make the deployment for the config, so return early. if kerrors.IsNotFound(err) { glog.V(5).Infof("Ignoring change for DeploymentConfig %s; no existing Deployment found", deployutil.LabelForDeploymentConfig(config)) return nil } return fmt.Errorf("couldn't retrieve Deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } deployedConfig, err := c.decodeConfig(deployment) if err != nil { return fatalError(fmt.Sprintf("error decoding DeploymentConfig from Deployment %s for DeploymentConfig %s: %v", deployutil.LabelForDeployment(deployment), deployutil.LabelForDeploymentConfig(config), err)) } // Detect template diffs, and return early if there aren't any changes. if kapi.Semantic.DeepEqual(config.Spec.Template, deployedConfig.Spec.Template) { glog.V(5).Infof("Ignoring DeploymentConfig change for %s (latestVersion=%d); same as Deployment %s", deployutil.LabelForDeploymentConfig(config), config.Status.LatestVersion, deployutil.LabelForDeployment(deployment)) return nil } // There was a template diff, so generate a new config version. fromVersion, toVersion, abort, err := c.generateDeployment(config) if err != nil { if kerrors.IsConflict(err) { return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err)) } return fmt.Errorf("couldn't generate deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } if !abort { glog.V(4).Infof("Updated DeploymentConfig %s from version %d to %d for existing deployment %s", deployutil.LabelForDeploymentConfig(config), fromVersion, toVersion, deployutil.LabelForDeployment(deployment)) } return nil }
func (c *DeploymentConfigController) updateStatus(config *deployapi.DeploymentConfig) error { if config.Generation > config.Status.ObservedGeneration { config.Status.ObservedGeneration = config.Generation } if _, err := c.osClient.DeploymentConfigs(config.Namespace).UpdateStatus(config); err != nil { glog.V(2).Infof("Cannot update the status for %q: %v", deployutil.LabelForDeploymentConfig(config), err) return transientError(fmt.Sprintf("cannot update the status for %q - requeuing", deployutil.LabelForDeploymentConfig(config))) } glog.V(4).Infof("Updated the status for %q (observed generation: %d)", deployutil.LabelForDeploymentConfig(config), config.Status.ObservedGeneration) return nil }
func (c *DeploymentConfigController) updateStatus(config *deployapi.DeploymentConfig) error { // NOTE: We should update the status of the deployment config only if we need to, otherwise // we hotloop between updates. if !needsUpdate(config) { return nil } config.Status.ObservedGeneration = config.Generation if _, err := c.dn.DeploymentConfigs(config.Namespace).UpdateStatus(config); err != nil { glog.V(2).Infof("Cannot update the status for %q: %v", deployutil.LabelForDeploymentConfig(config), err) return err } glog.V(4).Infof("Updated the status for %q (observed generation: %d)", deployutil.LabelForDeploymentConfig(config), config.Status.ObservedGeneration) return nil }
// Handle processes change triggers for config. func (c *DeploymentConfigChangeController) Handle(config *deployapi.DeploymentConfig) error { if !deployutil.HasChangeTrigger(config) { glog.V(5).Infof("Ignoring deployment config %q; no change triggers detected", deployutil.LabelForDeploymentConfig(config)) return nil } // Try to decode this deployment config from the encoded annotation found in // its latest deployment. decoded, err := c.decodeFromLatest(config) if err != nil { return err } // If this is the initial deployment, then wait for any images that need to be resolved, otherwise // automatically start a new deployment. if config.Status.LatestVersion == 0 { canTrigger, causes := canTrigger(config, decoded) if !canTrigger { // If we cannot trigger then we need to wait for the image change controller. glog.V(5).Infof("Ignoring deployment config %q; template image needs to be resolved by the image change controller", deployutil.LabelForDeploymentConfig(config)) return nil } return c.updateStatus(config, causes) } // If this is not the initial deployment, check if there is any template difference between // this and the decoded deploymentconfig. if kapi.Semantic.DeepEqual(config.Spec.Template, decoded.Spec.Template) { return nil } _, causes := canTrigger(config, decoded) return c.updateStatus(config, causes) }
// regenerate calls the generator to get a new config. If the newly generated // config's version is newer, update the old config to be the new config. // Otherwise do nothing. func (c *ImageChangeController) regenerate(config *deployapi.DeploymentConfig) error { // Get a regenerated config which includes the new image repo references newConfig, err := c.deploymentConfigClient.generateDeploymentConfig(config.Namespace, config.Name) if err != nil { return fmt.Errorf("error generating new version of DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } // No update occurred if config.LatestVersion == newConfig.LatestVersion { glog.V(4).Infof("No version difference for generated DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config)) return nil } // Persist the new config _, err = c.deploymentConfigClient.updateDeploymentConfig(newConfig.Namespace, newConfig) if err != nil { return err } glog.Infof("Regenerated DeploymentConfig %s for image updates", deployutil.LabelForDeploymentConfig(config)) return nil }
func (g *DeploymentConfigGenerator) findImageStream(config *deployapi.DeploymentConfig, params *deployapi.DeploymentTriggerImageChangeParams) (*imageapi.ImageStream, error) { if len(params.From.Name) > 0 { namespace := params.From.Namespace if len(namespace) == 0 { namespace = config.Namespace } name, _, ok := imageapi.SplitImageStreamTag(params.From.Name) if !ok { return nil, fmt.Errorf("invalid ImageStreamTag: %s", params.From.Name) } return g.Client.GetImageStream(kapi.WithNamespace(kapi.NewContext(), namespace), name) } return nil, fmt.Errorf("couldn't find image stream for config %s trigger params", deployutil.LabelForDeploymentConfig(config)) }
func (g *DeploymentConfigGenerator) findImageStream(config *deployapi.DeploymentConfig, params *deployapi.DeploymentTriggerImageChangeParams) (*imageapi.ImageStream, error) { // Try to find the repo by ObjectReference if len(params.From.Name) > 0 { namespace := params.From.Namespace if len(namespace) == 0 { namespace = config.Namespace } return g.Client.GetImageStream(kapi.WithNamespace(kapi.NewContext(), namespace), params.From.Name) } // Fall back to a list based lookup on RepositoryName repos, err := g.Client.ListImageStreams(kapi.WithNamespace(kapi.NewContext(), config.Namespace)) if err != nil { return nil, err } for _, repo := range repos.Items { if len(repo.Status.DockerImageRepository) > 0 && params.RepositoryName == repo.Status.DockerImageRepository { return &repo, nil } } return nil, fmt.Errorf("couldn't find image stream for config %s trigger params", deployutil.LabelForDeploymentConfig(config)) }
// Handle implements the loop that processes deployment configs. Since this controller started // using caches, the provided config MUST be deep-copied beforehand (see work() in factory.go). func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error { // There's nothing to reconcile until the version is nonzero. if config.Status.LatestVersion == 0 { return c.updateStatus(config, []kapi.ReplicationController{}) } // Find all deployments owned by the deployment config. selector := deployutil.ConfigSelector(config.Name) existingDeployments, err := c.rcStore.ReplicationControllers(config.Namespace).List(selector) if err != nil { return err } // In case the deployment config has been marked for deletion, merely update its status with // the latest available information. Some deletions make take some time to complete so there // is value in doing this. if config.DeletionTimestamp != nil { return c.updateStatus(config, existingDeployments) } latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) // If the latest deployment doesn't exist yet, cancel any running // deployments to allow them to be superceded by the new config version. awaitingCancellations := false if !latestIsDeployed { for i := range existingDeployments { deployment := existingDeployments[i] // Skip deployments with an outcome. if deployutil.IsTerminatedDeployment(&deployment) { continue } // Cancel running deployments. awaitingCancellations = true if !deployutil.IsDeploymentCancelled(&deployment) { // Retry faster on conflicts var updatedDeployment *kapi.ReplicationController if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error { rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name) if kapierrors.IsNotFound(err) { return nil } if err != nil { return err } copied, err := deployutil.DeploymentDeepCopy(rc) if err != nil { return err } copied.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue copied.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists updatedDeployment, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied) return err }); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err) } else { if updatedDeployment != nil { // replace the current deployment with the updated copy so that a future update has a chance at working existingDeployments[i] = *updatedDeployment c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion) } } } } } // Wait for deployment cancellations before reconciling or creating a new // deployment to avoid competing with existing deployment processes. if awaitingCancellations { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion) return fmt.Errorf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config)) } // If the latest deployment already exists, reconcile existing deployments // and return early. if latestIsDeployed { // If the latest deployment is still running, try again later. We don't // want to compete with the deployer. if !deployutil.IsTerminatedDeployment(latestDeployment) { return c.updateStatus(config, existingDeployments) } return c.reconcileDeployments(existingDeployments, config) } // If the config is paused we shouldn't create new deployments for it. if config.Spec.Paused { // in order for revision history limit cleanup to work for paused // deployments, we need to trigger it here if err := c.cleanupOldDeployments(existingDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err) } return c.updateStatus(config, existingDeployments) } // No deployments are running and the latest deployment doesn't exist, so // create the new deployment. deployment, err := deployutil.MakeDeployment(config, c.codec) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)) } created, err := c.rn.ReplicationControllers(config.Namespace).Create(deployment) if err != nil { // If the deployment was already created, just move on. The cache could be // stale, or another process could have already handled this update. if kapierrors.IsAlreadyExists(err) { return c.updateStatus(config, existingDeployments) } c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err) return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err) } c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion) // As we've just created a new deployment, we need to make sure to clean // up old deployments if we have reached our deployment history quota existingDeployments = append(existingDeployments, *created) if err := c.cleanupOldDeployments(existingDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err) } return c.updateStatus(config, existingDeployments) }
// Handle processes change triggers for config. func (c *DeploymentConfigChangeController) Handle(config *deployapi.DeploymentConfig) error { hasChangeTrigger := false for _, trigger := range config.Triggers { if trigger.Type == deployapi.DeploymentTriggerOnConfigChange { hasChangeTrigger = true break } } if !hasChangeTrigger { glog.V(4).Infof("Ignoring DeploymentConfig %s; no change triggers detected", deployutil.LabelForDeploymentConfig(config)) return nil } if config.LatestVersion == 0 { _, _, err := c.generateDeployment(config) if err != nil { if kerrors.IsConflict(err) { return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err)) } // TODO: This needs handled by setting some state within the API so // users know why the trigger isn't doing anything. // https://github.com/openshift/origin/issues/3526 return nil } glog.V(4).Infof("Created initial Deployment for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config)) return nil } latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config) deployment, err := c.changeStrategy.getDeployment(config.Namespace, latestDeploymentName) if err != nil { // If there's no deployment for the latest config, we have no basis of // comparison. It's the responsibility of the deployment config controller // to make the deployment for the config, so return early. if kerrors.IsNotFound(err) { glog.V(2).Infof("Ignoring change for DeploymentConfig %s; no existing Deployment found", deployutil.LabelForDeploymentConfig(config)) return nil } return fmt.Errorf("couldn't retrieve Deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } deployedConfig, err := c.decodeConfig(deployment) if err != nil { return fatalError(fmt.Sprintf("error decoding DeploymentConfig from Deployment %s for DeploymentConfig %s: %v", deployutil.LabelForDeployment(deployment), deployutil.LabelForDeploymentConfig(config), err)) } newSpec, oldSpec := config.Template.ControllerTemplate.Template.Spec, deployedConfig.Template.ControllerTemplate.Template.Spec if kapi.Semantic.DeepEqual(oldSpec, newSpec) { glog.V(2).Infof("Ignoring DeploymentConfig change for %s (latestVersion=%d); same as Deployment %s", deployutil.LabelForDeploymentConfig(config), config.LatestVersion, deployutil.LabelForDeployment(deployment)) return nil } fromVersion, toVersion, err := c.generateDeployment(config) if err != nil { if kerrors.IsConflict(err) { return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err)) } return fmt.Errorf("couldn't generate deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } glog.V(4).Infof("Updated DeploymentConfig %s from version %d to %d for existing deployment %s", deployutil.LabelForDeploymentConfig(config), fromVersion, toVersion, deployutil.LabelForDeployment(deployment)) return nil }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. // // Previously, scaling behavior was that the config replica count was used // only for initial deployments and the active deployment had to be scaled up // directly. To continue supporting that old behavior we must detect when the // deployment has been directly manipulated, and if so, preserve the directly // updated value and sync the config with the deployment. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []kapi.ReplicationController, config *deployapi.DeploymentConfig) error { latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) if !latestIsDeployed { // We shouldn't be reconciling if the latest deployment hasn't been // created; this is enforced on the calling side, but double checking // can't hurt. return c.updateStatus(config, existingDeployments) } activeDeployment := deployutil.ActiveDeployment(existingDeployments) // Compute the replica count for the active deployment (even if the active // deployment doesn't exist). The active replica count is the value that // should be assigned to the config, to allow the replica propagation to // flow downward from the config. // // By default we'll assume the config replicas should be used to update the // active deployment except in special cases (like first sync or externally // updated deployments.) activeReplicas := config.Spec.Replicas source := "the deploymentConfig itself (no change)" activeDeploymentExists := activeDeployment != nil activeDeploymentIsLatest := activeDeploymentExists && activeDeployment.Name == latestDeployment.Name latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) switch { case activeDeploymentExists && activeDeploymentIsLatest: // The active/latest deployment follows the config unless this is its first // sync or if an external change to the deployment replicas is detected. lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment)) } case activeDeploymentExists && !activeDeploymentIsLatest: // The active/non-latest deployment follows the config if it was // previously synced; if this is the first sync, infer what the config // value should be based on either the latest desired or whatever the // deployment is currently scaled to. _, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if hasLastActiveReplicas { break } if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment)) } else if activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment)) } case !activeDeploymentExists && latestHasDesiredReplicas: // If there's no active deployment, use the latest desired, if available. activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment)) } // Bring the config in sync with the deployment. Once we know the config // accurately represents the desired replica count of the active deployment, // we can safely reconcile deployments. // // If the deployment config is test, never update the deployment config based // on deployments, since test behavior overrides user scaling. switch { case config.Spec.Replicas == activeReplicas: case config.Spec.Test: glog.V(4).Infof("Detected changed replicas for test deploymentConfig %q, ignoring that change", deployutil.LabelForDeploymentConfig(config)) default: copied, err := deployutil.DeploymentConfigDeepCopy(config) if err != nil { return err } oldReplicas := copied.Spec.Replicas copied.Spec.Replicas = activeReplicas config, err = c.dn.DeploymentConfigs(copied.Namespace).Update(copied) if err != nil { return err } glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source) } // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. var updatedDeployments []kapi.ReplicationController for i := range existingDeployments { deployment := existingDeployments[i] toAppend := deployment isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := int32(0) if isActiveDeployment { newReplicaCount = activeReplicas } if config.Spec.Test { glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment)) newReplicaCount = 0 } lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment) // Only update if necessary. var copied *kapi.ReplicationController if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount { if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error { // refresh the replication controller version rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name) if err != nil { return err } copied, err = deployutil.DeploymentDeepCopy(rc) if err != nil { glog.V(2).Infof("Deep copy of deployment %q failed: %v", rc.Name, err) return err } copied.Spec.Replicas = newReplicaCount copied.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(int(newReplicaCount)) _, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied) return err }); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentScaleFailed", "Failed to scale deployment %q from %d to %d: %v", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } // Only report scaling events if we changed the replica count. if oldReplicaCount != newReplicaCount { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentScaled", "Scaled deployment %q from %d to %d", copied.Name, oldReplicaCount, newReplicaCount) } else { glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(copied), newReplicaCount) } toAppend = *copied } updatedDeployments = append(updatedDeployments, toAppend) } // As the deployment configuration has changed, we need to make sure to clean // up old deployments if we have now reached our deployment history quota if err := c.cleanupOldDeployments(existingDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err) } return c.updateStatus(config, updatedDeployments) }
func (c *DeploymentTriggerController) updateDeploymentConfig(old, cur interface{}) { newDc := cur.(*deployapi.DeploymentConfig) oldDc := old.(*deployapi.DeploymentConfig) // A periodic relist will send update events for all known deployment configs. if newDc.ResourceVersion == oldDc.ResourceVersion { return } // No need to enqueue deployment configs that have no triggers or are paused. if len(newDc.Spec.Triggers) == 0 || newDc.Spec.Paused { return } // We don't want to compete with the main deployment config controller. Let's process this // config once it's synced. Note that this does not eliminate conflicts between the two // controllers because the main controller is constantly updating deployment configs as // owning replication controllers and pods are updated. if !deployutil.HasSynced(newDc, newDc.Generation) { return } // Enqueue the deployment config if it hasn't been deployed yet. if newDc.Status.LatestVersion == 0 { c.enqueueDeploymentConfig(newDc) return } // Compare deployment config templates before enqueueing. This reduces the amount of times // we will try to instantiate a deployment config at the expense of duplicating some of the // work that the instantiate endpoint is already doing but I think this is fine. shouldInstantiate := true latestRc, err := c.rcLister.ReplicationControllers(newDc.Namespace).Get(deployutil.LatestDeploymentNameForConfig(newDc)) if err != nil { // If we get an error here it may be due to the rc cache lagging behind. In such a case // just defer to the api server (instantiate REST) where we will retry this. glog.V(2).Infof("Cannot get latest rc for dc %s:%d (%v) - will defer to instantiate", deployutil.LabelForDeploymentConfig(newDc), newDc.Status.LatestVersion, err) } else { initial, err := deployutil.DecodeDeploymentConfig(latestRc, c.codec) if err != nil { glog.V(2).Infof("Cannot decode dc from replication controller %s: %v", deployutil.LabelForDeployment(latestRc), err) return } shouldInstantiate = !reflect.DeepEqual(newDc.Spec.Template, initial.Spec.Template) } if !shouldInstantiate { return } c.enqueueDeploymentConfig(newDc) }
// addDeploymentConfig is used for making sure that new deployment configs with triggers pointing // to existing images will be deployed as soon as they are created. func (c *ImageChangeController) addDeploymentConfig(obj interface{}) { dc := obj.(*deployapi.DeploymentConfig) for _, stream := range c.streamLister.GetStreamsForConfig(dc) { glog.V(4).Infof("Reconciling stream %q for config %q\n", imageapi.LabelForStream(stream), deployutil.LabelForDeploymentConfig(dc)) c.enqueueImageStream(stream) } }
// decodeFromLatest will try to return the decoded version of the current deploymentconfig found // in the annotations of its latest deployment. If there is no previous deploymentconfig (ie. // latestVersion == 0), the returned deploymentconfig will be the same. func (c *DeploymentTriggerController) decodeFromLatest(config *deployapi.DeploymentConfig) (*deployapi.DeploymentConfig, error) { if config.Status.LatestVersion == 0 { return config, nil } latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config) deployment, err := c.rn.ReplicationControllers(config.Namespace).Get(latestDeploymentName) if err != nil { // If there's no deployment for the latest config, we have no basis of // comparison. It's the responsibility of the deployment config controller // to make the deployment for the config, so return early. return nil, fmt.Errorf("couldn't retrieve deployment for deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err) } latest, err := deployutil.DecodeDeploymentConfig(deployment, c.codec) if err != nil { return nil, fatalError(err.Error()) } return latest, nil }
// Create generates a new DeploymentConfig representing a rollback. func (r *REST) Create(ctx kapi.Context, obj runtime.Object) (runtime.Object, error) { namespace, ok := kapi.NamespaceFrom(ctx) if !ok { return nil, kerrors.NewBadRequest("namespace parameter required.") } rollback, ok := obj.(*deployapi.DeploymentConfigRollback) if !ok { return nil, kerrors.NewBadRequest(fmt.Sprintf("not a rollback spec: %#v", obj)) } if errs := validation.ValidateDeploymentConfigRollback(rollback); len(errs) > 0 { return nil, kerrors.NewInvalid(deployapi.Kind("DeploymentConfigRollback"), rollback.Name, errs) } from, err := r.dn.DeploymentConfigs(namespace).Get(rollback.Name) if err != nil { return nil, newInvalidError(rollback, fmt.Sprintf("cannot get deployment config %q: %v", rollback.Name, err)) } switch from.Status.LatestVersion { case 0: return nil, newInvalidError(rollback, "cannot rollback an undeployed config") case 1: return nil, newInvalidError(rollback, fmt.Sprintf("no previous deployment exists for %q", deployutil.LabelForDeploymentConfig(from))) } revision := from.Status.LatestVersion - 1 if rollback.Spec.Revision > 0 { revision = rollback.Spec.Revision } // Find the target deployment and decode its config. name := deployutil.DeploymentNameForConfigVersion(from.Name, revision) targetDeployment, err := r.rn.ReplicationControllers(namespace).Get(name) if err != nil { return nil, newInvalidError(rollback, err.Error()) } to, err := deployutil.DecodeDeploymentConfig(targetDeployment, r.codec) if err != nil { return nil, newInvalidError(rollback, fmt.Sprintf("couldn't decode deployment config from deployment: %v", err)) } if from.Annotations == nil && len(rollback.UpdatedAnnotations) > 0 { from.Annotations = make(map[string]string) } for key, value := range rollback.UpdatedAnnotations { from.Annotations[key] = value } return r.generator.GenerateRollback(from, to, &rollback.Spec) }
// Handle processes image change triggers associated with imagestream. func (c *ImageChangeController) Handle(stream *imageapi.ImageStream) error { configs, err := c.listDeploymentConfigs() if err != nil { return fmt.Errorf("couldn't get list of deployment configs while handling image stream %q: %v", imageapi.LabelForStream(stream), err) } // Find any configs which should be updated based on the new image state configsToUpdate := []*deployapi.DeploymentConfig{} for _, config := range configs { glog.V(4).Infof("Detecting image changes for deployment config %q", deployutil.LabelForDeploymentConfig(config)) hasImageChange := false for _, trigger := range config.Spec.Triggers { params := trigger.ImageChangeParams // Only automatic image change triggers should fire if trigger.Type != deployapi.DeploymentTriggerOnImageChange { continue } // All initial deployments (latestVersion == 0) should have their images resolved in order // to be able to work and not try to pull non-existent images from DockerHub. // Deployments with automatic set to false that have been deployed at least once (latestVersion > 0) // shouldn't have their images updated. if !params.Automatic && config.Status.LatestVersion != 0 { continue } // Check if the image stream matches the trigger if !triggerMatchesImage(config, params, stream) { continue } _, tag, ok := imageapi.SplitImageStreamTag(params.From.Name) if !ok { glog.Warningf("Invalid image stream tag %q in %q", params.From.Name, deployutil.LabelForDeploymentConfig(config)) continue } // Find the latest tag event for the trigger tag latestEvent := imageapi.LatestTaggedImage(stream, tag) if latestEvent == nil { glog.V(5).Infof("Couldn't find latest tag event for tag %q in image stream %q", tag, imageapi.LabelForStream(stream)) continue } // Ensure a change occurred if len(latestEvent.DockerImageReference) == 0 || latestEvent.DockerImageReference == params.LastTriggeredImage { glog.V(4).Infof("No image changes for deployment config %q were detected", deployutil.LabelForDeploymentConfig(config)) continue } names := sets.NewString(params.ContainerNames...) for i := range config.Spec.Template.Spec.Containers { container := &config.Spec.Template.Spec.Containers[i] if !names.Has(container.Name) { continue } // Update the image container.Image = latestEvent.DockerImageReference // Log the last triggered image ID params.LastTriggeredImage = latestEvent.DockerImageReference hasImageChange = true } } if hasImageChange { configsToUpdate = append(configsToUpdate, config) } } // Attempt to regenerate all configs which may contain image updates anyFailed := false for _, config := range configsToUpdate { if _, err := c.client.DeploymentConfigs(config.Namespace).Update(config); err != nil { anyFailed = true glog.V(2).Infof("Couldn't update deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err) } } if anyFailed { return fatalError(fmt.Sprintf("couldn't update some deployment configs for trigger on image stream %q", imageapi.LabelForStream(stream))) } glog.V(5).Infof("Updated all deployment configs for trigger on image stream %q", imageapi.LabelForStream(stream)) return nil }
// updateDeploymentConfig is used for making sure that deployment configs with new triggers // pointing to existing images will be deployed as soon as they are updated. func (c *ImageChangeController) updateDeploymentConfig(old, cur interface{}) { // A periodic relist will send update events for all known configs. newDc := cur.(*deployapi.DeploymentConfig) oldDc := old.(*deployapi.DeploymentConfig) if newDc.ResourceVersion == oldDc.ResourceVersion { return } for _, stream := range c.streamLister.GetStreamsForConfig(newDc) { glog.V(4).Infof("Reconciling stream %q for config %q\n", imageapi.LabelForStream(stream), deployutil.LabelForDeploymentConfig(newDc)) c.enqueueImageStream(stream) } }
// Handle processes config and creates a new deployment if necessary. func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error { // Inspect a deployment configuration every time the controller reconciles it details, existingDeployments, latestDeploymentExists, err := c.findDetails(config) if err != nil { return err } config, err = c.updateDetails(config, details) if err != nil { return transientError(err.Error()) } // Only deploy when the version has advanced past 0. if config.LatestVersion == 0 { glog.V(5).Infof("Waiting for first version of %s", deployutil.LabelForDeploymentConfig(config)) return nil } var inflightDeployment *kapi.ReplicationController for _, deployment := range existingDeployments.Items { deploymentStatus := deployutil.DeploymentStatusFor(&deployment) switch deploymentStatus { case deployapi.DeploymentStatusFailed, deployapi.DeploymentStatusComplete: // Previous deployment in terminal state - can ignore // Ignoring specific deployment states so that any newly introduced // deployment state will not be ignored default: if inflightDeployment == nil { inflightDeployment = &deployment continue } var deploymentForCancellation *kapi.ReplicationController if deployutil.DeploymentVersionFor(inflightDeployment) < deployutil.DeploymentVersionFor(&deployment) { deploymentForCancellation, inflightDeployment = inflightDeployment, &deployment } else { deploymentForCancellation = &deployment } deploymentForCancellation.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue deploymentForCancellation.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists if _, err := c.deploymentClient.updateDeployment(deploymentForCancellation.Namespace, deploymentForCancellation); err != nil { util.HandleError(fmt.Errorf("couldn't cancel deployment %s: %v", deployutil.LabelForDeployment(deploymentForCancellation), err)) } glog.V(4).Infof("Cancelled deployment %s for deployment config %s", deployutil.LabelForDeployment(deploymentForCancellation), deployutil.LabelForDeploymentConfig(config)) } } // if the latest deployment exists then nothing else needs to be done if latestDeploymentExists { return nil } // check to see if there are inflight deployments if inflightDeployment != nil { // raise a transientError so that the deployment config can be re-queued glog.V(4).Infof("Found previous inflight deployment for %s - will requeue", deployutil.LabelForDeploymentConfig(config)) return transientError(fmt.Sprintf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))) } // Try and build a deployment for the config. deployment, err := c.makeDeployment(config) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)) } // Compute the desired replicas for the deployment. Use the last completed // deployment's current replica count, or the config template if there is no // prior completed deployment available. desiredReplicas := config.Template.ControllerTemplate.Replicas if len(existingDeployments.Items) > 0 { sort.Sort(deployutil.ByLatestVersionDesc(existingDeployments.Items)) for _, existing := range existingDeployments.Items { if deployutil.DeploymentStatusFor(&existing) == deployapi.DeploymentStatusComplete { desiredReplicas = existing.Spec.Replicas glog.V(4).Infof("Desired replicas for %s set to %d based on prior completed deployment %s", deployutil.LabelForDeploymentConfig(config), desiredReplicas, existing.Name) break } } } deployment.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(desiredReplicas) // Create the deployment. if _, err := c.deploymentClient.createDeployment(config.Namespace, deployment); err == nil { glog.V(4).Infof("Created deployment for deployment config %s", deployutil.LabelForDeploymentConfig(config)) return nil } else { // If the deployment was already created, just move on. The cache could be stale, or another // process could have already handled this update. if errors.IsAlreadyExists(err) { glog.V(4).Infof("Deployment already exists for deployment config %s", deployutil.LabelForDeploymentConfig(config)) return nil } glog.Warningf("Cannot create latest deployment for deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err) return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err) } }
// findDetails inspects the given deployment configuration for any failure causes // and returns any found details about it func (c *DeploymentConfigController) findDetails(config *deployapi.DeploymentConfig) (string, *kapi.ReplicationControllerList, bool, error) { // Check if any existing inflight deployments (any non-terminal state). existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(config.Namespace, config.Name) if err != nil { return "", nil, false, fmt.Errorf("couldn't list deployments for deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err) } // check if the latest deployment exists // we'll return after we've dealt with the multiple-active-deployments case latestDeploymentExists, latestDeploymentStatus := deployutil.LatestDeploymentInfo(config, existingDeployments) if latestDeploymentExists && latestDeploymentStatus != deployapi.DeploymentStatusFailed { // If the latest deployment exists and is not failed, clear the dc message return "", existingDeployments, latestDeploymentExists, nil } // Rest of the code will handle non-existing or failed latest deployment causes // TODO: Inspect pod logs in case of failed latest details := "" allDetails := []string{} if config.Details != nil && len(config.Details.Message) > 0 { // Populate details with the previous message so that in case we stumble upon // an unexpected client error, the message won't be overwritten details = config.Details.Message } // Look into image change triggers and find out possible deployment failures such as // missing image stream tags with or without build configurations pointing at them for _, trigger := range config.Triggers { if trigger.Type != deployapi.DeploymentTriggerOnImageChange || trigger.ImageChangeParams == nil || !trigger.ImageChangeParams.Automatic { continue } name := trigger.ImageChangeParams.From.Name tag := trigger.ImageChangeParams.Tag istag := imageapi.JoinImageStreamTag(name, tag) // Check if the image stream tag pointed by the trigger exists if _, err := c.osClient.ImageStreamTags(config.Namespace).Get(name, tag); err != nil { if !errors.IsNotFound(err) { glog.V(2).Infof("Error while trying to get image stream tag %q: %v", istag, err) return details, existingDeployments, latestDeploymentExists, nil } // In case the image stream tag was not found, then it either doesn't exist or doesn't exist yet // (a build configuration output points to it so it's going to be populated at some point in the // future) details = fmt.Sprintf("The image trigger for image stream tag %q will have no effect because image stream tag %q does not exist.", istag, istag) bcList, err := c.osClient.BuildConfigs(kapi.NamespaceAll).List(labels.Everything(), fields.Everything()) if err != nil { glog.V(2).Infof("Error while trying to list build configs: %v", err) return details, existingDeployments, latestDeploymentExists, nil } for _, bc := range bcList.Items { if bc.Spec.Output.To != nil && bc.Spec.Output.To.Kind == "ImageStreamTag" { parts := strings.Split(bc.Spec.Output.To.Name, ":") if len(parts) != 2 { glog.V(2).Infof("Invalid image stream tag: %q", bc.Spec.Output.To.Name) return details, existingDeployments, latestDeploymentExists, nil } if parts[0] == name && parts[1] == tag { details = fmt.Sprintf("The image trigger for image stream tag %q will have no effect because image stream tag %q does not exist.\n\tIf image stream tag %q is expected, check build config %q which produces image stream tag %q.", istag, istag, istag, bc.Name, istag) break } } } // Try to see if the image stream exists, if not then the build will never be able to update the // tag in question if _, err := c.osClient.ImageStreams(config.Namespace).Get(name); err != nil { glog.V(2).Infof("Error while trying to get image stream %q: %v", name, err) if errors.IsNotFound(err) { details = fmt.Sprintf("The image trigger for image stream tag %q will have no effect because image stream %q does not exist.", istag, name) } } } allDetails = append(allDetails, details) } if len(allDetails) > 1 { for i := range allDetails { allDetails[i] = fmt.Sprintf("\t* %s", allDetails[i]) } // Prepend multiple errors warning multipleErrWarning := fmt.Sprintf("Deployment config %q blocked by multiple errors:\n", config.Name) allDetails = append([]string{multipleErrWarning}, allDetails...) } return strings.Join(allDetails, "\n"), existingDeployments, latestDeploymentExists, nil }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. // // Previously, scaling behavior was that the config replica count was used // only for initial deployments and the active deployment had to be scaled up // directly. To continue supporting that old behavior we must detect when the // deployment has been directly manipulated, and if so, preserve the directly // updated value and sync the config with the deployment. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments *kapi.ReplicationControllerList, config *deployapi.DeploymentConfig) error { latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) if !latestIsDeployed { // We shouldn't be reconciling if the latest deployment hasn't been // created; this is enforced on the calling side, but double checking // can't hurt. return nil } activeDeployment := deployutil.ActiveDeployment(config, existingDeployments) // Compute the replica count for the active deployment (even if the active // deployment doesn't exist). The active replica count is the value that // should be assigned to the config, to allow the replica propagation to // flow downward from the config. // // This takes into account resources predating the propagation behavior // change, as well as external modifications to the deployments (e.g. // scalers). activeReplicas := config.Template.ControllerTemplate.Replicas source := "the deploymentConfig itself (no change)" if activeDeployment != nil { activeDeploymentIsLatest := activeDeployment.Name == latestDeployment.Name lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if activeDeploymentIsLatest { if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment)) } } else { if hasLastActiveReplicas { if lastActiveReplicas != activeDeployment.Spec.Replicas && activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which was scaled directly", deployutil.LabelForDeployment(activeDeployment)) } } else { if activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment)) } else { latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment)) } } } } } else { latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment)) } } // Bring the config in sync with the deployment. Once we know the config // accurately represents the desired replica count of the active deployment, // we can safely reconcile deployments. if config.Template.ControllerTemplate.Replicas != activeReplicas { oldReplicas := config.Template.ControllerTemplate.Replicas config.Template.ControllerTemplate.Replicas = activeReplicas _, err := c.osClient.DeploymentConfigs(config.Namespace).Update(config) if err != nil { return err } glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source) } // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. for _, deployment := range existingDeployments.Items { isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := 0 if isActiveDeployment { newReplicaCount = activeReplicas } lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment) // Only update if necessary. if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount { deployment.Spec.Replicas = newReplicaCount deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(newReplicaCount) _, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err != nil { c.recorder.Eventf(config, "DeploymentScaleFailed", "Failed to scale deployment %q from %d to %d: %s", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } // Only report scaling events if we changed the replica count. if oldReplicaCount != newReplicaCount { c.recorder.Eventf(config, "DeploymentScaled", "Scaled deployment %q from %d to %d", deployment.Name, oldReplicaCount, newReplicaCount) } else { glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(&deployment), newReplicaCount) } } } return nil }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. // // Previously, scaling behavior was that the config replica count was used // only for initial deployments and the active deployment had to be scaled up // directly. To continue supporting that old behavior we must detect when the // deployment has been directly manipulated, and if so, preserve the directly // updated value and sync the config with the deployment. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments *kapi.ReplicationControllerList, config *deployapi.DeploymentConfig) error { latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) if !latestIsDeployed { // We shouldn't be reconciling if the latest deployment hasn't been // created; this is enforced on the calling side, but double checking // can't hurt. return nil } activeDeployment := deployutil.ActiveDeployment(config, existingDeployments) // Compute the replica count for the active deployment (even if the active // deployment doesn't exist). The active replica count is the value that // should be assigned to the config, to allow the replica propagation to // flow downward from the config. // // By default we'll assume the config replicas should be used to update the // active deployment except in special cases (like first sync or externally // updated deployments.) activeReplicas := config.Spec.Replicas source := "the deploymentConfig itself (no change)" activeDeploymentExists := activeDeployment != nil activeDeploymentIsLatest := activeDeploymentExists && activeDeployment.Name == latestDeployment.Name latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) switch { case activeDeploymentExists && activeDeploymentIsLatest: // The active/latest deployment follows the config unless this is its first // sync or if an external change to the deployment replicas is detected. lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment)) } case activeDeploymentExists && !activeDeploymentIsLatest: // The active/non-latest deployment follows the config if it was // previously synced; if this is the first sync, infer what the config // value should be based on either the latest desired or whatever the // deployment is currently scaled to. _, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if hasLastActiveReplicas { break } if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment)) } else if activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment)) } case !activeDeploymentExists && latestHasDesiredReplicas: // If there's no active deployment, use the latest desired, if available. activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment)) } // Bring the config in sync with the deployment. Once we know the config // accurately represents the desired replica count of the active deployment, // we can safely reconcile deployments. // // If the deployment config is test, never update the deployment config based // on deployments, since test behavior overrides user scaling. switch { case config.Spec.Replicas == activeReplicas: case config.Spec.Test: glog.V(4).Infof("Detected changed replicas for test deploymentConfig %q, ignoring that change", deployutil.LabelForDeploymentConfig(config)) default: oldReplicas := config.Spec.Replicas config.Spec.Replicas = activeReplicas var err error config, err = c.osClient.DeploymentConfigs(config.Namespace).Update(config) if err != nil { return err } glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source) } // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. for _, deployment := range existingDeployments.Items { isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := 0 if isActiveDeployment { newReplicaCount = activeReplicas } if config.Spec.Test { glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment)) newReplicaCount = 0 } lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment) // Only update if necessary. if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount { deployment.Spec.Replicas = newReplicaCount deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(newReplicaCount) _, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentScaleFailed", "Failed to scale deployment %q from %d to %d: %s", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } // Only report scaling events if we changed the replica count. if oldReplicaCount != newReplicaCount { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentScaled", "Scaled deployment %q from %d to %d", deployment.Name, oldReplicaCount, newReplicaCount) } else { glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(&deployment), newReplicaCount) } } } return c.updateStatus(config) }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []kapi.ReplicationController, config *deployapi.DeploymentConfig) error { activeDeployment := deployutil.ActiveDeployment(existingDeployments) // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. var updatedDeployments []kapi.ReplicationController for i := range existingDeployments { deployment := existingDeployments[i] toAppend := deployment isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := int32(0) if isActiveDeployment { newReplicaCount = config.Spec.Replicas } if config.Spec.Test { glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment)) newReplicaCount = 0 } // Only update if necessary. var copied *kapi.ReplicationController if newReplicaCount != oldReplicaCount { if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error { // refresh the replication controller version rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name) if err != nil { return err } copied, err = deployutil.DeploymentDeepCopy(rc) if err != nil { glog.V(2).Infof("Deep copy of deployment %q failed: %v", rc.Name, err) return err } copied.Spec.Replicas = newReplicaCount copied, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied) return err }); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "ReplicationControllerScaleFailed", "Failed to scale replication controler %q from %d to %d: %v", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } c.recorder.Eventf(config, kapi.EventTypeNormal, "ReplicationControllerScaled", "Scaled replication controller %q from %d to %d", copied.Name, oldReplicaCount, newReplicaCount) toAppend = *copied } updatedDeployments = append(updatedDeployments, toAppend) } // As the deployment configuration has changed, we need to make sure to clean // up old deployments if we have now reached our deployment history quota if err := c.cleanupOldDeployments(updatedDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "ReplicationControllerCleanupFailed", "Couldn't clean up replication controllers: %v", err) } return c.updateStatus(config, updatedDeployments) }
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error { // There's nothing to reconcile until the version is nonzero. if config.Status.LatestVersion == 0 { glog.V(5).Infof("Waiting for first version of %q", deployutil.LabelForDeploymentConfig(config)) return c.updateStatus(config) } // Find all deployments owned by the deploymentConfig. sel := deployutil.ConfigSelector(config.Name) existingDeployments, err := c.kubeClient.ReplicationControllers(config.Namespace).List(kapi.ListOptions{LabelSelector: sel}) if err != nil { return err } latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) // If the latest deployment doesn't exist yet, cancel any running // deployments to allow them to be superceded by the new config version. awaitingCancellations := false if !latestIsDeployed { for _, deployment := range existingDeployments.Items { // Skip deployments with an outcome. if deployutil.IsTerminatedDeployment(&deployment) { continue } // Cancel running deployments. awaitingCancellations = true if !deployutil.IsDeploymentCancelled(&deployment) { deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists _, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err) } else { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion) } } } } // Wait for deployment cancellations before reconciling or creating a new // deployment to avoid competing with existing deployment processes. if awaitingCancellations { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion) // raise a transientError so that the deployment config can be re-queued return transientError(fmt.Sprintf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))) } // If the latest deployment already exists, reconcile existing deployments // and return early. if latestIsDeployed { // If the latest deployment is still running, try again later. We don't // want to compete with the deployer. if !deployutil.IsTerminatedDeployment(latestDeployment) { return c.updateStatus(config) } return c.reconcileDeployments(existingDeployments, config) } // No deployments are running and the latest deployment doesn't exist, so // create the new deployment. deployment, err := deployutil.MakeDeployment(config, c.codec) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)) } created, err := c.kubeClient.ReplicationControllers(config.Namespace).Create(deployment) if err != nil { // If the deployment was already created, just move on. The cache could be // stale, or another process could have already handled this update. if errors.IsAlreadyExists(err) { return c.updateStatus(config) } c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err) return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err) } c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion) return c.updateStatus(config) }
// Handle processes image change triggers associated with imagestream. func (c *ImageChangeController) Handle(stream *imageapi.ImageStream) error { configs, err := c.dcLister.GetConfigsForImageStream(stream) if err != nil { return fmt.Errorf("couldn't get list of deployment configs while handling image stream %q: %v", imageapi.LabelForStream(stream), err) } // Find any configs which should be updated based on the new image state var configsToUpdate []*deployapi.DeploymentConfig for n, config := range configs { glog.V(4).Infof("Detecting image changes for deployment config %q", deployutil.LabelForDeploymentConfig(config)) hasImageChange := false for j := range config.Spec.Triggers { // because config can be copied during this loop, make sure we load from config for subsequent loops trigger := config.Spec.Triggers[j] params := trigger.ImageChangeParams // Only automatic image change triggers should fire if trigger.Type != deployapi.DeploymentTriggerOnImageChange { continue } // All initial deployments should have their images resolved in order to // be able to work and not try to pull non-existent images from DockerHub. // Deployments with automatic set to false that have been deployed at least // once shouldn't have their images updated. if (!params.Automatic || config.Spec.Paused) && len(params.LastTriggeredImage) > 0 { continue } // Check if the image stream matches the trigger if !triggerMatchesImage(config, params, stream) { continue } _, tag, ok := imageapi.SplitImageStreamTag(params.From.Name) if !ok { glog.Warningf("Invalid image stream tag %q in %q", params.From.Name, deployutil.LabelForDeploymentConfig(config)) continue } // Find the latest tag event for the trigger tag latestEvent := imageapi.LatestTaggedImage(stream, tag) if latestEvent == nil { glog.V(5).Infof("Couldn't find latest tag event for tag %q in image stream %q", tag, imageapi.LabelForStream(stream)) continue } // Ensure a change occurred if len(latestEvent.DockerImageReference) == 0 || latestEvent.DockerImageReference == params.LastTriggeredImage { glog.V(4).Infof("No image changes for deployment config %q were detected", deployutil.LabelForDeploymentConfig(config)) continue } names := sets.NewString(params.ContainerNames...) for i := range config.Spec.Template.Spec.Containers { container := &config.Spec.Template.Spec.Containers[i] if !names.Has(container.Name) { continue } if !hasImageChange { // create a copy prior to mutation result, err := deployutil.DeploymentConfigDeepCopy(configs[n]) if err != nil { utilruntime.HandleError(err) continue } configs[n] = result container = &configs[n].Spec.Template.Spec.Containers[i] params = configs[n].Spec.Triggers[j].ImageChangeParams } // Update the image container.Image = latestEvent.DockerImageReference // Log the last triggered image ID params.LastTriggeredImage = latestEvent.DockerImageReference hasImageChange = true } } if hasImageChange { configsToUpdate = append(configsToUpdate, configs[n]) } } // Attempt to regenerate all configs which may contain image updates anyFailed := false for _, config := range configsToUpdate { if _, err := c.dn.DeploymentConfigs(config.Namespace).Update(config); err != nil { utilruntime.HandleError(err) anyFailed = true } else { glog.V(4).Infof("Updated deployment config %q for trigger on image stream %q", deployutil.LabelForDeploymentConfig(config), imageapi.LabelForStream(stream)) } } if anyFailed { return fmt.Errorf("couldn't update some deployment configs for trigger on image stream %q", imageapi.LabelForStream(stream)) } return nil }