func (c *DeploymentConfigController) updateStatus(config *deployapi.DeploymentConfig, deployments []kapi.ReplicationController) error { newStatus, err := c.calculateStatus(*config, deployments) if err != nil { glog.V(2).Infof("Cannot calculate the status for %q: %v", deployutil.LabelForDeploymentConfig(config), err) return err } latestExists, latestRC := deployutil.LatestDeploymentInfo(config, deployments) if !latestExists { latestRC = nil } updateConditions(config, &newStatus, latestRC) // NOTE: We should update the status of the deployment config only if we need to, otherwise // we hotloop between updates. if reflect.DeepEqual(newStatus, config.Status) { return nil } copied, err := deployutil.DeploymentConfigDeepCopy(config) if err != nil { return err } copied.Status = newStatus if _, err := c.dn.DeploymentConfigs(copied.Namespace).UpdateStatus(copied); err != nil { glog.V(2).Infof("Cannot update the status for %q: %v", deployutil.LabelForDeploymentConfig(copied), err) return err } glog.V(4).Infof("Updated the status for %q (observed generation: %d)", deployutil.LabelForDeploymentConfig(copied), copied.Status.ObservedGeneration) return nil }
func (c *DeploymentConfigController) calculateStatus(config deployapi.DeploymentConfig, deployments []kapi.ReplicationController, additional ...deployapi.DeploymentCondition) (deployapi.DeploymentConfigStatus, error) { selector := labels.Set(config.Spec.Selector).AsSelector() // TODO: Replace with using rc.status.availableReplicas that comes with the next rebase. pods, err := c.podStore.Pods(config.Namespace).List(selector) if err != nil { return config.Status, err } available := deployutil.GetAvailablePods(pods, config.Spec.MinReadySeconds) // UpdatedReplicas represents the replicas that use the deployment config template which means // we should inform about the replicas of the latest deployment and not the active. latestReplicas := int32(0) latestExists, latestRC := deployutil.LatestDeploymentInfo(&config, deployments) if !latestExists { latestRC = nil } else { latestReplicas = deployutil.GetStatusReplicaCountForDeployments([]kapi.ReplicationController{*latestRC}) } total := deployutil.GetReplicaCountForDeployments(deployments) status := deployapi.DeploymentConfigStatus{ LatestVersion: config.Status.LatestVersion, Details: config.Status.Details, ObservedGeneration: config.Generation, Replicas: deployutil.GetStatusReplicaCountForDeployments(deployments), UpdatedReplicas: latestReplicas, AvailableReplicas: available, UnavailableReplicas: total - available, Conditions: config.Status.Conditions, } isProgressing := deployutil.IsProgressing(config, status) updateConditions(config, &status, latestRC, isProgressing) for _, cond := range additional { deployutil.SetDeploymentCondition(&status, cond) } return status, nil }
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error { // There's nothing to reconcile until the version is nonzero. if config.Status.LatestVersion == 0 { glog.V(5).Infof("Waiting for first version of %q", deployutil.LabelForDeploymentConfig(config)) return c.updateStatus(config) } // Find all deployments owned by the deploymentConfig. sel := deployutil.ConfigSelector(config.Name) existingDeployments, err := c.kubeClient.ReplicationControllers(config.Namespace).List(kapi.ListOptions{LabelSelector: sel}) if err != nil { return err } latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) // If the latest deployment doesn't exist yet, cancel any running // deployments to allow them to be superceded by the new config version. awaitingCancellations := false if !latestIsDeployed { for _, deployment := range existingDeployments.Items { // Skip deployments with an outcome. if deployutil.IsTerminatedDeployment(&deployment) { continue } // Cancel running deployments. awaitingCancellations = true if !deployutil.IsDeploymentCancelled(&deployment) { deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists _, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err) } else { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion) } } } } // Wait for deployment cancellations before reconciling or creating a new // deployment to avoid competing with existing deployment processes. if awaitingCancellations { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion) // raise a transientError so that the deployment config can be re-queued return transientError(fmt.Sprintf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))) } // If the latest deployment already exists, reconcile existing deployments // and return early. if latestIsDeployed { // If the latest deployment is still running, try again later. We don't // want to compete with the deployer. if !deployutil.IsTerminatedDeployment(latestDeployment) { return c.updateStatus(config) } return c.reconcileDeployments(existingDeployments, config) } // No deployments are running and the latest deployment doesn't exist, so // create the new deployment. deployment, err := deployutil.MakeDeployment(config, c.codec) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)) } created, err := c.kubeClient.ReplicationControllers(config.Namespace).Create(deployment) if err != nil { // If the deployment was already created, just move on. The cache could be // stale, or another process could have already handled this update. if errors.IsAlreadyExists(err) { return c.updateStatus(config) } c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err) return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err) } c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion) return c.updateStatus(config) }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. // // Previously, scaling behavior was that the config replica count was used // only for initial deployments and the active deployment had to be scaled up // directly. To continue supporting that old behavior we must detect when the // deployment has been directly manipulated, and if so, preserve the directly // updated value and sync the config with the deployment. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments *kapi.ReplicationControllerList, config *deployapi.DeploymentConfig) error { latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) if !latestIsDeployed { // We shouldn't be reconciling if the latest deployment hasn't been // created; this is enforced on the calling side, but double checking // can't hurt. return nil } activeDeployment := deployutil.ActiveDeployment(config, existingDeployments) // Compute the replica count for the active deployment (even if the active // deployment doesn't exist). The active replica count is the value that // should be assigned to the config, to allow the replica propagation to // flow downward from the config. // // By default we'll assume the config replicas should be used to update the // active deployment except in special cases (like first sync or externally // updated deployments.) activeReplicas := config.Spec.Replicas source := "the deploymentConfig itself (no change)" activeDeploymentExists := activeDeployment != nil activeDeploymentIsLatest := activeDeploymentExists && activeDeployment.Name == latestDeployment.Name latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) switch { case activeDeploymentExists && activeDeploymentIsLatest: // The active/latest deployment follows the config unless this is its first // sync or if an external change to the deployment replicas is detected. lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment)) } case activeDeploymentExists && !activeDeploymentIsLatest: // The active/non-latest deployment follows the config if it was // previously synced; if this is the first sync, infer what the config // value should be based on either the latest desired or whatever the // deployment is currently scaled to. _, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if hasLastActiveReplicas { break } if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment)) } else if activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment)) } case !activeDeploymentExists && latestHasDesiredReplicas: // If there's no active deployment, use the latest desired, if available. activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment)) } // Bring the config in sync with the deployment. Once we know the config // accurately represents the desired replica count of the active deployment, // we can safely reconcile deployments. // // If the deployment config is test, never update the deployment config based // on deployments, since test behavior overrides user scaling. switch { case config.Spec.Replicas == activeReplicas: case config.Spec.Test: glog.V(4).Infof("Detected changed replicas for test deploymentConfig %q, ignoring that change", deployutil.LabelForDeploymentConfig(config)) default: oldReplicas := config.Spec.Replicas config.Spec.Replicas = activeReplicas var err error config, err = c.osClient.DeploymentConfigs(config.Namespace).Update(config) if err != nil { return err } glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source) } // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. for _, deployment := range existingDeployments.Items { isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := 0 if isActiveDeployment { newReplicaCount = activeReplicas } if config.Spec.Test { glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment)) newReplicaCount = 0 } lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment) // Only update if necessary. if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount { deployment.Spec.Replicas = newReplicaCount deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(newReplicaCount) _, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentScaleFailed", "Failed to scale deployment %q from %d to %d: %s", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } // Only report scaling events if we changed the replica count. if oldReplicaCount != newReplicaCount { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentScaled", "Scaled deployment %q from %d to %d", deployment.Name, oldReplicaCount, newReplicaCount) } else { glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(&deployment), newReplicaCount) } } } return c.updateStatus(config) }
// Handle implements the loop that processes deployment configs. Since this controller started // using caches, the provided config MUST be deep-copied beforehand (see work() in factory.go). func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error { // There's nothing to reconcile until the version is nonzero. if config.Status.LatestVersion == 0 { return c.updateStatus(config, []kapi.ReplicationController{}) } // Find all deployments owned by the deployment config. selector := deployutil.ConfigSelector(config.Name) existingDeployments, err := c.rcStore.ReplicationControllers(config.Namespace).List(selector) if err != nil { return err } // In case the deployment config has been marked for deletion, merely update its status with // the latest available information. Some deletions make take some time to complete so there // is value in doing this. if config.DeletionTimestamp != nil { return c.updateStatus(config, existingDeployments) } latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) // If the latest deployment doesn't exist yet, cancel any running // deployments to allow them to be superceded by the new config version. awaitingCancellations := false if !latestIsDeployed { for i := range existingDeployments { deployment := existingDeployments[i] // Skip deployments with an outcome. if deployutil.IsTerminatedDeployment(&deployment) { continue } // Cancel running deployments. awaitingCancellations = true if !deployutil.IsDeploymentCancelled(&deployment) { // Retry faster on conflicts var updatedDeployment *kapi.ReplicationController if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error { rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name) if kapierrors.IsNotFound(err) { return nil } if err != nil { return err } copied, err := deployutil.DeploymentDeepCopy(rc) if err != nil { return err } copied.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue copied.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists updatedDeployment, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied) return err }); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err) } else { if updatedDeployment != nil { // replace the current deployment with the updated copy so that a future update has a chance at working existingDeployments[i] = *updatedDeployment c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion) } } } } } // Wait for deployment cancellations before reconciling or creating a new // deployment to avoid competing with existing deployment processes. if awaitingCancellations { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion) return fmt.Errorf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config)) } // If the latest deployment already exists, reconcile existing deployments // and return early. if latestIsDeployed { // If the latest deployment is still running, try again later. We don't // want to compete with the deployer. if !deployutil.IsTerminatedDeployment(latestDeployment) { return c.updateStatus(config, existingDeployments) } return c.reconcileDeployments(existingDeployments, config) } // If the config is paused we shouldn't create new deployments for it. if config.Spec.Paused { // in order for revision history limit cleanup to work for paused // deployments, we need to trigger it here if err := c.cleanupOldDeployments(existingDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err) } return c.updateStatus(config, existingDeployments) } // No deployments are running and the latest deployment doesn't exist, so // create the new deployment. deployment, err := deployutil.MakeDeployment(config, c.codec) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)) } created, err := c.rn.ReplicationControllers(config.Namespace).Create(deployment) if err != nil { // If the deployment was already created, just move on. The cache could be // stale, or another process could have already handled this update. if kapierrors.IsAlreadyExists(err) { return c.updateStatus(config, existingDeployments) } c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err) return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err) } c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion) // As we've just created a new deployment, we need to make sure to clean // up old deployments if we have reached our deployment history quota existingDeployments = append(existingDeployments, *created) if err := c.cleanupOldDeployments(existingDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err) } return c.updateStatus(config, existingDeployments) }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. // // Previously, scaling behavior was that the config replica count was used // only for initial deployments and the active deployment had to be scaled up // directly. To continue supporting that old behavior we must detect when the // deployment has been directly manipulated, and if so, preserve the directly // updated value and sync the config with the deployment. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []kapi.ReplicationController, config *deployapi.DeploymentConfig) error { latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) if !latestIsDeployed { // We shouldn't be reconciling if the latest deployment hasn't been // created; this is enforced on the calling side, but double checking // can't hurt. return c.updateStatus(config, existingDeployments) } activeDeployment := deployutil.ActiveDeployment(existingDeployments) // Compute the replica count for the active deployment (even if the active // deployment doesn't exist). The active replica count is the value that // should be assigned to the config, to allow the replica propagation to // flow downward from the config. // // By default we'll assume the config replicas should be used to update the // active deployment except in special cases (like first sync or externally // updated deployments.) activeReplicas := config.Spec.Replicas source := "the deploymentConfig itself (no change)" activeDeploymentExists := activeDeployment != nil activeDeploymentIsLatest := activeDeploymentExists && activeDeployment.Name == latestDeployment.Name latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) switch { case activeDeploymentExists && activeDeploymentIsLatest: // The active/latest deployment follows the config unless this is its first // sync or if an external change to the deployment replicas is detected. lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment)) } case activeDeploymentExists && !activeDeploymentIsLatest: // The active/non-latest deployment follows the config if it was // previously synced; if this is the first sync, infer what the config // value should be based on either the latest desired or whatever the // deployment is currently scaled to. _, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if hasLastActiveReplicas { break } if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment)) } else if activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment)) } case !activeDeploymentExists && latestHasDesiredReplicas: // If there's no active deployment, use the latest desired, if available. activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment)) } // Bring the config in sync with the deployment. Once we know the config // accurately represents the desired replica count of the active deployment, // we can safely reconcile deployments. // // If the deployment config is test, never update the deployment config based // on deployments, since test behavior overrides user scaling. switch { case config.Spec.Replicas == activeReplicas: case config.Spec.Test: glog.V(4).Infof("Detected changed replicas for test deploymentConfig %q, ignoring that change", deployutil.LabelForDeploymentConfig(config)) default: copied, err := deployutil.DeploymentConfigDeepCopy(config) if err != nil { return err } oldReplicas := copied.Spec.Replicas copied.Spec.Replicas = activeReplicas config, err = c.dn.DeploymentConfigs(copied.Namespace).Update(copied) if err != nil { return err } glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source) } // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. var updatedDeployments []kapi.ReplicationController for i := range existingDeployments { deployment := existingDeployments[i] toAppend := deployment isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := int32(0) if isActiveDeployment { newReplicaCount = activeReplicas } if config.Spec.Test { glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment)) newReplicaCount = 0 } lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment) // Only update if necessary. var copied *kapi.ReplicationController if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount { if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error { // refresh the replication controller version rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name) if err != nil { return err } copied, err = deployutil.DeploymentDeepCopy(rc) if err != nil { glog.V(2).Infof("Deep copy of deployment %q failed: %v", rc.Name, err) return err } copied.Spec.Replicas = newReplicaCount copied.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(int(newReplicaCount)) _, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied) return err }); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentScaleFailed", "Failed to scale deployment %q from %d to %d: %v", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } // Only report scaling events if we changed the replica count. if oldReplicaCount != newReplicaCount { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentScaled", "Scaled deployment %q from %d to %d", copied.Name, oldReplicaCount, newReplicaCount) } else { glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(copied), newReplicaCount) } toAppend = *copied } updatedDeployments = append(updatedDeployments, toAppend) } // As the deployment configuration has changed, we need to make sure to clean // up old deployments if we have now reached our deployment history quota if err := c.cleanupOldDeployments(existingDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err) } return c.updateStatus(config, updatedDeployments) }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. // // Previously, scaling behavior was that the config replica count was used // only for initial deployments and the active deployment had to be scaled up // directly. To continue supporting that old behavior we must detect when the // deployment has been directly manipulated, and if so, preserve the directly // updated value and sync the config with the deployment. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments *kapi.ReplicationControllerList, config *deployapi.DeploymentConfig) error { latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) if !latestIsDeployed { // We shouldn't be reconciling if the latest deployment hasn't been // created; this is enforced on the calling side, but double checking // can't hurt. return nil } activeDeployment := deployutil.ActiveDeployment(config, existingDeployments) // Compute the replica count for the active deployment (even if the active // deployment doesn't exist). The active replica count is the value that // should be assigned to the config, to allow the replica propagation to // flow downward from the config. // // This takes into account resources predating the propagation behavior // change, as well as external modifications to the deployments (e.g. // scalers). activeReplicas := config.Template.ControllerTemplate.Replicas source := "the deploymentConfig itself (no change)" if activeDeployment != nil { activeDeploymentIsLatest := activeDeployment.Name == latestDeployment.Name lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if activeDeploymentIsLatest { if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment)) } } else { if hasLastActiveReplicas { if lastActiveReplicas != activeDeployment.Spec.Replicas && activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which was scaled directly", deployutil.LabelForDeployment(activeDeployment)) } } else { if activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment)) } else { latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment)) } } } } } else { latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment)) } } // Bring the config in sync with the deployment. Once we know the config // accurately represents the desired replica count of the active deployment, // we can safely reconcile deployments. if config.Template.ControllerTemplate.Replicas != activeReplicas { oldReplicas := config.Template.ControllerTemplate.Replicas config.Template.ControllerTemplate.Replicas = activeReplicas _, err := c.osClient.DeploymentConfigs(config.Namespace).Update(config) if err != nil { return err } glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source) } // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. for _, deployment := range existingDeployments.Items { isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := 0 if isActiveDeployment { newReplicaCount = activeReplicas } lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment) // Only update if necessary. if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount { deployment.Spec.Replicas = newReplicaCount deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(newReplicaCount) _, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err != nil { c.recorder.Eventf(config, "DeploymentScaleFailed", "Failed to scale deployment %q from %d to %d: %s", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } // Only report scaling events if we changed the replica count. if oldReplicaCount != newReplicaCount { c.recorder.Eventf(config, "DeploymentScaled", "Scaled deployment %q from %d to %d", deployment.Name, oldReplicaCount, newReplicaCount) } else { glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(&deployment), newReplicaCount) } } } return nil }
// findDetails inspects the given deployment configuration for any failure causes // and returns any found details about it func (c *DeploymentConfigController) findDetails(config *deployapi.DeploymentConfig) (string, *kapi.ReplicationControllerList, bool, error) { // Check if any existing inflight deployments (any non-terminal state). existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(config.Namespace, config.Name) if err != nil { return "", nil, false, fmt.Errorf("couldn't list deployments for deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err) } // check if the latest deployment exists // we'll return after we've dealt with the multiple-active-deployments case latestDeploymentExists, latestDeploymentStatus := deployutil.LatestDeploymentInfo(config, existingDeployments) if latestDeploymentExists && latestDeploymentStatus != deployapi.DeploymentStatusFailed { // If the latest deployment exists and is not failed, clear the dc message return "", existingDeployments, latestDeploymentExists, nil } // Rest of the code will handle non-existing or failed latest deployment causes // TODO: Inspect pod logs in case of failed latest details := "" allDetails := []string{} if config.Details != nil && len(config.Details.Message) > 0 { // Populate details with the previous message so that in case we stumble upon // an unexpected client error, the message won't be overwritten details = config.Details.Message } // Look into image change triggers and find out possible deployment failures such as // missing image stream tags with or without build configurations pointing at them for _, trigger := range config.Triggers { if trigger.Type != deployapi.DeploymentTriggerOnImageChange || trigger.ImageChangeParams == nil || !trigger.ImageChangeParams.Automatic { continue } name := trigger.ImageChangeParams.From.Name tag := trigger.ImageChangeParams.Tag istag := imageapi.JoinImageStreamTag(name, tag) // Check if the image stream tag pointed by the trigger exists if _, err := c.osClient.ImageStreamTags(config.Namespace).Get(name, tag); err != nil { if !errors.IsNotFound(err) { glog.V(2).Infof("Error while trying to get image stream tag %q: %v", istag, err) return details, existingDeployments, latestDeploymentExists, nil } // In case the image stream tag was not found, then it either doesn't exist or doesn't exist yet // (a build configuration output points to it so it's going to be populated at some point in the // future) details = fmt.Sprintf("The image trigger for image stream tag %q will have no effect because image stream tag %q does not exist.", istag, istag) bcList, err := c.osClient.BuildConfigs(kapi.NamespaceAll).List(labels.Everything(), fields.Everything()) if err != nil { glog.V(2).Infof("Error while trying to list build configs: %v", err) return details, existingDeployments, latestDeploymentExists, nil } for _, bc := range bcList.Items { if bc.Spec.Output.To != nil && bc.Spec.Output.To.Kind == "ImageStreamTag" { parts := strings.Split(bc.Spec.Output.To.Name, ":") if len(parts) != 2 { glog.V(2).Infof("Invalid image stream tag: %q", bc.Spec.Output.To.Name) return details, existingDeployments, latestDeploymentExists, nil } if parts[0] == name && parts[1] == tag { details = fmt.Sprintf("The image trigger for image stream tag %q will have no effect because image stream tag %q does not exist.\n\tIf image stream tag %q is expected, check build config %q which produces image stream tag %q.", istag, istag, istag, bc.Name, istag) break } } } // Try to see if the image stream exists, if not then the build will never be able to update the // tag in question if _, err := c.osClient.ImageStreams(config.Namespace).Get(name); err != nil { glog.V(2).Infof("Error while trying to get image stream %q: %v", name, err) if errors.IsNotFound(err) { details = fmt.Sprintf("The image trigger for image stream tag %q will have no effect because image stream %q does not exist.", istag, name) } } } allDetails = append(allDetails, details) } if len(allDetails) > 1 { for i := range allDetails { allDetails[i] = fmt.Sprintf("\t* %s", allDetails[i]) } // Prepend multiple errors warning multipleErrWarning := fmt.Sprintf("Deployment config %q blocked by multiple errors:\n", config.Name) allDetails = append([]string{multipleErrWarning}, allDetails...) } return strings.Join(allDetails, "\n"), existingDeployments, latestDeploymentExists, nil }
// Handle processes config and creates a new deployment if necessary. func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error { // TODO(danmace): Don't use findDetails yet. See: https://github.com/openshift/origin/pull/5530 // Only deploy when the version has advanced past 0. if config.LatestVersion == 0 { glog.V(5).Infof("Waiting for first version of %s", deployutil.LabelForDeploymentConfig(config)) return nil } existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(config.Namespace, config.Name) if err != nil { return err } var inflightDeployment *kapi.ReplicationController for _, deployment := range existingDeployments.Items { deploymentStatus := deployutil.DeploymentStatusFor(&deployment) switch deploymentStatus { case deployapi.DeploymentStatusFailed, deployapi.DeploymentStatusComplete: // Previous deployment in terminal state - can ignore // Ignoring specific deployment states so that any newly introduced // deployment state will not be ignored default: if inflightDeployment == nil { inflightDeployment = &deployment continue } var deploymentForCancellation *kapi.ReplicationController if deployutil.DeploymentVersionFor(inflightDeployment) < deployutil.DeploymentVersionFor(&deployment) { deploymentForCancellation, inflightDeployment = inflightDeployment, &deployment } else { deploymentForCancellation = &deployment } deploymentForCancellation.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue deploymentForCancellation.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists if _, err := c.deploymentClient.updateDeployment(deploymentForCancellation.Namespace, deploymentForCancellation); err != nil { util.HandleError(fmt.Errorf("couldn't cancel deployment %s: %v", deployutil.LabelForDeployment(deploymentForCancellation), err)) } glog.V(4).Infof("Cancelled deployment %s for deployment config %s", deployutil.LabelForDeployment(deploymentForCancellation), deployutil.LabelForDeploymentConfig(config)) } } // if the latest deployment exists then nothing else needs to be done latestDeploymentExists, _ := deployutil.LatestDeploymentInfo(config, existingDeployments) if latestDeploymentExists { return nil } // check to see if there are inflight deployments if inflightDeployment != nil { // raise a transientError so that the deployment config can be re-queued glog.V(4).Infof("Found previous inflight deployment for %s - will requeue", deployutil.LabelForDeploymentConfig(config)) return transientError(fmt.Sprintf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))) } // Try and build a deployment for the config. deployment, err := c.makeDeployment(config) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)) } // Compute the desired replicas for the deployment. Use the last completed // deployment's current replica count, or the config template if there is no // prior completed deployment available. desiredReplicas := config.Template.ControllerTemplate.Replicas if len(existingDeployments.Items) > 0 { sort.Sort(deployutil.ByLatestVersionDesc(existingDeployments.Items)) for _, existing := range existingDeployments.Items { if deployutil.DeploymentStatusFor(&existing) == deployapi.DeploymentStatusComplete { desiredReplicas = existing.Spec.Replicas glog.V(4).Infof("Desired replicas for %s set to %d based on prior completed deployment %s", deployutil.LabelForDeploymentConfig(config), desiredReplicas, existing.Name) break } } } deployment.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(desiredReplicas) // Create the deployment. if _, err := c.deploymentClient.createDeployment(config.Namespace, deployment); err == nil { glog.V(4).Infof("Created deployment for deployment config %s", deployutil.LabelForDeploymentConfig(config)) return nil } else { // If the deployment was already created, just move on. The cache could be stale, or another // process could have already handled this update. if errors.IsAlreadyExists(err) { glog.V(4).Infof("Deployment already exists for deployment config %s", deployutil.LabelForDeploymentConfig(config)) return nil } glog.Warningf("Cannot create latest deployment for deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err) return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err) } }