// cancel cancels any deployment process in progress for config. func (o DeployOptions) cancel(config *deployapi.DeploymentConfig) error { if config.Spec.Paused { return fmt.Errorf("cannot cancel a paused deployment config") } deployments, err := o.kubeClient.ReplicationControllers(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.ConfigSelector(config.Name)}) if err != nil { return err } if len(deployments.Items) == 0 { fmt.Fprintf(o.out, "There have been no deployments for %s/%s\n", config.Namespace, config.Name) return nil } sort.Sort(deployutil.ByLatestVersionDesc(deployments.Items)) failedCancellations := []string{} anyCancelled := false for _, deployment := range deployments.Items { status := deployutil.DeploymentStatusFor(&deployment) switch status { case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning: if deployutil.IsDeploymentCancelled(&deployment) { continue } deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser _, err := o.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err == nil { fmt.Fprintf(o.out, "Cancelled deployment #%d\n", config.Status.LatestVersion) anyCancelled = true } else { fmt.Fprintf(o.out, "Couldn't cancel deployment #%d (status: %s): %v\n", deployutil.DeploymentVersionFor(&deployment), status, err) failedCancellations = append(failedCancellations, strconv.FormatInt(deployutil.DeploymentVersionFor(&deployment), 10)) } } } if len(failedCancellations) > 0 { return fmt.Errorf("couldn't cancel deployment %s", strings.Join(failedCancellations, ", ")) } if !anyCancelled { latest := &deployments.Items[0] maybeCancelling := "" if deployutil.IsDeploymentCancelled(latest) && !deployutil.IsTerminatedDeployment(latest) { maybeCancelling = " (cancelling)" } timeAt := strings.ToLower(units.HumanDuration(time.Now().Sub(latest.CreationTimestamp.Time))) fmt.Fprintf(o.out, "No deployments are in progress (latest deployment #%d %s%s %s ago)\n", deployutil.DeploymentVersionFor(latest), strings.ToLower(string(deployutil.DeploymentStatusFor(latest))), maybeCancelling, timeAt) } return nil }
func describeDeploymentStatus(deploy *kapi.ReplicationController, first, test bool) string { timeAt := strings.ToLower(formatRelativeTime(deploy.CreationTimestamp.Time)) status := deployutil.DeploymentStatusFor(deploy) version := deployutil.DeploymentVersionFor(deploy) maybeCancelling := "" if deployutil.IsDeploymentCancelled(deploy) && !deployutil.IsTerminatedDeployment(deploy) { maybeCancelling = " (cancelling)" } switch status { case deployapi.DeploymentStatusFailed: reason := deployutil.DeploymentStatusReasonFor(deploy) if len(reason) > 0 { reason = fmt.Sprintf(": %s", reason) } // TODO: encode fail time in the rc return fmt.Sprintf("deployment #%d failed %s ago%s%s", version, timeAt, reason, describePodSummaryInline(deploy, false)) case deployapi.DeploymentStatusComplete: // TODO: pod status output if test { return fmt.Sprintf("test deployment #%d deployed %s ago", version, timeAt) } return fmt.Sprintf("deployment #%d deployed %s ago%s", version, timeAt, describePodSummaryInline(deploy, first)) case deployapi.DeploymentStatusRunning: format := "deployment #%d running%s for %s%s" if test { format = "test deployment #%d running%s for %s%s" } return fmt.Sprintf(format, version, maybeCancelling, timeAt, describePodSummaryInline(deploy, false)) default: return fmt.Sprintf("deployment #%d %s%s %s ago%s", version, strings.ToLower(string(status)), maybeCancelling, timeAt, describePodSummaryInline(deploy, false)) } }
func deploymentRunning(dc *deployapi.DeploymentConfig, rcs []kapi.ReplicationController, pods []kapi.Pod) (bool, error) { if len(rcs) == 0 { return false, nil } rc := rcs[len(rcs)-1] version := deployutil.DeploymentVersionFor(&rc) if version != dc.Status.LatestVersion { //e2e.Logf("deployment %s is not the latest version on DC: %d", rc.Name, version) return false, nil } status := rc.Annotations[deployapi.DeploymentStatusAnnotation] switch deployapi.DeploymentStatus(status) { case deployapi.DeploymentStatusFailed: if deployutil.IsDeploymentCancelled(&rc) { return true, nil } reason := deployutil.DeploymentStatusReasonFor(&rc) if reason == "deployer pod no longer exists" { return true, nil } return false, fmt.Errorf("deployment failed: %v", deployutil.DeploymentStatusReasonFor(&rc)) case deployapi.DeploymentStatusRunning, deployapi.DeploymentStatusComplete: return true, nil default: return false, nil } }
func (c *DeploymentController) nextStatus(pod *kapi.Pod, deployment *kapi.ReplicationController) deployapi.DeploymentStatus { switch pod.Status.Phase { case kapi.PodPending: return deployapi.DeploymentStatusPending case kapi.PodRunning: return deployapi.DeploymentStatusRunning case kapi.PodSucceeded: // If the deployment was cancelled just prior to the deployer pod succeeding // then we need to remove the cancel annotations from the complete deployment // and emit an event letting users know their cancellation failed. if deployutil.IsDeploymentCancelled(deployment) { delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation) delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation) c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "FailedCancellation", "Succeeded before cancel recorded") } // Sync the internal replica annotation with the target so that we can // distinguish deployer updates from other scaling events. deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = deployment.Annotations[deployapi.DesiredReplicasAnnotation] delete(deployment.Annotations, deployapi.DesiredReplicasAnnotation) return deployapi.DeploymentStatusComplete case kapi.PodFailed: return deployapi.DeploymentStatusFailed } return deployapi.DeploymentStatusNew }
// cancel cancels any deployment process in progress for config. func (o *DeployOptions) cancel(config *deployapi.DeploymentConfig, out io.Writer) error { deployments, err := o.kubeClient.ReplicationControllers(config.Namespace).List(deployutil.ConfigSelector(config.Name)) if err != nil { return err } if len(deployments.Items) == 0 { fmt.Fprintln(out, "no deployments found to cancel") return nil } failedCancellations := []string{} anyCancelled := false for _, deployment := range deployments.Items { status := deployutil.DeploymentStatusFor(&deployment) switch status { case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning: if deployutil.IsDeploymentCancelled(&deployment) { continue } deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser _, err := o.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err == nil { fmt.Fprintf(out, "cancelled deployment #%d\n", config.LatestVersion) anyCancelled = true } else { fmt.Fprintf(out, "couldn't cancel deployment #%d (status: %s): %v\n", deployutil.DeploymentVersionFor(&deployment), status, err) failedCancellations = append(failedCancellations, strconv.Itoa(deployutil.DeploymentVersionFor(&deployment))) } } } if len(failedCancellations) > 0 { return fmt.Errorf("couldn't cancel deployment %s", strings.Join(failedCancellations, ", ")) } if !anyCancelled { fmt.Fprintln(out, "no active deployments to cancel") } return nil }
// cancel cancels any deployment process in progress for config. func (c *cancelDeploymentCommand) cancel(config *deployapi.DeploymentConfig, out io.Writer) error { deployments, err := c.client.ListDeploymentsForConfig(config.Namespace, config.Name) if err != nil { return err } if len(deployments.Items) == 0 { fmt.Fprintln(out, "no deployments found to cancel") return nil } failedCancellations := []string{} for _, deployment := range deployments.Items { status := deployutil.DeploymentStatusFor(&deployment) switch status { case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning: if deployutil.IsDeploymentCancelled(&deployment) { continue } deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser _, err := c.client.UpdateDeployment(&deployment) if err == nil { fmt.Fprintf(out, "cancelled #%d\n", config.LatestVersion) } else { fmt.Fprintf(out, "couldn't cancel deployment %d (status: %s): %v\n", deployutil.DeploymentVersionFor(&deployment), status, err) failedCancellations = append(failedCancellations, strconv.Itoa(deployutil.DeploymentVersionFor(&deployment))) } default: fmt.Fprintln(out, "no active deployments to cancel") } } if len(failedCancellations) == 0 { return nil } else { return fmt.Errorf("couldn't cancel deployment %s", strings.Join(failedCancellations, ", ")) } }
// Handle processes deployment and either creates a deployer pod or responds // to a terminal deployment status. func (c *DeploymentController) Handle(deployment *kapi.ReplicationController) error { currentStatus := deployutil.DeploymentStatusFor(deployment) nextStatus := currentStatus deploymentScaled := false switch currentStatus { case deployapi.DeploymentStatusNew: // If the deployment has been cancelled, don't create a deployer pod, and // transition to failed immediately. if deployutil.IsDeploymentCancelled(deployment) { nextStatus = deployapi.DeploymentStatusFailed break } // Generate a deployer pod spec. podTemplate, err := c.makeDeployerPod(deployment) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)) } // Create the deployer pod. deploymentPod, err := c.podClient.createPod(deployment.Namespace, podTemplate) if err == nil { deployment.Annotations[deployapi.DeploymentPodAnnotation] = deploymentPod.Name nextStatus = deployapi.DeploymentStatusPending glog.V(4).Infof("Created pod %s for deployment %s", deploymentPod.Name, deployutil.LabelForDeployment(deployment)) break } // Retry on error. if !kerrors.IsAlreadyExists(err) { c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedCreate", "Error creating deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err) return fmt.Errorf("couldn't create deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err) } // If the pod already exists, it's possible that a previous CreatePod // succeeded but the deployment state update failed and now we're re- // entering. Ensure that the pod is the one we created by verifying the // annotation on it, and throw a retryable error. existingPod, err := c.podClient.getPod(deployment.Namespace, deployutil.DeployerPodNameForDeployment(deployment.Name)) if err != nil { c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedCreate", "Error getting existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err) return fmt.Errorf("couldn't fetch existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err) } // Do a stronger check to validate that the existing deployer pod is // actually for this deployment, and if not, fail this deployment. // // TODO: Investigate checking the container image of the running pod and // comparing with the intended deployer pod image. If we do so, we'll need // to ensure that changes to 'unrelated' pods don't result in updates to // the deployment. So, the image check will have to be done in other areas // of the code as well. if deployutil.DeploymentNameFor(existingPod) != deployment.Name { nextStatus = deployapi.DeploymentStatusFailed deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnrelatedDeploymentExists c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedCreate", "Error creating deployer pod for %s since another pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), existingPod.Name) glog.V(2).Infof("Couldn't create deployer pod for %s since an unrelated pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), existingPod.Name) break } // Update to pending relative to the existing validated deployer pod. deployment.Annotations[deployapi.DeploymentPodAnnotation] = existingPod.Name nextStatus = deployapi.DeploymentStatusPending glog.V(4).Infof("Detected existing deployer pod %s for deployment %s", existingPod.Name, deployutil.LabelForDeployment(deployment)) case deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning: // If the deployer pod has vanished, consider the deployment a failure. deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name) if _, err := c.podClient.getPod(deployment.Namespace, deployerPodName); err != nil { if kerrors.IsNotFound(err) { nextStatus = deployapi.DeploymentStatusFailed deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedDeployerPodNoLongerExists c.recorder.Eventf(deployment, kapi.EventTypeWarning, "failed", "Deployer pod %q has gone missing", deployerPodName) glog.V(4).Infof("Failing deployment %q because its deployer pod %q disappeared", deployutil.LabelForDeployment(deployment), deployerPodName) break } else { // We'll try again later on resync. Continue to process cancellations. glog.V(2).Infof("Error getting deployer pod %s for deployment %s: %#v", deployerPodName, deployutil.LabelForDeployment(deployment), err) } } // If the deployment is cancelled, terminate any deployer/hook pods. // NOTE: Do not mark the deployment as Failed just yet. // The deployment will be marked as Failed by the deployer pod controller // when the deployer pod failure state is picked up // Also, it will scale down the failed deployment and scale back up // the last successful completed deployment if deployutil.IsDeploymentCancelled(deployment) { deployerPods, err := c.podClient.getDeployerPodsFor(deployment.Namespace, deployment.Name) if err != nil { return fmt.Errorf("couldn't fetch deployer pods for %s while trying to cancel deployment: %v", deployutil.LabelForDeployment(deployment), err) } glog.V(4).Infof("Cancelling %d deployer pods for deployment %s", len(deployerPods), deployutil.LabelForDeployment(deployment)) zeroDelay := int64(1) for _, deployerPod := range deployerPods { // Set the ActiveDeadlineSeconds on the pod so it's terminated very soon. if deployerPod.Spec.ActiveDeadlineSeconds == nil || *deployerPod.Spec.ActiveDeadlineSeconds != zeroDelay { deployerPod.Spec.ActiveDeadlineSeconds = &zeroDelay if _, err := c.podClient.updatePod(deployerPod.Namespace, &deployerPod); err != nil { c.recorder.Eventf(deployment, kapi.EventTypeWarning, "failedCancellation", "Error cancelling deployer pod %s for deployment %s: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err) return fmt.Errorf("couldn't cancel deployer pod %s for deployment %s: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err) } glog.V(4).Infof("Cancelled deployer pod %s for deployment %s", deployerPod.Name, deployutil.LabelForDeployment(deployment)) } } c.recorder.Eventf(deployment, kapi.EventTypeNormal, "Cancelled", "Cancelled deployment") } case deployapi.DeploymentStatusFailed: // Check for test deployment and ensure the deployment scale matches if config, err := c.decodeConfig(deployment); err == nil && config.Spec.Test { deploymentScaled = deployment.Spec.Replicas != 0 deployment.Spec.Replicas = 0 } case deployapi.DeploymentStatusComplete: // Check for test deployment and ensure the deployment scale matches if config, err := c.decodeConfig(deployment); err == nil && config.Spec.Test { deploymentScaled = deployment.Spec.Replicas != 0 deployment.Spec.Replicas = 0 } // now list any pods in the namespace that have the specified label deployerPods, err := c.podClient.getDeployerPodsFor(deployment.Namespace, deployment.Name) if err != nil { return fmt.Errorf("couldn't fetch deployer pods for %s after successful completion: %v", deployutil.LabelForDeployment(deployment), err) } if len(deployerPods) > 0 { glog.V(4).Infof("Deleting %d deployer pods for deployment %s", len(deployerPods), deployutil.LabelForDeployment(deployment)) } cleanedAll := true for _, deployerPod := range deployerPods { if err := c.podClient.deletePod(deployerPod.Namespace, deployerPod.Name); err != nil { if !kerrors.IsNotFound(err) { // if the pod deletion failed, then log the error and continue // we will try to delete any remaining deployer pods and return an error later utilruntime.HandleError(fmt.Errorf("couldn't delete completed deployer pod %s/%s for deployment %s: %v", deployment.Namespace, deployerPod.Name, deployutil.LabelForDeployment(deployment), err)) cleanedAll = false } // Already deleted } else { glog.V(4).Infof("Deleted completed deployer pod %s/%s for deployment %s", deployment.Namespace, deployerPod.Name, deployutil.LabelForDeployment(deployment)) } } if !cleanedAll { return fmt.Errorf("couldn't clean up all deployer pods for %s", deployutil.LabelForDeployment(deployment)) } } if currentStatus != nextStatus || deploymentScaled { deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) if _, err := c.deploymentClient.updateDeployment(deployment.Namespace, deployment); err != nil { c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedUpdate", "Error updating deployment %s status to %s", deployutil.LabelForDeployment(deployment), nextStatus) return fmt.Errorf("couldn't update deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err) } glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas) } return nil }
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error { // There's nothing to reconcile until the version is nonzero. if config.Status.LatestVersion == 0 { glog.V(5).Infof("Waiting for first version of %q", deployutil.LabelForDeploymentConfig(config)) return c.updateStatus(config) } // Find all deployments owned by the deploymentConfig. sel := deployutil.ConfigSelector(config.Name) existingDeployments, err := c.kubeClient.ReplicationControllers(config.Namespace).List(kapi.ListOptions{LabelSelector: sel}) if err != nil { return err } latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) // If the latest deployment doesn't exist yet, cancel any running // deployments to allow them to be superceded by the new config version. awaitingCancellations := false if !latestIsDeployed { for _, deployment := range existingDeployments.Items { // Skip deployments with an outcome. if deployutil.IsTerminatedDeployment(&deployment) { continue } // Cancel running deployments. awaitingCancellations = true if !deployutil.IsDeploymentCancelled(&deployment) { deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists _, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err) } else { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion) } } } } // Wait for deployment cancellations before reconciling or creating a new // deployment to avoid competing with existing deployment processes. if awaitingCancellations { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion) // raise a transientError so that the deployment config can be re-queued return transientError(fmt.Sprintf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))) } // If the latest deployment already exists, reconcile existing deployments // and return early. if latestIsDeployed { // If the latest deployment is still running, try again later. We don't // want to compete with the deployer. if !deployutil.IsTerminatedDeployment(latestDeployment) { return c.updateStatus(config) } return c.reconcileDeployments(existingDeployments, config) } // No deployments are running and the latest deployment doesn't exist, so // create the new deployment. deployment, err := deployutil.MakeDeployment(config, c.codec) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)) } created, err := c.kubeClient.ReplicationControllers(config.Namespace).Create(deployment) if err != nil { // If the deployment was already created, just move on. The cache could be // stale, or another process could have already handled this update. if errors.IsAlreadyExists(err) { return c.updateStatus(config) } c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err) return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err) } c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion) return c.updateStatus(config) }
// Handle processes deployment and either creates a deployer pod or responds // to a terminal deployment status. Since this controller started using caches, // the provided rc MUST be deep-copied beforehand (see work() in factory.go). func (c *DeploymentController) Handle(deployment *kapi.ReplicationController) error { currentStatus := deployutil.DeploymentStatusFor(deployment) nextStatus := currentStatus deploymentScaled := false deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name) deployer, deployerErr := c.getPod(deployment.Namespace, deployerPodName) switch currentStatus { case deployapi.DeploymentStatusNew: // If the deployment has been cancelled, don't create a deployer pod. // Instead try to delete any deployer pods found and transition the // deployment to Pending so that the deployment config controller // continues to see the deployment as in-flight. Eventually the deletion // of the deployer pod should cause a requeue of this deployment and // then it can be transitioned to Failed by this controller. if deployutil.IsDeploymentCancelled(deployment) { nextStatus = deployapi.DeploymentStatusPending if err := c.cleanupDeployerPods(deployment); err != nil { return err } break } // If the pod already exists, it's possible that a previous CreatePod // succeeded but the deployment state update failed and now we're re- // entering. Ensure that the pod is the one we created by verifying the // annotation on it, and throw a retryable error. if deployerErr != nil && !kerrors.IsNotFound(deployerErr) { return fmt.Errorf("couldn't fetch existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), deployerErr) } if deployerErr == nil && deployer != nil { // Do a stronger check to validate that the existing deployer pod is // actually for this deployment, and if not, fail this deployment. // // TODO: Investigate checking the container image of the running pod and // comparing with the intended deployer pod image. If we do so, we'll need // to ensure that changes to 'unrelated' pods don't result in updates to // the deployment. So, the image check will have to be done in other areas // of the code as well. if deployutil.DeploymentNameFor(deployer) != deployment.Name { nextStatus = deployapi.DeploymentStatusFailed deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnrelatedDeploymentExists c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "FailedCreate", fmt.Sprintf("Error creating deployer pod since another pod with the same name (%q) exists", deployer.Name)) glog.V(2).Infof("Couldn't create deployer pod for %s since an unrelated pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), deployer.Name) } else { // Update to pending relative to the existing validated deployer pod. deployment.Annotations[deployapi.DeploymentPodAnnotation] = deployer.Name nextStatus = deployapi.DeploymentStatusPending glog.V(4).Infof("Detected existing deployer pod %s for deployment %s", deployer.Name, deployutil.LabelForDeployment(deployment)) } // Don't try and re-create the deployer pod. break } if _, ok := deployment.Annotations[deployapi.DeploymentIgnorePodAnnotation]; ok { return nil } // Generate a deployer pod spec. deployerPod, err := c.makeDeployerPod(deployment) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)) } // Create the deployer pod. deploymentPod, err := c.pn.Pods(deployment.Namespace).Create(deployerPod) // Retry on error. if err != nil { return actionableError(fmt.Sprintf("couldn't create deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)) } deployment.Annotations[deployapi.DeploymentPodAnnotation] = deploymentPod.Name nextStatus = deployapi.DeploymentStatusPending glog.V(4).Infof("Created deployer pod %s for deployment %s", deploymentPod.Name, deployutil.LabelForDeployment(deployment)) case deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning: switch { case kerrors.IsNotFound(deployerErr): nextStatus = deployapi.DeploymentStatusFailed // If the deployment is cancelled here then we deleted the deployer in a previous // resync of the deployment. if !deployutil.IsDeploymentCancelled(deployment) { deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedDeployerPodNoLongerExists c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "Failed", fmt.Sprintf("Deployer pod %q has gone missing", deployerPodName)) glog.V(4).Infof("Failing deployment %q because its deployer pod %q disappeared", deployutil.LabelForDeployment(deployment), deployerPodName) } case deployerErr != nil: // We'll try again later on resync. Continue to process cancellations. glog.V(4).Infof("Error getting deployer pod %s for deployment %s: %v", deployerPodName, deployutil.LabelForDeployment(deployment), deployerErr) default: /* err == nil */ // If the deployment has been cancelled, delete any deployer pods // found and transition the deployment to Pending so that the // deployment config controller continues to see the deployment // as in-flight. Eventually the deletion of the deployer pod should // cause a requeue of this deployment and then it can be transitioned // to Failed by this controller. if deployutil.IsDeploymentCancelled(deployment) { if err := c.cleanupDeployerPods(deployment); err != nil { return err } } } case deployapi.DeploymentStatusFailed: // Check for test deployment and ensure the deployment scale matches if config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec); err == nil && config.Spec.Test { deploymentScaled = deployment.Spec.Replicas != 0 deployment.Spec.Replicas = 0 } // Try to cleanup once more a cancelled deployment in case hook pods // were created just after we issued the first cleanup request. if deployutil.IsDeploymentCancelled(deployment) { if err := c.cleanupDeployerPods(deployment); err != nil { return err } } case deployapi.DeploymentStatusComplete: // Check for test deployment and ensure the deployment scale matches if config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec); err == nil && config.Spec.Test { deploymentScaled = deployment.Spec.Replicas != 0 deployment.Spec.Replicas = 0 } if err := c.cleanupDeployerPods(deployment); err != nil { return err } } if deployutil.CanTransitionPhase(currentStatus, nextStatus) || deploymentScaled { deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) if _, err := c.rn.ReplicationControllers(deployment.Namespace).Update(deployment); err != nil { return fmt.Errorf("couldn't update deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err) } glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas) } return nil }
// Handle implements the loop that processes deployment configs. Since this controller started // using caches, the provided config MUST be deep-copied beforehand (see work() in factory.go). func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error { // There's nothing to reconcile until the version is nonzero. if config.Status.LatestVersion == 0 { return c.updateStatus(config, []kapi.ReplicationController{}) } // Find all deployments owned by the deployment config. selector := deployutil.ConfigSelector(config.Name) existingDeployments, err := c.rcStore.ReplicationControllers(config.Namespace).List(selector) if err != nil { return err } // In case the deployment config has been marked for deletion, merely update its status with // the latest available information. Some deletions make take some time to complete so there // is value in doing this. if config.DeletionTimestamp != nil { return c.updateStatus(config, existingDeployments) } latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) // If the latest deployment doesn't exist yet, cancel any running // deployments to allow them to be superceded by the new config version. awaitingCancellations := false if !latestIsDeployed { for i := range existingDeployments { deployment := existingDeployments[i] // Skip deployments with an outcome. if deployutil.IsTerminatedDeployment(&deployment) { continue } // Cancel running deployments. awaitingCancellations = true if !deployutil.IsDeploymentCancelled(&deployment) { // Retry faster on conflicts var updatedDeployment *kapi.ReplicationController if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error { rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name) if kapierrors.IsNotFound(err) { return nil } if err != nil { return err } copied, err := deployutil.DeploymentDeepCopy(rc) if err != nil { return err } copied.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue copied.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists updatedDeployment, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied) return err }); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err) } else { if updatedDeployment != nil { // replace the current deployment with the updated copy so that a future update has a chance at working existingDeployments[i] = *updatedDeployment c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion) } } } } } // Wait for deployment cancellations before reconciling or creating a new // deployment to avoid competing with existing deployment processes. if awaitingCancellations { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion) return fmt.Errorf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config)) } // If the latest deployment already exists, reconcile existing deployments // and return early. if latestIsDeployed { // If the latest deployment is still running, try again later. We don't // want to compete with the deployer. if !deployutil.IsTerminatedDeployment(latestDeployment) { return c.updateStatus(config, existingDeployments) } return c.reconcileDeployments(existingDeployments, config) } // If the config is paused we shouldn't create new deployments for it. if config.Spec.Paused { // in order for revision history limit cleanup to work for paused // deployments, we need to trigger it here if err := c.cleanupOldDeployments(existingDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err) } return c.updateStatus(config, existingDeployments) } // No deployments are running and the latest deployment doesn't exist, so // create the new deployment. deployment, err := deployutil.MakeDeployment(config, c.codec) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)) } created, err := c.rn.ReplicationControllers(config.Namespace).Create(deployment) if err != nil { // If the deployment was already created, just move on. The cache could be // stale, or another process could have already handled this update. if kapierrors.IsAlreadyExists(err) { return c.updateStatus(config, existingDeployments) } c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err) return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err) } c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion) // As we've just created a new deployment, we need to make sure to clean // up old deployments if we have reached our deployment history quota existingDeployments = append(existingDeployments, *created) if err := c.cleanupOldDeployments(existingDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err) } return c.updateStatus(config, existingDeployments) }
// TestCmdDeploy_retryOk ensures that a failed deployment can be retried. func TestCmdDeploy_retryOk(t *testing.T) { deletedPods := []string{} config := deploytest.OkDeploymentConfig(1) var updatedDeployment *kapi.ReplicationController existingDeployment := deploymentFor(config, deployapi.DeploymentStatusFailed) existingDeployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue existingDeployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser mkpod := func(name string) kapi.Pod { return kapi.Pod{ ObjectMeta: kapi.ObjectMeta{ Name: name, Labels: map[string]string{ deployapi.DeployerPodForDeploymentLabel: existingDeployment.Name, }, }, } } existingDeployerPods := []kapi.Pod{ mkpod("prehook"), mkpod("posthook"), mkpod("deployerpod"), } kubeClient := &ktc.Fake{} kubeClient.AddReactor("get", "replicationcontrollers", func(action ktc.Action) (handled bool, ret runtime.Object, err error) { return true, existingDeployment, nil }) kubeClient.AddReactor("update", "replicationcontrollers", func(action ktc.Action) (handled bool, ret runtime.Object, err error) { updatedDeployment = action.(ktc.UpdateAction).GetObject().(*kapi.ReplicationController) return true, updatedDeployment, nil }) kubeClient.AddReactor("list", "pods", func(action ktc.Action) (handled bool, ret runtime.Object, err error) { return true, &kapi.PodList{Items: existingDeployerPods}, nil }) kubeClient.AddReactor("delete", "pods", func(action ktc.Action) (handled bool, ret runtime.Object, err error) { deletedPods = append(deletedPods, action.(ktc.DeleteAction).GetName()) return true, nil, nil }) o := &DeployOptions{kubeClient: kubeClient} err := o.retry(config, ioutil.Discard) if err != nil { t.Fatalf("unexpected error: %v", err) } if updatedDeployment == nil { t.Fatalf("expected updated config") } if deployutil.IsDeploymentCancelled(updatedDeployment) { t.Fatalf("deployment should not have the cancelled flag set anymore") } if deployutil.DeploymentStatusReasonFor(updatedDeployment) != "" { t.Fatalf("deployment status reason should be empty") } sort.Strings(deletedPods) expectedDeletions := []string{"deployerpod", "posthook", "prehook"} if e, a := expectedDeletions, deletedPods; !reflect.DeepEqual(e, a) { t.Fatalf("Not all deployer pods for the failed deployment were deleted.\nEXPECTED: %v\nACTUAL: %v", e, a) } if e, a := deployapi.DeploymentStatusNew, deployutil.DeploymentStatusFor(updatedDeployment); e != a { t.Fatalf("expected deployment status %s, got %s", e, a) } }
// TestCmdDeploy_retryOk ensures that a failed deployment can be retried. func TestCmdDeploy_retryOk(t *testing.T) { deletedPods := []string{} config := deploytest.OkDeploymentConfig(1) var updatedDeployment *kapi.ReplicationController existingDeployment := deploymentFor(config, deployapi.DeploymentStatusFailed) existingDeployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue existingDeployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser existingDeployerPods := []kapi.Pod{ {ObjectMeta: kapi.ObjectMeta{Name: "prehook"}}, {ObjectMeta: kapi.ObjectMeta{Name: "posthook"}}, {ObjectMeta: kapi.ObjectMeta{Name: "deployerpod"}}, } kubeClient := &ktc.Fake{} kubeClient.ReactFn = func(action ktc.Action) (runtime.Object, error) { switch a := action.(type) { case ktc.GetActionImpl: return existingDeployment, nil case ktc.UpdateActionImpl: updatedDeployment = a.GetObject().(*kapi.ReplicationController) return updatedDeployment, nil case ktc.ListActionImpl: return &kapi.PodList{Items: existingDeployerPods}, nil case ktc.DeleteActionImpl: deletedPods = append(deletedPods, a.GetName()) return nil, nil } t.Fatalf("unexpected action: %+v", action) return nil, nil } o := &DeployOptions{kubeClient: kubeClient} err := o.retry(config, ioutil.Discard) if err != nil { t.Fatalf("unexpected error: %v", err) } if updatedDeployment == nil { t.Fatalf("expected updated config") } if deployutil.IsDeploymentCancelled(updatedDeployment) { t.Fatalf("deployment should not have the cancelled flag set anymore") } if deployutil.DeploymentStatusReasonFor(updatedDeployment) != "" { t.Fatalf("deployment status reason should be empty") } sort.Strings(deletedPods) expectedDeletions := []string{"deployerpod", "posthook", "prehook"} if e, a := expectedDeletions, deletedPods; !reflect.DeepEqual(e, a) { t.Fatalf("Not all deployer pods for the failed deployment were deleted.\nEXPECTED: %v\nACTUAL: %v", e, a) } if e, a := deployapi.DeploymentStatusNew, deployutil.DeploymentStatusFor(updatedDeployment); e != a { t.Fatalf("expected deployment status %s, got %s", e, a) } }
// Handle syncs pod's status with any associated deployment. func (c *DeployerPodController) Handle(pod *kapi.Pod) error { // Find the deployment associated with the deployer pod. deploymentName := deployutil.DeploymentNameFor(pod) if len(deploymentName) == 0 { return nil } // Reject updates to anything but the main deployer pod // TODO: Find a way to filter this on the watch side. if pod.Name != deployutil.DeployerPodNameForDeployment(deploymentName) { return nil } deployment := &kapi.ReplicationController{ObjectMeta: kapi.ObjectMeta{Namespace: pod.Namespace, Name: deploymentName}} cached, exists, err := c.store.Get(deployment) if err == nil && exists { // Try to use the cache first. Trust hits and return them. deployment = cached.(*kapi.ReplicationController) } else { // Double-check with the master for cache misses/errors, since those // are rare and API calls are expensive but more reliable. deployment, err = c.kClient.ReplicationControllers(pod.Namespace).Get(deploymentName) } // If the deployment for this pod has disappeared, we should clean up this // and any other deployer pods, then bail out. if err != nil { // Some retrieval error occurred. Retry. if !kerrors.IsNotFound(err) { return fmt.Errorf("couldn't get deployment %s/%s which owns deployer pod %s/%s", pod.Namespace, deploymentName, pod.Name, pod.Namespace) } // Find all the deployer pods for the deployment (including this one). opts := kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(deploymentName)} deployers, err := c.kClient.Pods(pod.Namespace).List(opts) if err != nil { // Retry. return fmt.Errorf("couldn't get deployer pods for %s: %v", deployutil.LabelForDeployment(deployment), err) } // Delete all deployers. for _, deployer := range deployers.Items { err := c.kClient.Pods(deployer.Namespace).Delete(deployer.Name, kapi.NewDeleteOptions(0)) if err != nil { if !kerrors.IsNotFound(err) { // TODO: Should this fire an event? glog.V(2).Infof("Couldn't delete orphaned deployer pod %s/%s: %v", deployer.Namespace, deployer.Name, err) } } else { // TODO: Should this fire an event? glog.V(2).Infof("Deleted orphaned deployer pod %s/%s", deployer.Namespace, deployer.Name) } } return nil } currentStatus := deployutil.DeploymentStatusFor(deployment) nextStatus := currentStatus switch pod.Status.Phase { case kapi.PodRunning: nextStatus = deployapi.DeploymentStatusRunning case kapi.PodSucceeded: nextStatus = deployapi.DeploymentStatusComplete config, decodeErr := c.decodeConfig(deployment) // If the deployment was cancelled just prior to the deployer pod succeeding // then we need to remove the cancel annotations from the complete deployment // and emit an event letting users know their cancellation failed. if deployutil.IsDeploymentCancelled(deployment) { delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation) delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation) if decodeErr == nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "FailedCancellation", "Deployment %q succeeded before cancel recorded", deployutil.LabelForDeployment(deployment)) } else { c.recorder.Event(deployment, kapi.EventTypeWarning, "FailedCancellation", "Succeeded before cancel recorded") } } // Sync the internal replica annotation with the target so that we can // distinguish deployer updates from other scaling events. deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = deployment.Annotations[deployapi.DesiredReplicasAnnotation] if nextStatus == deployapi.DeploymentStatusComplete { delete(deployment.Annotations, deployapi.DesiredReplicasAnnotation) } // reset the size of any test container, since we are the ones updating the RC if decodeErr == nil && config.Spec.Test { deployment.Spec.Replicas = 0 } case kapi.PodFailed: nextStatus = deployapi.DeploymentStatusFailed // reset the size of any test container, since we are the ones updating the RC if config, err := c.decodeConfig(deployment); err == nil && config.Spec.Test { deployment.Spec.Replicas = 0 } } if deployutil.CanTransitionPhase(currentStatus, nextStatus) { deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) if _, err := c.kClient.ReplicationControllers(deployment.Namespace).Update(deployment); err != nil { if kerrors.IsNotFound(err) { return nil } return fmt.Errorf("couldn't update Deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err) } glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas) // If the deployment was canceled, trigger a reconcilation of its deployment config // so that the latest complete deployment can immediately rollback in place of the // canceled deployment. if nextStatus == deployapi.DeploymentStatusFailed && deployutil.IsDeploymentCancelled(deployment) { // If we are unable to get the deployment config, then the deploymentconfig controller will // perform its duties once the resync interval forces the deploymentconfig to be reconciled. name := deployutil.DeploymentConfigNameFor(deployment) kclient.RetryOnConflict(kclient.DefaultRetry, func() error { config, err := c.client.DeploymentConfigs(deployment.Namespace).Get(name) if err != nil { return err } if config.Annotations == nil { config.Annotations = make(map[string]string) } config.Annotations[deployapi.DeploymentCancelledAnnotation] = strconv.Itoa(config.Status.LatestVersion) _, err = c.client.DeploymentConfigs(config.Namespace).Update(config) return err }) } } return nil }
// TestCmdDeploy_retryOk ensures that a failed deployment can be retried. func TestCmdDeploy_retryOk(t *testing.T) { deletedPods := []string{} config := deploytest.OkDeploymentConfig(1) existingDeployment := deploymentFor(config, deployapi.DeploymentStatusFailed) existingDeployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue existingDeployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser existingDeployerPods := []kapi.Pod{ {ObjectMeta: kapi.ObjectMeta{Name: "prehook"}}, {ObjectMeta: kapi.ObjectMeta{Name: "posthook"}}, {ObjectMeta: kapi.ObjectMeta{Name: "deployerpod"}}, } var updatedDeployment *kapi.ReplicationController commandClient := &deployCommandClientImpl{ GetDeploymentFn: func(namespace, name string) (*kapi.ReplicationController, error) { return existingDeployment, nil }, UpdateDeploymentConfigFn: func(config *deployapi.DeploymentConfig) (*deployapi.DeploymentConfig, error) { t.Fatalf("unexpected call to UpdateDeploymentConfig") return nil, nil }, UpdateDeploymentFn: func(deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) { updatedDeployment = deployment return deployment, nil }, ListDeployerPodsForFn: func(namespace, name string) (*kapi.PodList, error) { return &kapi.PodList{Items: existingDeployerPods}, nil }, DeletePodFn: func(pod *kapi.Pod) error { deletedPods = append(deletedPods, pod.Name) return nil }, } c := &retryDeploymentCommand{client: commandClient} err := c.retry(config, ioutil.Discard) if err != nil { t.Fatalf("unexpected error: %v", err) } if updatedDeployment == nil { t.Fatalf("expected updated config") } if deployutil.IsDeploymentCancelled(updatedDeployment) { t.Fatalf("deployment should not have the cancelled flag set anymore") } if deployutil.DeploymentStatusReasonFor(updatedDeployment) != "" { t.Fatalf("deployment status reason should be empty") } sort.Strings(deletedPods) if !reflect.DeepEqual(deletedPods, []string{"deployerpod", "posthook", "prehook"}) { t.Fatalf("Not all deployer pods for the failed deployment were deleted") } if e, a := deployapi.DeploymentStatusNew, deployutil.DeploymentStatusFor(updatedDeployment); e != a { t.Fatalf("expected deployment status %s, got %s", e, a) } }
func (o CancelOptions) Run() error { allErrs := []error{} for _, info := range o.Infos { config, ok := info.Object.(*deployapi.DeploymentConfig) if !ok { allErrs = append(allErrs, kcmdutil.AddSourceToErr("cancelling", info.Source, fmt.Errorf("expected deployment configuration, got %T", info.Object))) } if config.Spec.Paused { allErrs = append(allErrs, kcmdutil.AddSourceToErr("cancelling", info.Source, fmt.Errorf("unable to cancel paused deployment %s/%s", config.Namespace, config.Name))) } mapping, err := o.Mapper.RESTMapping(kapi.Kind("ReplicationController")) if err != nil { return err } mutateFn := func(rc *kapi.ReplicationController) bool { if deployutil.IsDeploymentCancelled(rc) { kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, "already cancelled") return false } patches := set.CalculatePatches([]*resource.Info{{Object: rc, Mapping: mapping}}, o.Encoder, func(*resource.Info) (bool, error) { rc.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue rc.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser return true, nil }) if len(patches) == 0 { kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, "already cancelled") return false } _, err := o.Clientset.ReplicationControllers(rc.Namespace).Patch(rc.Name, kapi.StrategicMergePatchType, patches[0].Patch) if err != nil { allErrs = append(allErrs, kcmdutil.AddSourceToErr("cancelling", info.Source, err)) return false } kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, "cancelling") return true } deployments, cancelled, err := o.forEachControllerInConfig(config.Namespace, config.Name, mutateFn) if err != nil { allErrs = append(allErrs, kcmdutil.AddSourceToErr("cancelling", info.Source, err)) continue } if !cancelled { latest := &deployments[0] maybeCancelling := "" if deployutil.IsDeploymentCancelled(latest) && !deployutil.IsTerminatedDeployment(latest) { maybeCancelling = " (cancelling)" } timeAt := strings.ToLower(units.HumanDuration(time.Now().Sub(latest.CreationTimestamp.Time))) fmt.Fprintf(o.Out, "No rollout is in progress (latest rollout #%d %s%s %s ago)\n", deployutil.DeploymentVersionFor(latest), strings.ToLower(string(deployutil.DeploymentStatusFor(latest))), maybeCancelling, timeAt) } } return utilerrors.NewAggregate(allErrs) }