// retry resets the status of the latest deployment to New, which will cause // the deployment to be retried. An error is returned if the deployment is not // currently in a failed state. func (o DeployOptions) retry(config *deployapi.DeploymentConfig) error { if config.Spec.Paused { return fmt.Errorf("cannot retry a paused deployment config") } if config.Status.LatestVersion == 0 { return fmt.Errorf("no deployments found for %s/%s", config.Namespace, config.Name) } // TODO: This implies that deploymentconfig.status.latestVersion is always synced. Currently, // that's the case because clients (oc, trigger controllers) are updating the status directly. // Clients should be acting either on spec or on annotations and status updates should be a // responsibility of the main controller. We need to start by unplugging this assumption from // our client tools. deploymentName := deployutil.LatestDeploymentNameForConfig(config) deployment, err := o.kubeClient.ReplicationControllers(config.Namespace).Get(deploymentName) if err != nil { if kerrors.IsNotFound(err) { return fmt.Errorf("unable to find the latest deployment (#%d).\nYou can start a new deployment with 'oc deploy --latest dc/%s'.", config.Status.LatestVersion, config.Name) } return err } if !deployutil.IsFailedDeployment(deployment) { message := fmt.Sprintf("#%d is %s; only failed deployments can be retried.\n", config.Status.LatestVersion, deployutil.DeploymentStatusFor(deployment)) if deployutil.IsCompleteDeployment(deployment) { message += fmt.Sprintf("You can start a new deployment with 'oc deploy --latest dc/%s'.", config.Name) } else { message += fmt.Sprintf("Optionally, you can cancel this deployment with 'oc deploy --cancel dc/%s'.", config.Name) } return fmt.Errorf(message) } // Delete the deployer pod as well as the deployment hooks pods, if any pods, err := o.kubeClient.Pods(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(deploymentName)}) if err != nil { return fmt.Errorf("failed to list deployer/hook pods for deployment #%d: %v", config.Status.LatestVersion, err) } for _, pod := range pods.Items { err := o.kubeClient.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0)) if err != nil { return fmt.Errorf("failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.Status.LatestVersion, err) } } deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew) // clear out the cancellation flag as well as any previous status-reason annotation delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation) delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation) _, err = o.kubeClient.ReplicationControllers(deployment.Namespace).Update(deployment) if err != nil { return err } fmt.Fprintf(o.out, "Retried #%d\n", config.Status.LatestVersion) if o.follow { return o.getLogs(config) } fmt.Fprintf(o.out, "Use '%s logs -f dc/%s' to track its progress.\n", o.baseCommandName, config.Name) return nil }
func deploymentFailed(dc *deployapi.DeploymentConfig, rcs []kapi.ReplicationController, _ []kapi.Pod) (bool, error) { if len(rcs) == 0 { return false, nil } rc := rcs[len(rcs)-1] version := deployutil.DeploymentVersionFor(&rc) if version != dc.Status.LatestVersion { return false, nil } return deployutil.IsFailedDeployment(&rc), nil }
func deploymentFailed(dc *deployapi.DeploymentConfig, rcs []kapi.ReplicationController, _ []kapi.Pod) (bool, error) { if len(rcs) == 0 { return false, nil } rc := rcs[len(rcs)-1] version := deployutil.DeploymentVersionFor(&rc) if version != dc.Status.LatestVersion { return false, nil } if !deployutil.IsFailedDeployment(&rc) { return false, nil } cond := deployutil.GetDeploymentCondition(dc.Status, deployapi.DeploymentProgressing) return cond != nil && cond.Reason == deployutil.TimedOutReason, nil }
func (p *deploymentDeleter) DeleteDeployment(deployment *kapi.ReplicationController) error { glog.V(4).Infof("Deleting deployment %q", deployment.Name) // If the deployment is failed we need to remove its deployer pods, too. if deployutil.IsFailedDeployment(deployment) { dpSelector := deployutil.DeployerPodSelector(deployment.Name) deployers, err := p.pods.Pods(deployment.Namespace).List(kapi.ListOptions{LabelSelector: dpSelector}) if err != nil { glog.Warning("Cannot list deployer pods for %q: %v\n", deployment.Name, err) } else { for _, pod := range deployers.Items { if err := p.pods.Pods(pod.Namespace).Delete(pod.Name, nil); err != nil { glog.Warning("Cannot remove deployer pod %q: %v\n", pod.Name, err) } } } } return p.deployments.ReplicationControllers(deployment.Namespace).Delete(deployment.Name, nil) }
} case <-time.After(endTime.Sub(time.Now())): return nil, wait.ErrWaitTimeout } } return nil, wait.ErrWaitTimeout } // CheckDeploymentCompletedFn returns true if the deployment completed var CheckDeploymentCompletedFn = func(d *kapi.ReplicationController) bool { return deployutil.IsCompleteDeployment(d) } // CheckDeploymentFailedFn returns true if the deployment failed var CheckDeploymentFailedFn = func(d *kapi.ReplicationController) bool { return deployutil.IsFailedDeployment(d) } // GetPodNamesByFilter looks up pods that satisfy the predicate and returns their names. func GetPodNamesByFilter(c kclient.PodInterface, label labels.Selector, predicate func(kapi.Pod) bool) (podNames []string, err error) { podList, err := c.List(kapi.ListOptions{LabelSelector: label}) if err != nil { return nil, err } for _, pod := range podList.Items { if predicate(pod) { podNames = append(podNames, pod.Name) } } return podNames, nil }
// Stop scales a replication controller via its deployment configuration down to // zero replicas, waits for all of them to get deleted and then deletes both the // replication controller and its deployment configuration. func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *kapi.DeleteOptions) error { // Pause the deployment configuration to prevent the new deployments from // being triggered. config, err := reaper.pause(namespace, name) configNotFound := kerrors.IsNotFound(err) if err != nil && !configNotFound { return err } var ( isPaused bool legacy bool ) // Determine if the deployment config controller noticed the pause. if !configNotFound { if err := wait.Poll(1*time.Second, 1*time.Minute, func() (bool, error) { dc, err := reaper.oc.DeploymentConfigs(namespace).Get(name) if err != nil { return false, err } isPaused = dc.Spec.Paused return dc.Status.ObservedGeneration >= config.Generation, nil }); err != nil { return err } // If we failed to pause the deployment config, it means we are talking to // old API that does not support pausing. In that case, we delete the // deployment config to stay backward compatible. if !isPaused { if err := reaper.oc.DeploymentConfigs(namespace).Delete(name); err != nil { return err } // Setting this to true avoid deleting the config at the end. legacy = true } } // Clean up deployments related to the config. Even if the deployment // configuration has been deleted, we want to sweep the existing replication // controllers and clean them up. options := kapi.ListOptions{LabelSelector: util.ConfigSelector(name)} rcList, err := reaper.kc.ReplicationControllers(namespace).List(options) if err != nil { return err } rcReaper, err := kubectl.ReaperFor(kapi.Kind("ReplicationController"), reaper.kc) if err != nil { return err } // If there is neither a config nor any deployments, nor any deployer pods, we can return NotFound. deployments := rcList.Items if configNotFound && len(deployments) == 0 { return kerrors.NewNotFound(kapi.Resource("deploymentconfig"), name) } for _, rc := range deployments { if err = rcReaper.Stop(rc.Namespace, rc.Name, timeout, gracePeriod); err != nil { // Better not error out here... glog.Infof("Cannot delete ReplicationController %s/%s for deployment config %s/%s: %v", rc.Namespace, rc.Name, namespace, name, err) } // Only remove deployer pods when the deployment was failed. For completed // deployment the pods should be already deleted. if !util.IsFailedDeployment(&rc) { continue } // Delete all deployer and hook pods options = kapi.ListOptions{LabelSelector: util.DeployerPodSelector(rc.Name)} podList, err := reaper.kc.Pods(rc.Namespace).List(options) if err != nil { return err } for _, pod := range podList.Items { err := reaper.kc.Pods(pod.Namespace).Delete(pod.Name, gracePeriod) if err != nil { // Better not error out here... glog.Infof("Cannot delete lifecycle Pod %s/%s for deployment config %s/%s: %v", pod.Namespace, pod.Name, namespace, name, err) } } } // Nothing to delete or we already deleted the deployment config because we // failed to pause. if configNotFound || legacy { return nil } return reaper.oc.DeploymentConfigs(namespace).Delete(name) }
func (o RetryOptions) Run() error { allErrs := []error{} mapping, err := o.Mapper.RESTMapping(kapi.Kind("ReplicationController")) if err != nil { return err } for _, info := range o.Infos { config, ok := info.Object.(*deployapi.DeploymentConfig) if !ok { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("expected deployment configuration, got %T", info.Object))) continue } if config.Spec.Paused { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to retry paused deployment config %q", config.Name))) continue } if config.Status.LatestVersion == 0 { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("no rollouts found for %q", config.Name))) continue } latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config) rc, err := o.Clientset.ReplicationControllers(config.Namespace).Get(latestDeploymentName) if err != nil { if kerrors.IsNotFound(err) { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to find the latest rollout (#%d).\nYou can start a new rollout with 'oc rollout latest dc/%s'.", config.Status.LatestVersion, config.Name))) continue } allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to fetch replication controller %q", config.Name))) continue } if !deployutil.IsFailedDeployment(rc) { message := fmt.Sprintf("rollout #%d is %s; only failed deployments can be retried.\n", config.Status.LatestVersion, strings.ToLower(string(deployutil.DeploymentStatusFor(rc)))) if deployutil.IsCompleteDeployment(rc) { message += fmt.Sprintf("You can start a new deployment with 'oc rollout latest dc/%s'.", config.Name) } else { message += fmt.Sprintf("Optionally, you can cancel this deployment with 'oc rollout cancel dc/%s'.", config.Name) } allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, errors.New(message))) continue } // Delete the deployer pod as well as the deployment hooks pods, if any pods, err := o.Clientset.Pods(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(latestDeploymentName)}) if err != nil { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("failed to list deployer/hook pods for deployment #%d: %v", config.Status.LatestVersion, err))) continue } hasError := false for _, pod := range pods.Items { err := o.Clientset.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0)) if err != nil { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.Status.LatestVersion, err))) hasError = true } } if hasError { continue } patches := set.CalculatePatches([]*resource.Info{{Object: rc, Mapping: mapping}}, o.Encoder, func(*resource.Info) (bool, error) { rc.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew) delete(rc.Annotations, deployapi.DeploymentStatusReasonAnnotation) delete(rc.Annotations, deployapi.DeploymentCancelledAnnotation) return true, nil }) if len(patches) == 0 { kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, "already retried") continue } if _, err := o.Clientset.ReplicationControllers(rc.Namespace).Patch(rc.Name, kapi.StrategicMergePatchType, patches[0].Patch); err != nil { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, err)) continue } kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, fmt.Sprintf("retried rollout #%d", config.Status.LatestVersion)) } return utilerrors.NewAggregate(allErrs) }
// Stop scales a replication controller via its deployment configuration down to // zero replicas, waits for all of them to get deleted and then deletes both the // replication controller and its deployment configuration. func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *kapi.DeleteOptions) error { // If the config is already deleted, it may still have associated // deployments which didn't get cleaned up during prior calls to Stop. If // the config can't be found, still make an attempt to clean up the // deployments. // // It's important to delete the config first to avoid an undesirable side // effect which can cause the deployment to be re-triggered upon the // config's deletion. See https://github.com/openshift/origin/issues/2721 // for more details. err := reaper.oc.DeploymentConfigs(namespace).Delete(name) configNotFound := kerrors.IsNotFound(err) if err != nil && !configNotFound { return err } // Clean up deployments related to the config. options := kapi.ListOptions{LabelSelector: util.ConfigSelector(name)} rcList, err := reaper.kc.ReplicationControllers(namespace).List(options) if err != nil { return err } rcReaper, err := kubectl.ReaperFor(kapi.Kind("ReplicationController"), reaper.kc) if err != nil { return err } // If there is neither a config nor any deployments, nor any deployer pods, we can return NotFound. deployments := rcList.Items if configNotFound && len(deployments) == 0 { return kerrors.NewNotFound(kapi.Resource("deploymentconfig"), name) } for _, rc := range deployments { if err = rcReaper.Stop(rc.Namespace, rc.Name, timeout, gracePeriod); err != nil { // Better not error out here... glog.Infof("Cannot delete ReplicationController %s/%s: %v", rc.Namespace, rc.Name, err) } // Only remove deployer pods when the deployment was failed. For completed // deployment the pods should be already deleted. if !util.IsFailedDeployment(&rc) { continue } // Delete all deployer and hook pods options = kapi.ListOptions{LabelSelector: util.DeployerPodSelector(rc.Name)} podList, err := reaper.kc.Pods(rc.Namespace).List(options) if err != nil { return err } for _, pod := range podList.Items { err := reaper.kc.Pods(pod.Namespace).Delete(pod.Name, gracePeriod) if err != nil { // Better not error out here... glog.Infof("Cannot delete Pod %s/%s: %v", pod.Namespace, pod.Name, err) } } } return nil }
// Handle processes deployment and either creates a deployer pod or responds // to a terminal deployment status. Since this controller started using caches, // the provided rc MUST be deep-copied beforehand (see work() in factory.go). func (c *DeploymentController) Handle(deployment *kapi.ReplicationController) error { // Copy all the annotations from the deployment. updatedAnnotations := make(map[string]string) for key, value := range deployment.Annotations { updatedAnnotations[key] = value } currentStatus := deployutil.DeploymentStatusFor(deployment) nextStatus := currentStatus deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name) deployer, deployerErr := c.podStore.Pods(deployment.Namespace).Get(deployerPodName) if deployerErr == nil { nextStatus = c.nextStatus(deployer, deployment, updatedAnnotations) } switch currentStatus { case deployapi.DeploymentStatusNew: // If the deployment has been cancelled, don't create a deployer pod. // Instead try to delete any deployer pods found and transition the // deployment to Pending so that the deployment config controller // continues to see the deployment as in-flight. Eventually the deletion // of the deployer pod should cause a requeue of this deployment and // then it can be transitioned to Failed by this controller. if deployutil.IsDeploymentCancelled(deployment) { nextStatus = deployapi.DeploymentStatusPending if err := c.cleanupDeployerPods(deployment); err != nil { return err } break } switch { case kerrors.IsNotFound(deployerErr): if _, ok := deployment.Annotations[deployapi.DeploymentIgnorePodAnnotation]; ok { return nil } // Generate a deployer pod spec. deployerPod, err := c.makeDeployerPod(deployment) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)) } // Create the deployer pod. deploymentPod, err := c.pn.Pods(deployment.Namespace).Create(deployerPod) // Retry on error. if err != nil { return actionableError(fmt.Sprintf("couldn't create deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)) } updatedAnnotations[deployapi.DeploymentPodAnnotation] = deploymentPod.Name nextStatus = deployapi.DeploymentStatusPending glog.V(4).Infof("Created deployer pod %s for deployment %s", deploymentPod.Name, deployutil.LabelForDeployment(deployment)) case deployerErr != nil: // If the pod already exists, it's possible that a previous CreatePod // succeeded but the deployment state update failed and now we're re- // entering. Ensure that the pod is the one we created by verifying the // annotation on it, and throw a retryable error. return fmt.Errorf("couldn't fetch existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), deployerErr) default: /* deployerErr == nil */ // Do a stronger check to validate that the existing deployer pod is // actually for this deployment, and if not, fail this deployment. // // TODO: Investigate checking the container image of the running pod and // comparing with the intended deployer pod image. If we do so, we'll need // to ensure that changes to 'unrelated' pods don't result in updates to // the deployment. So, the image check will have to be done in other areas // of the code as well. if deployutil.DeploymentNameFor(deployer) != deployment.Name { nextStatus = deployapi.DeploymentStatusFailed updatedAnnotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnrelatedDeploymentExists c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "FailedCreate", fmt.Sprintf("Error creating deployer pod since another pod with the same name (%q) exists", deployer.Name)) glog.V(2).Infof("Couldn't create deployer pod for %s since an unrelated pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), deployer.Name) } else { // Update to pending or to the appropriate status relative to the existing validated deployer pod. updatedAnnotations[deployapi.DeploymentPodAnnotation] = deployer.Name nextStatus = nextStatusComp(nextStatus, deployapi.DeploymentStatusPending) glog.V(4).Infof("Detected existing deployer pod %s for deployment %s", deployer.Name, deployutil.LabelForDeployment(deployment)) } } case deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning: switch { case kerrors.IsNotFound(deployerErr): nextStatus = deployapi.DeploymentStatusFailed // If the deployment is cancelled here then we deleted the deployer in a previous // resync of the deployment. if !deployutil.IsDeploymentCancelled(deployment) { updatedAnnotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedDeployerPodNoLongerExists c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "Failed", fmt.Sprintf("Deployer pod %q has gone missing", deployerPodName)) deployerErr = fmt.Errorf("Failing deployment %q because its deployer pod %q disappeared", deployutil.LabelForDeployment(deployment), deployerPodName) utilruntime.HandleError(deployerErr) } case deployerErr != nil: // We'll try again later on resync. Continue to process cancellations. deployerErr = fmt.Errorf("Error getting deployer pod %q for deployment %q: %v", deployerPodName, deployutil.LabelForDeployment(deployment), deployerErr) utilruntime.HandleError(deployerErr) default: /* err == nil */ // If the deployment has been cancelled, delete any deployer pods // found. Eventually the deletion of the deployer pod should cause // a requeue of this deployment and then it can be transitioned to // Failed. if deployutil.IsDeploymentCancelled(deployment) { if err := c.cleanupDeployerPods(deployment); err != nil { return err } } } case deployapi.DeploymentStatusFailed: // Try to cleanup once more a cancelled deployment in case hook pods // were created just after we issued the first cleanup request. if deployutil.IsDeploymentCancelled(deployment) { if err := c.cleanupDeployerPods(deployment); err != nil { return err } } case deployapi.DeploymentStatusComplete: if err := c.cleanupDeployerPods(deployment); err != nil { return err } } // Update only if we need to transition to a new phase. if deployutil.CanTransitionPhase(currentStatus, nextStatus) { deployment, err := deployutil.DeploymentDeepCopy(deployment) if err != nil { return err } updatedAnnotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) deployment.Annotations = updatedAnnotations // if we are going to transition to failed or complete and scale is non-zero, we'll check one more // time to see if we are a test deployment to guarantee that we maintain the test invariant. if deployment.Spec.Replicas != 0 && deployutil.IsTerminatedDeployment(deployment) { if config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec); err == nil && config.Spec.Test { deployment.Spec.Replicas = 0 } } if _, err := c.rn.ReplicationControllers(deployment.Namespace).Update(deployment); err != nil { return fmt.Errorf("couldn't update deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err) } glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas) if deployutil.IsDeploymentCancelled(deployment) && deployutil.IsFailedDeployment(deployment) { c.emitDeploymentEvent(deployment, kapi.EventTypeNormal, "DeploymentCancelled", fmt.Sprintf("Deployment %q cancelled", deployutil.LabelForDeployment(deployment))) } } return nil }