// findTargetDeployment finds the deployment which is the rollback target by // searching for deployments associated with config. If desiredVersion is >0, // the deployment matching desiredVersion will be returned. If desiredVersion // is <=0, the last completed deployment which is older than the config's // version will be returned. func (o *RollbackOptions) findTargetDeployment(config *deployapi.DeploymentConfig, desiredVersion int64) (*kapi.ReplicationController, error) { // Find deployments for the config sorted by version descending. deployments, err := o.kc.ReplicationControllers(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.ConfigSelector(config.Name)}) if err != nil { return nil, err } sort.Sort(deployutil.ByLatestVersionDesc(deployments.Items)) // Find the target deployment for rollback. If a version was specified, // use the version for a search. Otherwise, use the last completed // deployment. var target *kapi.ReplicationController for _, deployment := range deployments.Items { version := deployutil.DeploymentVersionFor(&deployment) if desiredVersion > 0 { if version == desiredVersion { target = &deployment break } } else { if version < config.Status.LatestVersion && deployutil.IsCompleteDeployment(&deployment) { target = &deployment break } } } if target == nil { return nil, fmt.Errorf("couldn't find deployment for rollback") } return target, nil }
func deploymentReachedCompletion(dc *deployapi.DeploymentConfig, rcs []kapi.ReplicationController, pods []kapi.Pod) (bool, error) { if len(rcs) == 0 { return false, nil } rc := rcs[len(rcs)-1] version := deployutil.DeploymentVersionFor(&rc) if version != dc.Status.LatestVersion { return false, nil } if !deployutil.IsCompleteDeployment(&rc) { return false, nil } cond := deployutil.GetDeploymentCondition(dc.Status, deployapi.DeploymentProgressing) if cond == nil || cond.Reason != deployutil.NewRcAvailableReason { return false, nil } expectedReplicas := dc.Spec.Replicas if dc.Spec.Test { expectedReplicas = 0 } if rc.Spec.Replicas != int32(expectedReplicas) { return false, fmt.Errorf("deployment is complete but doesn't have expected spec replicas: %d %d", rc.Spec.Replicas, expectedReplicas) } if rc.Status.Replicas != int32(expectedReplicas) { e2e.Logf("POSSIBLE_ANOMALY: deployment is complete but doesn't have expected status replicas: %d %d", rc.Status.Replicas, expectedReplicas) return false, nil } e2e.Logf("Latest rollout of dc/%s (rc/%s) is complete.", dc.Name, rc.Name) return true, nil }
// retry resets the status of the latest deployment to New, which will cause // the deployment to be retried. An error is returned if the deployment is not // currently in a failed state. func (o DeployOptions) retry(config *deployapi.DeploymentConfig) error { if config.Spec.Paused { return fmt.Errorf("cannot retry a paused deployment config") } if config.Status.LatestVersion == 0 { return fmt.Errorf("no deployments found for %s/%s", config.Namespace, config.Name) } // TODO: This implies that deploymentconfig.status.latestVersion is always synced. Currently, // that's the case because clients (oc, trigger controllers) are updating the status directly. // Clients should be acting either on spec or on annotations and status updates should be a // responsibility of the main controller. We need to start by unplugging this assumption from // our client tools. deploymentName := deployutil.LatestDeploymentNameForConfig(config) deployment, err := o.kubeClient.ReplicationControllers(config.Namespace).Get(deploymentName) if err != nil { if kerrors.IsNotFound(err) { return fmt.Errorf("unable to find the latest deployment (#%d).\nYou can start a new deployment with 'oc deploy --latest dc/%s'.", config.Status.LatestVersion, config.Name) } return err } if !deployutil.IsFailedDeployment(deployment) { message := fmt.Sprintf("#%d is %s; only failed deployments can be retried.\n", config.Status.LatestVersion, deployutil.DeploymentStatusFor(deployment)) if deployutil.IsCompleteDeployment(deployment) { message += fmt.Sprintf("You can start a new deployment with 'oc deploy --latest dc/%s'.", config.Name) } else { message += fmt.Sprintf("Optionally, you can cancel this deployment with 'oc deploy --cancel dc/%s'.", config.Name) } return fmt.Errorf(message) } // Delete the deployer pod as well as the deployment hooks pods, if any pods, err := o.kubeClient.Pods(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(deploymentName)}) if err != nil { return fmt.Errorf("failed to list deployer/hook pods for deployment #%d: %v", config.Status.LatestVersion, err) } for _, pod := range pods.Items { err := o.kubeClient.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0)) if err != nil { return fmt.Errorf("failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.Status.LatestVersion, err) } } deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew) // clear out the cancellation flag as well as any previous status-reason annotation delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation) delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation) _, err = o.kubeClient.ReplicationControllers(deployment.Namespace).Update(deployment) if err != nil { return err } fmt.Fprintf(o.out, "Retried #%d\n", config.Status.LatestVersion) if o.follow { return o.getLogs(config) } fmt.Fprintf(o.out, "Use '%s logs -f dc/%s' to track its progress.\n", o.baseCommandName, config.Name) return nil }
func describeDeployments(f formatter, dcNode *deploygraph.DeploymentConfigNode, activeDeployment *kubegraph.ReplicationControllerNode, inactiveDeployments []*kubegraph.ReplicationControllerNode, restartFn func(*kubegraph.ReplicationControllerNode) int32, count int) []string { if dcNode == nil { return nil } out := []string{} deploymentsToPrint := append([]*kubegraph.ReplicationControllerNode{}, inactiveDeployments...) if activeDeployment == nil { on, auto := describeDeploymentConfigTriggers(dcNode.DeploymentConfig) if dcNode.DeploymentConfig.Status.LatestVersion == 0 { out = append(out, fmt.Sprintf("deployment #1 waiting %s", on)) } else if auto { out = append(out, fmt.Sprintf("deployment #%d pending %s", dcNode.DeploymentConfig.Status.LatestVersion, on)) } // TODO: detect new image available? } else { deploymentsToPrint = append([]*kubegraph.ReplicationControllerNode{activeDeployment}, inactiveDeployments...) } for i, deployment := range deploymentsToPrint { restartCount := int32(0) if restartFn != nil { restartCount = restartFn(deployment) } out = append(out, describeDeploymentStatus(deployment.ReplicationController, i == 0, dcNode.DeploymentConfig.Spec.Test, restartCount)) switch { case count == -1: if deployutil.IsCompleteDeployment(deployment.ReplicationController) { return out } default: if i+1 >= count { return out } } } return out }
if rq, ok := val.Object.(*kapi.ResourceQuota); ok { used := quota.Mask(rq.Status.Used, expectedResourceNames) if isUsageSynced(used, expectedUsage, expectedIsUpperLimit) { return used, nil } } case <-time.After(endTime.Sub(time.Now())): return nil, wait.ErrWaitTimeout } } return nil, wait.ErrWaitTimeout } // CheckDeploymentCompletedFn returns true if the deployment completed var CheckDeploymentCompletedFn = func(d *kapi.ReplicationController) bool { return deployutil.IsCompleteDeployment(d) } // CheckDeploymentFailedFn returns true if the deployment failed var CheckDeploymentFailedFn = func(d *kapi.ReplicationController) bool { return deployutil.IsFailedDeployment(d) } // GetPodNamesByFilter looks up pods that satisfy the predicate and returns their names. func GetPodNamesByFilter(c kclient.PodInterface, label labels.Selector, predicate func(kapi.Pod) bool) (podNames []string, err error) { podList, err := c.List(kapi.ListOptions{LabelSelector: label}) if err != nil { return nil, err } for _, pod := range podList.Items { if predicate(pod) {
// Get returns a streamer resource with the contents of the deployment log func (r *REST) Get(ctx kapi.Context, name string, opts runtime.Object) (runtime.Object, error) { // Ensure we have a namespace in the context namespace, ok := kapi.NamespaceFrom(ctx) if !ok { return nil, errors.NewBadRequest("namespace parameter required.") } // Validate DeploymentLogOptions deployLogOpts, ok := opts.(*deployapi.DeploymentLogOptions) if !ok { return nil, errors.NewBadRequest("did not get an expected options.") } if errs := validation.ValidateDeploymentLogOptions(deployLogOpts); len(errs) > 0 { return nil, errors.NewInvalid(deployapi.Kind("DeploymentLogOptions"), "", errs) } // Fetch deploymentConfig and check latest version; if 0, there are no deployments // for this config config, err := r.dn.DeploymentConfigs(namespace).Get(name) if err != nil { return nil, errors.NewNotFound(deployapi.Resource("deploymentconfig"), name) } desiredVersion := config.Status.LatestVersion if desiredVersion == 0 { return nil, errors.NewBadRequest(fmt.Sprintf("no deployment exists for deploymentConfig %q", config.Name)) } // Support retrieving logs for older deployments switch { case deployLogOpts.Version == nil: // Latest or previous if deployLogOpts.Previous { desiredVersion-- if desiredVersion < 1 { return nil, errors.NewBadRequest(fmt.Sprintf("no previous deployment exists for deploymentConfig %q", config.Name)) } } case *deployLogOpts.Version <= 0 || *deployLogOpts.Version > config.Status.LatestVersion: // Invalid version return nil, errors.NewBadRequest(fmt.Sprintf("invalid version for deploymentConfig %q: %d", config.Name, *deployLogOpts.Version)) default: desiredVersion = *deployLogOpts.Version } // Get desired deployment targetName := deployutil.DeploymentNameForConfigVersion(config.Name, desiredVersion) target, err := r.waitForExistingDeployment(namespace, targetName) if err != nil { return nil, err } podName := deployutil.DeployerPodNameForDeployment(target.Name) // Check for deployment status; if it is new or pending, we will wait for it. If it is complete, // the deployment completed successfully and the deployer pod will be deleted so we will return a // success message. If it is running or failed, retrieve the log from the deployer pod. status := deployutil.DeploymentStatusFor(target) switch status { case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending: if deployLogOpts.NoWait { glog.V(4).Infof("Deployment %s is in %s state. No logs to retrieve yet.", deployutil.LabelForDeployment(target), status) return &genericrest.LocationStreamer{}, nil } glog.V(4).Infof("Deployment %s is in %s state, waiting for it to start...", deployutil.LabelForDeployment(target), status) if err := deployutil.WaitForRunningDeployerPod(r.pn, target, r.timeout); err != nil { return nil, errors.NewBadRequest(fmt.Sprintf("failed to run deployer pod %s: %v", podName, err)) } latest, ok, err := registry.WaitForRunningDeployment(r.rn, target, r.timeout) if err != nil { return nil, errors.NewBadRequest(fmt.Sprintf("unable to wait for deployment %s to run: %v", deployutil.LabelForDeployment(target), err)) } if !ok { return nil, errors.NewServerTimeout(kapi.Resource("ReplicationController"), "get", 2) } if deployutil.IsCompleteDeployment(latest) { podName, err = r.returnApplicationPodName(target) if err != nil { return nil, err } } case deployapi.DeploymentStatusComplete: podName, err = r.returnApplicationPodName(target) if err != nil { return nil, err } } logOpts := deployapi.DeploymentToPodLogOptions(deployLogOpts) location, transport, err := pod.LogLocation(&podGetter{r.pn}, r.connInfo, ctx, podName, logOpts) if err != nil { return nil, errors.NewBadRequest(err.Error()) } return &genericrest.LocationStreamer{ Location: location, Transport: transport, ContentType: "text/plain", Flush: deployLogOpts.Follow, ResponseChecker: genericrest.NewGenericHttpResponseChecker(kapi.Resource("pod"), podName), }, nil }
func (o RetryOptions) Run() error { allErrs := []error{} mapping, err := o.Mapper.RESTMapping(kapi.Kind("ReplicationController")) if err != nil { return err } for _, info := range o.Infos { config, ok := info.Object.(*deployapi.DeploymentConfig) if !ok { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("expected deployment configuration, got %T", info.Object))) continue } if config.Spec.Paused { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to retry paused deployment config %q", config.Name))) continue } if config.Status.LatestVersion == 0 { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("no rollouts found for %q", config.Name))) continue } latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config) rc, err := o.Clientset.ReplicationControllers(config.Namespace).Get(latestDeploymentName) if err != nil { if kerrors.IsNotFound(err) { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to find the latest rollout (#%d).\nYou can start a new rollout with 'oc rollout latest dc/%s'.", config.Status.LatestVersion, config.Name))) continue } allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to fetch replication controller %q", config.Name))) continue } if !deployutil.IsFailedDeployment(rc) { message := fmt.Sprintf("rollout #%d is %s; only failed deployments can be retried.\n", config.Status.LatestVersion, strings.ToLower(string(deployutil.DeploymentStatusFor(rc)))) if deployutil.IsCompleteDeployment(rc) { message += fmt.Sprintf("You can start a new deployment with 'oc rollout latest dc/%s'.", config.Name) } else { message += fmt.Sprintf("Optionally, you can cancel this deployment with 'oc rollout cancel dc/%s'.", config.Name) } allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, errors.New(message))) continue } // Delete the deployer pod as well as the deployment hooks pods, if any pods, err := o.Clientset.Pods(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(latestDeploymentName)}) if err != nil { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("failed to list deployer/hook pods for deployment #%d: %v", config.Status.LatestVersion, err))) continue } hasError := false for _, pod := range pods.Items { err := o.Clientset.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0)) if err != nil { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.Status.LatestVersion, err))) hasError = true } } if hasError { continue } patches := set.CalculatePatches([]*resource.Info{{Object: rc, Mapping: mapping}}, o.Encoder, func(*resource.Info) (bool, error) { rc.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew) delete(rc.Annotations, deployapi.DeploymentStatusReasonAnnotation) delete(rc.Annotations, deployapi.DeploymentCancelledAnnotation) return true, nil }) if len(patches) == 0 { kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, "already retried") continue } if _, err := o.Clientset.ReplicationControllers(rc.Namespace).Patch(rc.Name, kapi.StrategicMergePatchType, patches[0].Patch); err != nil { allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, err)) continue } kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, fmt.Sprintf("retried rollout #%d", config.Status.LatestVersion)) } return utilerrors.NewAggregate(allErrs) }
// Deploy starts the deployment process for rcName. func (d *Deployer) Deploy(namespace, rcName string) error { // Look up the new deployment. to, err := d.getDeployment(namespace, rcName) if err != nil { return fmt.Errorf("couldn't get deployment %s: %v", rcName, err) } // Decode the config from the deployment. config, err := deployutil.DecodeDeploymentConfig(to, kapi.Codecs.UniversalDecoder()) if err != nil { return fmt.Errorf("couldn't decode deployment config from deployment %s: %v", to.Name, err) } // Get a strategy for the deployment. s, err := d.strategyFor(config) if err != nil { return err } // New deployments must have a desired replica count. desiredReplicas, hasDesired := deployutil.DeploymentDesiredReplicas(to) if !hasDesired { return fmt.Errorf("deployment %s has already run to completion", to.Name) } // Find all deployments for the config. unsortedDeployments, err := d.getDeployments(namespace, config.Name) if err != nil { return fmt.Errorf("couldn't get controllers in namespace %s: %v", namespace, err) } deployments := unsortedDeployments.Items // Sort all the deployments by version. sort.Sort(deployutil.ByLatestVersionDesc(deployments)) // Find any last completed deployment. var from *kapi.ReplicationController for _, candidate := range deployments { if candidate.Name == to.Name { continue } if deployutil.IsCompleteDeployment(&candidate) { from = &candidate break } } if deployutil.DeploymentVersionFor(to) < deployutil.DeploymentVersionFor(from) { return fmt.Errorf("deployment %s is older than %s", to.Name, from.Name) } // Scale down any deployments which aren't the new or last deployment. for _, candidate := range deployments { // Skip the from/to deployments. if candidate.Name == to.Name { continue } if from != nil && candidate.Name == from.Name { continue } // Skip the deployment if it's already scaled down. if candidate.Spec.Replicas == 0 { continue } // Scale the deployment down to zero. retryWaitParams := kubectl.NewRetryParams(1*time.Second, 120*time.Second) if err := d.scaler.Scale(candidate.Namespace, candidate.Name, uint(0), &kubectl.ScalePrecondition{Size: -1, ResourceVersion: ""}, retryWaitParams, retryWaitParams); err != nil { fmt.Fprintf(d.errOut, "error: Couldn't scale down prior deployment %s: %v\n", deployutil.LabelForDeployment(&candidate), err) } else { fmt.Fprintf(d.out, "--> Scaled older deployment %s down\n", candidate.Name) } } if d.until == "start" { return strategy.NewConditionReachedErr("Ready to start deployment") } // Perform the deployment. if err := s.Deploy(from, to, int(desiredReplicas)); err != nil { return err } fmt.Fprintf(d.out, "--> Success\n") return nil }