// findTargetDeployment finds the deployment which is the rollback target by
// searching for deployments associated with config. If desiredVersion is >0,
// the deployment matching desiredVersion will be returned. If desiredVersion
// is <=0, the last completed deployment which is older than the config's
// version will be returned.
func (o *RollbackOptions) findTargetDeployment(config *deployapi.DeploymentConfig, desiredVersion int64) (*kapi.ReplicationController, error) {
// Find deployments for the config sorted by version descending.
deployments, err := o.kc.ReplicationControllers(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.ConfigSelector(config.Name)})
if err != nil {
return nil, err
}
sort.Sort(deployutil.ByLatestVersionDesc(deployments.Items))
// Find the target deployment for rollback. If a version was specified,
// use the version for a search. Otherwise, use the last completed
// deployment.
var target *kapi.ReplicationController
for _, deployment := range deployments.Items {
version := deployutil.DeploymentVersionFor(&deployment)
if desiredVersion > 0 {
if version == desiredVersion {
target = &deployment
break
}
} else {
if version < config.Status.LatestVersion && deployutil.IsCompleteDeployment(&deployment) {
target = &deployment
break
}
}
}
if target == nil {
return nil, fmt.Errorf("couldn't find deployment for rollback")
}
return target, nil
}
func deploymentReachedCompletion(dc *deployapi.DeploymentConfig, rcs []kapi.ReplicationController, pods []kapi.Pod) (bool, error) {
if len(rcs) == 0 {
return false, nil
}
rc := rcs[len(rcs)-1]
version := deployutil.DeploymentVersionFor(&rc)
if version != dc.Status.LatestVersion {
return false, nil
}
if !deployutil.IsCompleteDeployment(&rc) {
return false, nil
}
cond := deployutil.GetDeploymentCondition(dc.Status, deployapi.DeploymentProgressing)
if cond == nil || cond.Reason != deployutil.NewRcAvailableReason {
return false, nil
}
expectedReplicas := dc.Spec.Replicas
if dc.Spec.Test {
expectedReplicas = 0
}
if rc.Spec.Replicas != int32(expectedReplicas) {
return false, fmt.Errorf("deployment is complete but doesn't have expected spec replicas: %d %d", rc.Spec.Replicas, expectedReplicas)
}
if rc.Status.Replicas != int32(expectedReplicas) {
e2e.Logf("POSSIBLE_ANOMALY: deployment is complete but doesn't have expected status replicas: %d %d", rc.Status.Replicas, expectedReplicas)
return false, nil
}
e2e.Logf("Latest rollout of dc/%s (rc/%s) is complete.", dc.Name, rc.Name)
return true, nil
}
// retry resets the status of the latest deployment to New, which will cause
// the deployment to be retried. An error is returned if the deployment is not
// currently in a failed state.
func (o DeployOptions) retry(config *deployapi.DeploymentConfig) error {
if config.Spec.Paused {
return fmt.Errorf("cannot retry a paused deployment config")
}
if config.Status.LatestVersion == 0 {
return fmt.Errorf("no deployments found for %s/%s", config.Namespace, config.Name)
}
// TODO: This implies that deploymentconfig.status.latestVersion is always synced. Currently,
// that's the case because clients (oc, trigger controllers) are updating the status directly.
// Clients should be acting either on spec or on annotations and status updates should be a
// responsibility of the main controller. We need to start by unplugging this assumption from
// our client tools.
deploymentName := deployutil.LatestDeploymentNameForConfig(config)
deployment, err := o.kubeClient.ReplicationControllers(config.Namespace).Get(deploymentName)
if err != nil {
if kerrors.IsNotFound(err) {
return fmt.Errorf("unable to find the latest deployment (#%d).\nYou can start a new deployment with 'oc deploy --latest dc/%s'.", config.Status.LatestVersion, config.Name)
}
return err
}
if !deployutil.IsFailedDeployment(deployment) {
message := fmt.Sprintf("#%d is %s; only failed deployments can be retried.\n", config.Status.LatestVersion, deployutil.DeploymentStatusFor(deployment))
if deployutil.IsCompleteDeployment(deployment) {
message += fmt.Sprintf("You can start a new deployment with 'oc deploy --latest dc/%s'.", config.Name)
} else {
message += fmt.Sprintf("Optionally, you can cancel this deployment with 'oc deploy --cancel dc/%s'.", config.Name)
}
return fmt.Errorf(message)
}
// Delete the deployer pod as well as the deployment hooks pods, if any
pods, err := o.kubeClient.Pods(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(deploymentName)})
if err != nil {
return fmt.Errorf("failed to list deployer/hook pods for deployment #%d: %v", config.Status.LatestVersion, err)
}
for _, pod := range pods.Items {
err := o.kubeClient.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0))
if err != nil {
return fmt.Errorf("failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.Status.LatestVersion, err)
}
}
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew)
// clear out the cancellation flag as well as any previous status-reason annotation
delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation)
delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation)
_, err = o.kubeClient.ReplicationControllers(deployment.Namespace).Update(deployment)
if err != nil {
return err
}
fmt.Fprintf(o.out, "Retried #%d\n", config.Status.LatestVersion)
if o.follow {
return o.getLogs(config)
}
fmt.Fprintf(o.out, "Use '%s logs -f dc/%s' to track its progress.\n", o.baseCommandName, config.Name)
return nil
}
func describeDeployments(f formatter, dcNode *deploygraph.DeploymentConfigNode, activeDeployment *kubegraph.ReplicationControllerNode, inactiveDeployments []*kubegraph.ReplicationControllerNode, restartFn func(*kubegraph.ReplicationControllerNode) int32, count int) []string {
if dcNode == nil {
return nil
}
out := []string{}
deploymentsToPrint := append([]*kubegraph.ReplicationControllerNode{}, inactiveDeployments...)
if activeDeployment == nil {
on, auto := describeDeploymentConfigTriggers(dcNode.DeploymentConfig)
if dcNode.DeploymentConfig.Status.LatestVersion == 0 {
out = append(out, fmt.Sprintf("deployment #1 waiting %s", on))
} else if auto {
out = append(out, fmt.Sprintf("deployment #%d pending %s", dcNode.DeploymentConfig.Status.LatestVersion, on))
}
// TODO: detect new image available?
} else {
deploymentsToPrint = append([]*kubegraph.ReplicationControllerNode{activeDeployment}, inactiveDeployments...)
}
for i, deployment := range deploymentsToPrint {
restartCount := int32(0)
if restartFn != nil {
restartCount = restartFn(deployment)
}
out = append(out, describeDeploymentStatus(deployment.ReplicationController, i == 0, dcNode.DeploymentConfig.Spec.Test, restartCount))
switch {
case count == -1:
if deployutil.IsCompleteDeployment(deployment.ReplicationController) {
return out
}
default:
if i+1 >= count {
return out
}
}
}
return out
}
if rq, ok := val.Object.(*kapi.ResourceQuota); ok {
used := quota.Mask(rq.Status.Used, expectedResourceNames)
if isUsageSynced(used, expectedUsage, expectedIsUpperLimit) {
return used, nil
}
}
case <-time.After(endTime.Sub(time.Now())):
return nil, wait.ErrWaitTimeout
}
}
return nil, wait.ErrWaitTimeout
}
// CheckDeploymentCompletedFn returns true if the deployment completed
var CheckDeploymentCompletedFn = func(d *kapi.ReplicationController) bool {
return deployutil.IsCompleteDeployment(d)
}
// CheckDeploymentFailedFn returns true if the deployment failed
var CheckDeploymentFailedFn = func(d *kapi.ReplicationController) bool {
return deployutil.IsFailedDeployment(d)
}
// GetPodNamesByFilter looks up pods that satisfy the predicate and returns their names.
func GetPodNamesByFilter(c kclient.PodInterface, label labels.Selector, predicate func(kapi.Pod) bool) (podNames []string, err error) {
podList, err := c.List(kapi.ListOptions{LabelSelector: label})
if err != nil {
return nil, err
}
for _, pod := range podList.Items {
if predicate(pod) {
// Get returns a streamer resource with the contents of the deployment log
func (r *REST) Get(ctx kapi.Context, name string, opts runtime.Object) (runtime.Object, error) {
// Ensure we have a namespace in the context
namespace, ok := kapi.NamespaceFrom(ctx)
if !ok {
return nil, errors.NewBadRequest("namespace parameter required.")
}
// Validate DeploymentLogOptions
deployLogOpts, ok := opts.(*deployapi.DeploymentLogOptions)
if !ok {
return nil, errors.NewBadRequest("did not get an expected options.")
}
if errs := validation.ValidateDeploymentLogOptions(deployLogOpts); len(errs) > 0 {
return nil, errors.NewInvalid(deployapi.Kind("DeploymentLogOptions"), "", errs)
}
// Fetch deploymentConfig and check latest version; if 0, there are no deployments
// for this config
config, err := r.dn.DeploymentConfigs(namespace).Get(name)
if err != nil {
return nil, errors.NewNotFound(deployapi.Resource("deploymentconfig"), name)
}
desiredVersion := config.Status.LatestVersion
if desiredVersion == 0 {
return nil, errors.NewBadRequest(fmt.Sprintf("no deployment exists for deploymentConfig %q", config.Name))
}
// Support retrieving logs for older deployments
switch {
case deployLogOpts.Version == nil:
// Latest or previous
if deployLogOpts.Previous {
desiredVersion--
if desiredVersion < 1 {
return nil, errors.NewBadRequest(fmt.Sprintf("no previous deployment exists for deploymentConfig %q", config.Name))
}
}
case *deployLogOpts.Version <= 0 || *deployLogOpts.Version > config.Status.LatestVersion:
// Invalid version
return nil, errors.NewBadRequest(fmt.Sprintf("invalid version for deploymentConfig %q: %d", config.Name, *deployLogOpts.Version))
default:
desiredVersion = *deployLogOpts.Version
}
// Get desired deployment
targetName := deployutil.DeploymentNameForConfigVersion(config.Name, desiredVersion)
target, err := r.waitForExistingDeployment(namespace, targetName)
if err != nil {
return nil, err
}
podName := deployutil.DeployerPodNameForDeployment(target.Name)
// Check for deployment status; if it is new or pending, we will wait for it. If it is complete,
// the deployment completed successfully and the deployer pod will be deleted so we will return a
// success message. If it is running or failed, retrieve the log from the deployer pod.
status := deployutil.DeploymentStatusFor(target)
switch status {
case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending:
if deployLogOpts.NoWait {
glog.V(4).Infof("Deployment %s is in %s state. No logs to retrieve yet.", deployutil.LabelForDeployment(target), status)
return &genericrest.LocationStreamer{}, nil
}
glog.V(4).Infof("Deployment %s is in %s state, waiting for it to start...", deployutil.LabelForDeployment(target), status)
if err := deployutil.WaitForRunningDeployerPod(r.pn, target, r.timeout); err != nil {
return nil, errors.NewBadRequest(fmt.Sprintf("failed to run deployer pod %s: %v", podName, err))
}
latest, ok, err := registry.WaitForRunningDeployment(r.rn, target, r.timeout)
if err != nil {
return nil, errors.NewBadRequest(fmt.Sprintf("unable to wait for deployment %s to run: %v", deployutil.LabelForDeployment(target), err))
}
if !ok {
return nil, errors.NewServerTimeout(kapi.Resource("ReplicationController"), "get", 2)
}
if deployutil.IsCompleteDeployment(latest) {
podName, err = r.returnApplicationPodName(target)
if err != nil {
return nil, err
}
}
case deployapi.DeploymentStatusComplete:
podName, err = r.returnApplicationPodName(target)
if err != nil {
return nil, err
}
}
logOpts := deployapi.DeploymentToPodLogOptions(deployLogOpts)
location, transport, err := pod.LogLocation(&podGetter{r.pn}, r.connInfo, ctx, podName, logOpts)
if err != nil {
return nil, errors.NewBadRequest(err.Error())
}
return &genericrest.LocationStreamer{
Location: location,
Transport: transport,
ContentType: "text/plain",
Flush: deployLogOpts.Follow,
ResponseChecker: genericrest.NewGenericHttpResponseChecker(kapi.Resource("pod"), podName),
}, nil
}
func (o RetryOptions) Run() error {
allErrs := []error{}
mapping, err := o.Mapper.RESTMapping(kapi.Kind("ReplicationController"))
if err != nil {
return err
}
for _, info := range o.Infos {
config, ok := info.Object.(*deployapi.DeploymentConfig)
if !ok {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("expected deployment configuration, got %T", info.Object)))
continue
}
if config.Spec.Paused {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to retry paused deployment config %q", config.Name)))
continue
}
if config.Status.LatestVersion == 0 {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("no rollouts found for %q", config.Name)))
continue
}
latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config)
rc, err := o.Clientset.ReplicationControllers(config.Namespace).Get(latestDeploymentName)
if err != nil {
if kerrors.IsNotFound(err) {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to find the latest rollout (#%d).\nYou can start a new rollout with 'oc rollout latest dc/%s'.", config.Status.LatestVersion, config.Name)))
continue
}
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to fetch replication controller %q", config.Name)))
continue
}
if !deployutil.IsFailedDeployment(rc) {
message := fmt.Sprintf("rollout #%d is %s; only failed deployments can be retried.\n", config.Status.LatestVersion, strings.ToLower(string(deployutil.DeploymentStatusFor(rc))))
if deployutil.IsCompleteDeployment(rc) {
message += fmt.Sprintf("You can start a new deployment with 'oc rollout latest dc/%s'.", config.Name)
} else {
message += fmt.Sprintf("Optionally, you can cancel this deployment with 'oc rollout cancel dc/%s'.", config.Name)
}
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, errors.New(message)))
continue
}
// Delete the deployer pod as well as the deployment hooks pods, if any
pods, err := o.Clientset.Pods(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(latestDeploymentName)})
if err != nil {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("failed to list deployer/hook pods for deployment #%d: %v", config.Status.LatestVersion, err)))
continue
}
hasError := false
for _, pod := range pods.Items {
err := o.Clientset.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0))
if err != nil {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.Status.LatestVersion, err)))
hasError = true
}
}
if hasError {
continue
}
patches := set.CalculatePatches([]*resource.Info{{Object: rc, Mapping: mapping}}, o.Encoder, func(*resource.Info) (bool, error) {
rc.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew)
delete(rc.Annotations, deployapi.DeploymentStatusReasonAnnotation)
delete(rc.Annotations, deployapi.DeploymentCancelledAnnotation)
return true, nil
})
if len(patches) == 0 {
kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, "already retried")
continue
}
if _, err := o.Clientset.ReplicationControllers(rc.Namespace).Patch(rc.Name, kapi.StrategicMergePatchType, patches[0].Patch); err != nil {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, err))
continue
}
kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, fmt.Sprintf("retried rollout #%d", config.Status.LatestVersion))
}
return utilerrors.NewAggregate(allErrs)
}
// Deploy starts the deployment process for rcName.
func (d *Deployer) Deploy(namespace, rcName string) error {
// Look up the new deployment.
to, err := d.getDeployment(namespace, rcName)
if err != nil {
return fmt.Errorf("couldn't get deployment %s: %v", rcName, err)
}
// Decode the config from the deployment.
config, err := deployutil.DecodeDeploymentConfig(to, kapi.Codecs.UniversalDecoder())
if err != nil {
return fmt.Errorf("couldn't decode deployment config from deployment %s: %v", to.Name, err)
}
// Get a strategy for the deployment.
s, err := d.strategyFor(config)
if err != nil {
return err
}
// New deployments must have a desired replica count.
desiredReplicas, hasDesired := deployutil.DeploymentDesiredReplicas(to)
if !hasDesired {
return fmt.Errorf("deployment %s has already run to completion", to.Name)
}
// Find all deployments for the config.
unsortedDeployments, err := d.getDeployments(namespace, config.Name)
if err != nil {
return fmt.Errorf("couldn't get controllers in namespace %s: %v", namespace, err)
}
deployments := unsortedDeployments.Items
// Sort all the deployments by version.
sort.Sort(deployutil.ByLatestVersionDesc(deployments))
// Find any last completed deployment.
var from *kapi.ReplicationController
for _, candidate := range deployments {
if candidate.Name == to.Name {
continue
}
if deployutil.IsCompleteDeployment(&candidate) {
from = &candidate
break
}
}
if deployutil.DeploymentVersionFor(to) < deployutil.DeploymentVersionFor(from) {
return fmt.Errorf("deployment %s is older than %s", to.Name, from.Name)
}
// Scale down any deployments which aren't the new or last deployment.
for _, candidate := range deployments {
// Skip the from/to deployments.
if candidate.Name == to.Name {
continue
}
if from != nil && candidate.Name == from.Name {
continue
}
// Skip the deployment if it's already scaled down.
if candidate.Spec.Replicas == 0 {
continue
}
// Scale the deployment down to zero.
retryWaitParams := kubectl.NewRetryParams(1*time.Second, 120*time.Second)
if err := d.scaler.Scale(candidate.Namespace, candidate.Name, uint(0), &kubectl.ScalePrecondition{Size: -1, ResourceVersion: ""}, retryWaitParams, retryWaitParams); err != nil {
fmt.Fprintf(d.errOut, "error: Couldn't scale down prior deployment %s: %v\n", deployutil.LabelForDeployment(&candidate), err)
} else {
fmt.Fprintf(d.out, "--> Scaled older deployment %s down\n", candidate.Name)
}
}
if d.until == "start" {
return strategy.NewConditionReachedErr("Ready to start deployment")
}
// Perform the deployment.
if err := s.Deploy(from, to, int(desiredReplicas)); err != nil {
return err
}
fmt.Fprintf(d.out, "--> Success\n")
return nil
}
