// retry resets the status of the latest deployment to New, which will cause
// the deployment to be retried. An error is returned if the deployment is not
// currently in a failed state.
func (o DeployOptions) retry(config *deployapi.DeploymentConfig) error {
if config.Spec.Paused {
return fmt.Errorf("cannot retry a paused deployment config")
}
if config.Status.LatestVersion == 0 {
return fmt.Errorf("no deployments found for %s/%s", config.Namespace, config.Name)
}
// TODO: This implies that deploymentconfig.status.latestVersion is always synced. Currently,
// that's the case because clients (oc, trigger controllers) are updating the status directly.
// Clients should be acting either on spec or on annotations and status updates should be a
// responsibility of the main controller. We need to start by unplugging this assumption from
// our client tools.
deploymentName := deployutil.LatestDeploymentNameForConfig(config)
deployment, err := o.kubeClient.ReplicationControllers(config.Namespace).Get(deploymentName)
if err != nil {
if kerrors.IsNotFound(err) {
return fmt.Errorf("unable to find the latest deployment (#%d).\nYou can start a new deployment with 'oc deploy --latest dc/%s'.", config.Status.LatestVersion, config.Name)
}
return err
}
if !deployutil.IsFailedDeployment(deployment) {
message := fmt.Sprintf("#%d is %s; only failed deployments can be retried.\n", config.Status.LatestVersion, deployutil.DeploymentStatusFor(deployment))
if deployutil.IsCompleteDeployment(deployment) {
message += fmt.Sprintf("You can start a new deployment with 'oc deploy --latest dc/%s'.", config.Name)
} else {
message += fmt.Sprintf("Optionally, you can cancel this deployment with 'oc deploy --cancel dc/%s'.", config.Name)
}
return fmt.Errorf(message)
}
// Delete the deployer pod as well as the deployment hooks pods, if any
pods, err := o.kubeClient.Pods(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(deploymentName)})
if err != nil {
return fmt.Errorf("failed to list deployer/hook pods for deployment #%d: %v", config.Status.LatestVersion, err)
}
for _, pod := range pods.Items {
err := o.kubeClient.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0))
if err != nil {
return fmt.Errorf("failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.Status.LatestVersion, err)
}
}
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew)
// clear out the cancellation flag as well as any previous status-reason annotation
delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation)
delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation)
_, err = o.kubeClient.ReplicationControllers(deployment.Namespace).Update(deployment)
if err != nil {
return err
}
fmt.Fprintf(o.out, "Retried #%d\n", config.Status.LatestVersion)
if o.follow {
return o.getLogs(config)
}
fmt.Fprintf(o.out, "Use '%s logs -f dc/%s' to track its progress.\n", o.baseCommandName, config.Name)
return nil
}
func deploymentFailed(dc *deployapi.DeploymentConfig, rcs []kapi.ReplicationController, _ []kapi.Pod) (bool, error) {
if len(rcs) == 0 {
return false, nil
}
rc := rcs[len(rcs)-1]
version := deployutil.DeploymentVersionFor(&rc)
if version != dc.Status.LatestVersion {
return false, nil
}
return deployutil.IsFailedDeployment(&rc), nil
}
func deploymentFailed(dc *deployapi.DeploymentConfig, rcs []kapi.ReplicationController, _ []kapi.Pod) (bool, error) {
if len(rcs) == 0 {
return false, nil
}
rc := rcs[len(rcs)-1]
version := deployutil.DeploymentVersionFor(&rc)
if version != dc.Status.LatestVersion {
return false, nil
}
if !deployutil.IsFailedDeployment(&rc) {
return false, nil
}
cond := deployutil.GetDeploymentCondition(dc.Status, deployapi.DeploymentProgressing)
return cond != nil && cond.Reason == deployutil.TimedOutReason, nil
}
func (p *deploymentDeleter) DeleteDeployment(deployment *kapi.ReplicationController) error {
glog.V(4).Infof("Deleting deployment %q", deployment.Name)
// If the deployment is failed we need to remove its deployer pods, too.
if deployutil.IsFailedDeployment(deployment) {
dpSelector := deployutil.DeployerPodSelector(deployment.Name)
deployers, err := p.pods.Pods(deployment.Namespace).List(kapi.ListOptions{LabelSelector: dpSelector})
if err != nil {
glog.Warning("Cannot list deployer pods for %q: %v\n", deployment.Name, err)
} else {
for _, pod := range deployers.Items {
if err := p.pods.Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
glog.Warning("Cannot remove deployer pod %q: %v\n", pod.Name, err)
}
}
}
}
return p.deployments.ReplicationControllers(deployment.Namespace).Delete(deployment.Name, nil)
}
}
case <-time.After(endTime.Sub(time.Now())):
return nil, wait.ErrWaitTimeout
}
}
return nil, wait.ErrWaitTimeout
}
// CheckDeploymentCompletedFn returns true if the deployment completed
var CheckDeploymentCompletedFn = func(d *kapi.ReplicationController) bool {
return deployutil.IsCompleteDeployment(d)
}
// CheckDeploymentFailedFn returns true if the deployment failed
var CheckDeploymentFailedFn = func(d *kapi.ReplicationController) bool {
return deployutil.IsFailedDeployment(d)
}
// GetPodNamesByFilter looks up pods that satisfy the predicate and returns their names.
func GetPodNamesByFilter(c kclient.PodInterface, label labels.Selector, predicate func(kapi.Pod) bool) (podNames []string, err error) {
podList, err := c.List(kapi.ListOptions{LabelSelector: label})
if err != nil {
return nil, err
}
for _, pod := range podList.Items {
if predicate(pod) {
podNames = append(podNames, pod.Name)
}
}
return podNames, nil
}
// Stop scales a replication controller via its deployment configuration down to
// zero replicas, waits for all of them to get deleted and then deletes both the
// replication controller and its deployment configuration.
func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *kapi.DeleteOptions) error {
// Pause the deployment configuration to prevent the new deployments from
// being triggered.
config, err := reaper.pause(namespace, name)
configNotFound := kerrors.IsNotFound(err)
if err != nil && !configNotFound {
return err
}
var (
isPaused bool
legacy bool
)
// Determine if the deployment config controller noticed the pause.
if !configNotFound {
if err := wait.Poll(1*time.Second, 1*time.Minute, func() (bool, error) {
dc, err := reaper.oc.DeploymentConfigs(namespace).Get(name)
if err != nil {
return false, err
}
isPaused = dc.Spec.Paused
return dc.Status.ObservedGeneration >= config.Generation, nil
}); err != nil {
return err
}
// If we failed to pause the deployment config, it means we are talking to
// old API that does not support pausing. In that case, we delete the
// deployment config to stay backward compatible.
if !isPaused {
if err := reaper.oc.DeploymentConfigs(namespace).Delete(name); err != nil {
return err
}
// Setting this to true avoid deleting the config at the end.
legacy = true
}
}
// Clean up deployments related to the config. Even if the deployment
// configuration has been deleted, we want to sweep the existing replication
// controllers and clean them up.
options := kapi.ListOptions{LabelSelector: util.ConfigSelector(name)}
rcList, err := reaper.kc.ReplicationControllers(namespace).List(options)
if err != nil {
return err
}
rcReaper, err := kubectl.ReaperFor(kapi.Kind("ReplicationController"), reaper.kc)
if err != nil {
return err
}
// If there is neither a config nor any deployments, nor any deployer pods, we can return NotFound.
deployments := rcList.Items
if configNotFound && len(deployments) == 0 {
return kerrors.NewNotFound(kapi.Resource("deploymentconfig"), name)
}
for _, rc := range deployments {
if err = rcReaper.Stop(rc.Namespace, rc.Name, timeout, gracePeriod); err != nil {
// Better not error out here...
glog.Infof("Cannot delete ReplicationController %s/%s for deployment config %s/%s: %v", rc.Namespace, rc.Name, namespace, name, err)
}
// Only remove deployer pods when the deployment was failed. For completed
// deployment the pods should be already deleted.
if !util.IsFailedDeployment(&rc) {
continue
}
// Delete all deployer and hook pods
options = kapi.ListOptions{LabelSelector: util.DeployerPodSelector(rc.Name)}
podList, err := reaper.kc.Pods(rc.Namespace).List(options)
if err != nil {
return err
}
for _, pod := range podList.Items {
err := reaper.kc.Pods(pod.Namespace).Delete(pod.Name, gracePeriod)
if err != nil {
// Better not error out here...
glog.Infof("Cannot delete lifecycle Pod %s/%s for deployment config %s/%s: %v", pod.Namespace, pod.Name, namespace, name, err)
}
}
}
// Nothing to delete or we already deleted the deployment config because we
// failed to pause.
if configNotFound || legacy {
return nil
}
return reaper.oc.DeploymentConfigs(namespace).Delete(name)
}
func (o RetryOptions) Run() error {
allErrs := []error{}
mapping, err := o.Mapper.RESTMapping(kapi.Kind("ReplicationController"))
if err != nil {
return err
}
for _, info := range o.Infos {
config, ok := info.Object.(*deployapi.DeploymentConfig)
if !ok {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("expected deployment configuration, got %T", info.Object)))
continue
}
if config.Spec.Paused {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to retry paused deployment config %q", config.Name)))
continue
}
if config.Status.LatestVersion == 0 {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("no rollouts found for %q", config.Name)))
continue
}
latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config)
rc, err := o.Clientset.ReplicationControllers(config.Namespace).Get(latestDeploymentName)
if err != nil {
if kerrors.IsNotFound(err) {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to find the latest rollout (#%d).\nYou can start a new rollout with 'oc rollout latest dc/%s'.", config.Status.LatestVersion, config.Name)))
continue
}
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to fetch replication controller %q", config.Name)))
continue
}
if !deployutil.IsFailedDeployment(rc) {
message := fmt.Sprintf("rollout #%d is %s; only failed deployments can be retried.\n", config.Status.LatestVersion, strings.ToLower(string(deployutil.DeploymentStatusFor(rc))))
if deployutil.IsCompleteDeployment(rc) {
message += fmt.Sprintf("You can start a new deployment with 'oc rollout latest dc/%s'.", config.Name)
} else {
message += fmt.Sprintf("Optionally, you can cancel this deployment with 'oc rollout cancel dc/%s'.", config.Name)
}
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, errors.New(message)))
continue
}
// Delete the deployer pod as well as the deployment hooks pods, if any
pods, err := o.Clientset.Pods(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(latestDeploymentName)})
if err != nil {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("failed to list deployer/hook pods for deployment #%d: %v", config.Status.LatestVersion, err)))
continue
}
hasError := false
for _, pod := range pods.Items {
err := o.Clientset.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0))
if err != nil {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.Status.LatestVersion, err)))
hasError = true
}
}
if hasError {
continue
}
patches := set.CalculatePatches([]*resource.Info{{Object: rc, Mapping: mapping}}, o.Encoder, func(*resource.Info) (bool, error) {
rc.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew)
delete(rc.Annotations, deployapi.DeploymentStatusReasonAnnotation)
delete(rc.Annotations, deployapi.DeploymentCancelledAnnotation)
return true, nil
})
if len(patches) == 0 {
kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, "already retried")
continue
}
if _, err := o.Clientset.ReplicationControllers(rc.Namespace).Patch(rc.Name, kapi.StrategicMergePatchType, patches[0].Patch); err != nil {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, err))
continue
}
kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, fmt.Sprintf("retried rollout #%d", config.Status.LatestVersion))
}
return utilerrors.NewAggregate(allErrs)
}
// Stop scales a replication controller via its deployment configuration down to
// zero replicas, waits for all of them to get deleted and then deletes both the
// replication controller and its deployment configuration.
func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *kapi.DeleteOptions) error {
// If the config is already deleted, it may still have associated
// deployments which didn't get cleaned up during prior calls to Stop. If
// the config can't be found, still make an attempt to clean up the
// deployments.
//
// It's important to delete the config first to avoid an undesirable side
// effect which can cause the deployment to be re-triggered upon the
// config's deletion. See https://github.com/openshift/origin/issues/2721
// for more details.
err := reaper.oc.DeploymentConfigs(namespace).Delete(name)
configNotFound := kerrors.IsNotFound(err)
if err != nil && !configNotFound {
return err
}
// Clean up deployments related to the config.
options := kapi.ListOptions{LabelSelector: util.ConfigSelector(name)}
rcList, err := reaper.kc.ReplicationControllers(namespace).List(options)
if err != nil {
return err
}
rcReaper, err := kubectl.ReaperFor(kapi.Kind("ReplicationController"), reaper.kc)
if err != nil {
return err
}
// If there is neither a config nor any deployments, nor any deployer pods, we can return NotFound.
deployments := rcList.Items
if configNotFound && len(deployments) == 0 {
return kerrors.NewNotFound(kapi.Resource("deploymentconfig"), name)
}
for _, rc := range deployments {
if err = rcReaper.Stop(rc.Namespace, rc.Name, timeout, gracePeriod); err != nil {
// Better not error out here...
glog.Infof("Cannot delete ReplicationController %s/%s: %v", rc.Namespace, rc.Name, err)
}
// Only remove deployer pods when the deployment was failed. For completed
// deployment the pods should be already deleted.
if !util.IsFailedDeployment(&rc) {
continue
}
// Delete all deployer and hook pods
options = kapi.ListOptions{LabelSelector: util.DeployerPodSelector(rc.Name)}
podList, err := reaper.kc.Pods(rc.Namespace).List(options)
if err != nil {
return err
}
for _, pod := range podList.Items {
err := reaper.kc.Pods(pod.Namespace).Delete(pod.Name, gracePeriod)
if err != nil {
// Better not error out here...
glog.Infof("Cannot delete Pod %s/%s: %v", pod.Namespace, pod.Name, err)
}
}
}
return nil
}
// Handle processes deployment and either creates a deployer pod or responds
// to a terminal deployment status. Since this controller started using caches,
// the provided rc MUST be deep-copied beforehand (see work() in factory.go).
func (c *DeploymentController) Handle(deployment *kapi.ReplicationController) error {
// Copy all the annotations from the deployment.
updatedAnnotations := make(map[string]string)
for key, value := range deployment.Annotations {
updatedAnnotations[key] = value
}
currentStatus := deployutil.DeploymentStatusFor(deployment)
nextStatus := currentStatus
deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name)
deployer, deployerErr := c.podStore.Pods(deployment.Namespace).Get(deployerPodName)
if deployerErr == nil {
nextStatus = c.nextStatus(deployer, deployment, updatedAnnotations)
}
switch currentStatus {
case deployapi.DeploymentStatusNew:
// If the deployment has been cancelled, don't create a deployer pod.
// Instead try to delete any deployer pods found and transition the
// deployment to Pending so that the deployment config controller
// continues to see the deployment as in-flight. Eventually the deletion
// of the deployer pod should cause a requeue of this deployment and
// then it can be transitioned to Failed by this controller.
if deployutil.IsDeploymentCancelled(deployment) {
nextStatus = deployapi.DeploymentStatusPending
if err := c.cleanupDeployerPods(deployment); err != nil {
return err
}
break
}
switch {
case kerrors.IsNotFound(deployerErr):
if _, ok := deployment.Annotations[deployapi.DeploymentIgnorePodAnnotation]; ok {
return nil
}
// Generate a deployer pod spec.
deployerPod, err := c.makeDeployerPod(deployment)
if err != nil {
return fatalError(fmt.Sprintf("couldn't make deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err))
}
// Create the deployer pod.
deploymentPod, err := c.pn.Pods(deployment.Namespace).Create(deployerPod)
// Retry on error.
if err != nil {
return actionableError(fmt.Sprintf("couldn't create deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err))
}
updatedAnnotations[deployapi.DeploymentPodAnnotation] = deploymentPod.Name
nextStatus = deployapi.DeploymentStatusPending
glog.V(4).Infof("Created deployer pod %s for deployment %s", deploymentPod.Name, deployutil.LabelForDeployment(deployment))
case deployerErr != nil:
// If the pod already exists, it's possible that a previous CreatePod
// succeeded but the deployment state update failed and now we're re-
// entering. Ensure that the pod is the one we created by verifying the
// annotation on it, and throw a retryable error.
return fmt.Errorf("couldn't fetch existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), deployerErr)
default: /* deployerErr == nil */
// Do a stronger check to validate that the existing deployer pod is
// actually for this deployment, and if not, fail this deployment.
//
// TODO: Investigate checking the container image of the running pod and
// comparing with the intended deployer pod image. If we do so, we'll need
// to ensure that changes to 'unrelated' pods don't result in updates to
// the deployment. So, the image check will have to be done in other areas
// of the code as well.
if deployutil.DeploymentNameFor(deployer) != deployment.Name {
nextStatus = deployapi.DeploymentStatusFailed
updatedAnnotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnrelatedDeploymentExists
c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "FailedCreate", fmt.Sprintf("Error creating deployer pod since another pod with the same name (%q) exists", deployer.Name))
glog.V(2).Infof("Couldn't create deployer pod for %s since an unrelated pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), deployer.Name)
} else {
// Update to pending or to the appropriate status relative to the existing validated deployer pod.
updatedAnnotations[deployapi.DeploymentPodAnnotation] = deployer.Name
nextStatus = nextStatusComp(nextStatus, deployapi.DeploymentStatusPending)
glog.V(4).Infof("Detected existing deployer pod %s for deployment %s", deployer.Name, deployutil.LabelForDeployment(deployment))
}
}
case deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning:
switch {
case kerrors.IsNotFound(deployerErr):
nextStatus = deployapi.DeploymentStatusFailed
// If the deployment is cancelled here then we deleted the deployer in a previous
// resync of the deployment.
if !deployutil.IsDeploymentCancelled(deployment) {
updatedAnnotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedDeployerPodNoLongerExists
c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "Failed", fmt.Sprintf("Deployer pod %q has gone missing", deployerPodName))
deployerErr = fmt.Errorf("Failing deployment %q because its deployer pod %q disappeared", deployutil.LabelForDeployment(deployment), deployerPodName)
utilruntime.HandleError(deployerErr)
}
case deployerErr != nil:
// We'll try again later on resync. Continue to process cancellations.
deployerErr = fmt.Errorf("Error getting deployer pod %q for deployment %q: %v", deployerPodName, deployutil.LabelForDeployment(deployment), deployerErr)
utilruntime.HandleError(deployerErr)
default: /* err == nil */
// If the deployment has been cancelled, delete any deployer pods
// found. Eventually the deletion of the deployer pod should cause
// a requeue of this deployment and then it can be transitioned to
// Failed.
if deployutil.IsDeploymentCancelled(deployment) {
if err := c.cleanupDeployerPods(deployment); err != nil {
return err
}
}
}
case deployapi.DeploymentStatusFailed:
// Try to cleanup once more a cancelled deployment in case hook pods
// were created just after we issued the first cleanup request.
if deployutil.IsDeploymentCancelled(deployment) {
if err := c.cleanupDeployerPods(deployment); err != nil {
return err
}
}
case deployapi.DeploymentStatusComplete:
if err := c.cleanupDeployerPods(deployment); err != nil {
return err
}
}
// Update only if we need to transition to a new phase.
if deployutil.CanTransitionPhase(currentStatus, nextStatus) {
deployment, err := deployutil.DeploymentDeepCopy(deployment)
if err != nil {
return err
}
updatedAnnotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus)
deployment.Annotations = updatedAnnotations
// if we are going to transition to failed or complete and scale is non-zero, we'll check one more
// time to see if we are a test deployment to guarantee that we maintain the test invariant.
if deployment.Spec.Replicas != 0 && deployutil.IsTerminatedDeployment(deployment) {
if config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec); err == nil && config.Spec.Test {
deployment.Spec.Replicas = 0
}
}
if _, err := c.rn.ReplicationControllers(deployment.Namespace).Update(deployment); err != nil {
return fmt.Errorf("couldn't update deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err)
}
glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas)
if deployutil.IsDeploymentCancelled(deployment) && deployutil.IsFailedDeployment(deployment) {
c.emitDeploymentEvent(deployment, kapi.EventTypeNormal, "DeploymentCancelled", fmt.Sprintf("Deployment %q cancelled", deployutil.LabelForDeployment(deployment)))
}
}
return nil
}