// retry resets the status of the latest deployment to New, which will cause
// the deployment to be retried. An error is returned if the deployment is not
// currently in a failed state.
func (o *DeployOptions) retry(config *deployapi.DeploymentConfig, out io.Writer) error {
if config.LatestVersion == 0 {
return fmt.Errorf("no deployments found for %s/%s", config.Namespace, config.Name)
}
deploymentName := deployutil.LatestDeploymentNameForConfig(config)
deployment, err := o.kubeClient.ReplicationControllers(config.Namespace).Get(deploymentName)
if err != nil {
if kerrors.IsNotFound(err) {
return fmt.Errorf("Unable to find the latest deployment (#%d).\nYou can start a new deployment using the --latest option.", config.LatestVersion)
}
return err
}
if status := deployutil.DeploymentStatusFor(deployment); status != deployapi.DeploymentStatusFailed {
message := fmt.Sprintf("#%d is %s; only failed deployments can be retried.\n", config.LatestVersion, status)
if status == deployapi.DeploymentStatusComplete {
message += "You can start a new deployment using the --latest option."
} else {
message += "Optionally, you can cancel this deployment using the --cancel option."
}
return fmt.Errorf(message)
}
// Delete the deployer pod as well as the deployment hooks pods, if any
pods, err := o.kubeClient.Pods(config.Namespace).List(deployutil.DeployerPodSelector(deploymentName), fields.Everything())
if err != nil {
return fmt.Errorf("Failed to list deployer/hook pods for deployment #%d: %v", config.LatestVersion, err)
}
for _, pod := range pods.Items {
err := o.kubeClient.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0))
if err != nil {
return fmt.Errorf("Failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.LatestVersion, err)
}
}
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew)
// clear out the cancellation flag as well as any previous status-reason annotation
delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation)
delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation)
_, err = o.kubeClient.ReplicationControllers(deployment.Namespace).Update(deployment)
if err == nil {
fmt.Fprintf(out, "retried #%d\n", config.LatestVersion)
}
return err
}
func (p *deploymentDeleter) DeleteDeployment(deployment *kapi.ReplicationController) error {
glog.V(4).Infof("Deleting deployment %q", deployment.Name)
// If the deployment is failed we need to remove its deployer pods, too.
if deployutil.IsFailedDeployment(deployment) {
dpSelector := deployutil.DeployerPodSelector(deployment.Name)
deployers, err := p.pods.Pods(deployment.Namespace).List(kapi.ListOptions{LabelSelector: dpSelector})
if err != nil {
glog.Warning("Cannot list deployer pods for %q: %v\n", deployment.Name, err)
} else {
for _, pod := range deployers.Items {
if err := p.pods.Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
glog.Warning("Cannot remove deployer pod %q: %v\n", pod.Name, err)
}
}
}
}
return p.deployments.ReplicationControllers(deployment.Namespace).Delete(deployment.Name, nil)
}
func (c *DeploymentController) cleanupDeployerPods(deployment *kapi.ReplicationController) error {
selector := deployutil.DeployerPodSelector(deployment.Name)
deployerList, err := c.podStore.Pods(deployment.Namespace).List(selector)
if err != nil {
return fmt.Errorf("couldn't fetch deployer pods for %q: %v", deployutil.LabelForDeployment(deployment), err)
}
cleanedAll := true
for _, deployerPod := range deployerList.Items {
if err := c.pn.Pods(deployerPod.Namespace).Delete(deployerPod.Name, &kapi.DeleteOptions{}); err != nil && !kerrors.IsNotFound(err) {
// if the pod deletion failed, then log the error and continue
// we will try to delete any remaining deployer pods and return an error later
utilruntime.HandleError(fmt.Errorf("couldn't delete completed deployer pod %q for deployment %q: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err))
cleanedAll = false
}
}
if !cleanedAll {
return actionableError(fmt.Sprintf("couldn't clean up all deployer pods for %s", deployment.Name))
}
return nil
}
// retry resets the status of the latest deployment to New, which will cause
// the deployment to be retried. An error is returned if the deployment is not
// currently in a failed state.
func (o DeployOptions) retry(config *deployapi.DeploymentConfig) error {
if config.Spec.Paused {
return fmt.Errorf("cannot retry a paused deployment config")
}
if config.Status.LatestVersion == 0 {
return fmt.Errorf("no deployments found for %s/%s", config.Namespace, config.Name)
}
// TODO: This implies that deploymentconfig.status.latestVersion is always synced. Currently,
// that's the case because clients (oc, trigger controllers) are updating the status directly.
// Clients should be acting either on spec or on annotations and status updates should be a
// responsibility of the main controller. We need to start by unplugging this assumption from
// our client tools.
deploymentName := deployutil.LatestDeploymentNameForConfig(config)
deployment, err := o.kubeClient.ReplicationControllers(config.Namespace).Get(deploymentName)
if err != nil {
if kerrors.IsNotFound(err) {
return fmt.Errorf("unable to find the latest deployment (#%d).\nYou can start a new deployment with 'oc deploy --latest dc/%s'.", config.Status.LatestVersion, config.Name)
}
return err
}
if !deployutil.IsFailedDeployment(deployment) {
message := fmt.Sprintf("#%d is %s; only failed deployments can be retried.\n", config.Status.LatestVersion, deployutil.DeploymentStatusFor(deployment))
if deployutil.IsCompleteDeployment(deployment) {
message += fmt.Sprintf("You can start a new deployment with 'oc deploy --latest dc/%s'.", config.Name)
} else {
message += fmt.Sprintf("Optionally, you can cancel this deployment with 'oc deploy --cancel dc/%s'.", config.Name)
}
return fmt.Errorf(message)
}
// Delete the deployer pod as well as the deployment hooks pods, if any
pods, err := o.kubeClient.Pods(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(deploymentName)})
if err != nil {
return fmt.Errorf("failed to list deployer/hook pods for deployment #%d: %v", config.Status.LatestVersion, err)
}
for _, pod := range pods.Items {
err := o.kubeClient.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0))
if err != nil {
return fmt.Errorf("failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.Status.LatestVersion, err)
}
}
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew)
// clear out the cancellation flag as well as any previous status-reason annotation
delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation)
delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation)
_, err = o.kubeClient.ReplicationControllers(deployment.Namespace).Update(deployment)
if err != nil {
return err
}
fmt.Fprintf(o.out, "Retried #%d\n", config.Status.LatestVersion)
if o.follow {
return o.getLogs(config)
}
fmt.Fprintf(o.out, "Use '%s logs -f dc/%s' to track its progress.\n", o.baseCommandName, config.Name)
return nil
}
// Stop scales a replication controller via its deployment configuration down to
// zero replicas, waits for all of them to get deleted and then deletes both the
// replication controller and its deployment configuration.
func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *kapi.DeleteOptions) error {
// Pause the deployment configuration to prevent the new deployments from
// being triggered.
config, err := reaper.pause(namespace, name)
configNotFound := kerrors.IsNotFound(err)
if err != nil && !configNotFound {
return err
}
var (
isPaused bool
legacy bool
)
// Determine if the deployment config controller noticed the pause.
if !configNotFound {
if err := wait.Poll(1*time.Second, 1*time.Minute, func() (bool, error) {
dc, err := reaper.oc.DeploymentConfigs(namespace).Get(name)
if err != nil {
return false, err
}
isPaused = dc.Spec.Paused
return dc.Status.ObservedGeneration >= config.Generation, nil
}); err != nil {
return err
}
// If we failed to pause the deployment config, it means we are talking to
// old API that does not support pausing. In that case, we delete the
// deployment config to stay backward compatible.
if !isPaused {
if err := reaper.oc.DeploymentConfigs(namespace).Delete(name); err != nil {
return err
}
// Setting this to true avoid deleting the config at the end.
legacy = true
}
}
// Clean up deployments related to the config. Even if the deployment
// configuration has been deleted, we want to sweep the existing replication
// controllers and clean them up.
options := kapi.ListOptions{LabelSelector: util.ConfigSelector(name)}
rcList, err := reaper.kc.ReplicationControllers(namespace).List(options)
if err != nil {
return err
}
rcReaper, err := kubectl.ReaperFor(kapi.Kind("ReplicationController"), reaper.kc)
if err != nil {
return err
}
// If there is neither a config nor any deployments, nor any deployer pods, we can return NotFound.
deployments := rcList.Items
if configNotFound && len(deployments) == 0 {
return kerrors.NewNotFound(kapi.Resource("deploymentconfig"), name)
}
for _, rc := range deployments {
if err = rcReaper.Stop(rc.Namespace, rc.Name, timeout, gracePeriod); err != nil {
// Better not error out here...
glog.Infof("Cannot delete ReplicationController %s/%s for deployment config %s/%s: %v", rc.Namespace, rc.Name, namespace, name, err)
}
// Only remove deployer pods when the deployment was failed. For completed
// deployment the pods should be already deleted.
if !util.IsFailedDeployment(&rc) {
continue
}
// Delete all deployer and hook pods
options = kapi.ListOptions{LabelSelector: util.DeployerPodSelector(rc.Name)}
podList, err := reaper.kc.Pods(rc.Namespace).List(options)
if err != nil {
return err
}
for _, pod := range podList.Items {
err := reaper.kc.Pods(pod.Namespace).Delete(pod.Name, gracePeriod)
if err != nil {
// Better not error out here...
glog.Infof("Cannot delete lifecycle Pod %s/%s for deployment config %s/%s: %v", pod.Namespace, pod.Name, namespace, name, err)
}
}
}
// Nothing to delete or we already deleted the deployment config because we
// failed to pause.
if configNotFound || legacy {
return nil
}
return reaper.oc.DeploymentConfigs(namespace).Delete(name)
}
// Create creates a DeployerPodController.
func (factory *DeployerPodControllerFactory) Create() controller.RunnableController {
deploymentLW := &deployutil.ListWatcherImpl{
ListFunc: func() (runtime.Object, error) {
return factory.KubeClient.ReplicationControllers(kapi.NamespaceAll).List(labels.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return factory.KubeClient.ReplicationControllers(kapi.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
}
deploymentStore := cache.NewStore(cache.MetaNamespaceKeyFunc)
cache.NewReflector(deploymentLW, &kapi.ReplicationController{}, deploymentStore, 2*time.Minute).Run()
// TODO: These should be filtered somehow to include only the primary
// deployer pod. For now, the controller is filtering.
// TODO: Even with the label selector, this is inefficient on the backend
// and we should work to consolidate namespace-spanning pod watches. For
// example, the build infra is also watching pods across namespaces.
podLW := &deployutil.ListWatcherImpl{
ListFunc: func() (runtime.Object, error) {
return factory.KubeClient.Pods(kapi.NamespaceAll).List(deployutil.AnyDeployerPodSelector(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return factory.KubeClient.Pods(kapi.NamespaceAll).Watch(deployutil.AnyDeployerPodSelector(), fields.Everything(), resourceVersion)
},
}
podQueue := cache.NewFIFO(cache.MetaNamespaceKeyFunc)
cache.NewReflector(podLW, &kapi.Pod{}, podQueue, 2*time.Minute).Run()
podController := &DeployerPodController{
deploymentClient: &deploymentClientImpl{
getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
// Try to use the cache first. Trust hits and return them.
example := &kapi.ReplicationController{ObjectMeta: kapi.ObjectMeta{Namespace: namespace, Name: name}}
cached, exists, err := deploymentStore.Get(example)
if err == nil && exists {
return cached.(*kapi.ReplicationController), nil
}
// Double-check with the master for cache misses/errors, since those
// are rare and API calls are expensive but more reliable.
return factory.KubeClient.ReplicationControllers(namespace).Get(name)
},
updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
return factory.KubeClient.ReplicationControllers(namespace).Update(deployment)
},
listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
return factory.KubeClient.ReplicationControllers(namespace).List(deployutil.ConfigSelector(configName))
},
},
deployerPodsFor: func(namespace, name string) (*kapi.PodList, error) {
return factory.KubeClient.Pods(namespace).List(deployutil.DeployerPodSelector(name), fields.Everything())
},
deletePod: func(namespace, name string) error {
return factory.KubeClient.Pods(namespace).Delete(name, kapi.NewDeleteOptions(0))
},
}
return &controller.RetryController{
Queue: podQueue,
RetryManager: controller.NewQueueRetryManager(
podQueue,
cache.MetaNamespaceKeyFunc,
func(obj interface{}, err error, retries controller.Retry) bool {
kutil.HandleError(err)
// infinite retries for a transient error
if _, isTransient := err.(transientError); isTransient {
return true
}
// no retries for anything else
if retries.Count > 0 {
return false
}
return true
},
kutil.NewTokenBucketRateLimiter(1, 10),
),
Handle: func(obj interface{}) error {
pod := obj.(*kapi.Pod)
return podController.Handle(pod)
},
}
}
func (o RetryOptions) Run() error {
allErrs := []error{}
mapping, err := o.Mapper.RESTMapping(kapi.Kind("ReplicationController"))
if err != nil {
return err
}
for _, info := range o.Infos {
config, ok := info.Object.(*deployapi.DeploymentConfig)
if !ok {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("expected deployment configuration, got %T", info.Object)))
continue
}
if config.Spec.Paused {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to retry paused deployment config %q", config.Name)))
continue
}
if config.Status.LatestVersion == 0 {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("no rollouts found for %q", config.Name)))
continue
}
latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config)
rc, err := o.Clientset.ReplicationControllers(config.Namespace).Get(latestDeploymentName)
if err != nil {
if kerrors.IsNotFound(err) {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to find the latest rollout (#%d).\nYou can start a new rollout with 'oc rollout latest dc/%s'.", config.Status.LatestVersion, config.Name)))
continue
}
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("unable to fetch replication controller %q", config.Name)))
continue
}
if !deployutil.IsFailedDeployment(rc) {
message := fmt.Sprintf("rollout #%d is %s; only failed deployments can be retried.\n", config.Status.LatestVersion, strings.ToLower(string(deployutil.DeploymentStatusFor(rc))))
if deployutil.IsCompleteDeployment(rc) {
message += fmt.Sprintf("You can start a new deployment with 'oc rollout latest dc/%s'.", config.Name)
} else {
message += fmt.Sprintf("Optionally, you can cancel this deployment with 'oc rollout cancel dc/%s'.", config.Name)
}
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, errors.New(message)))
continue
}
// Delete the deployer pod as well as the deployment hooks pods, if any
pods, err := o.Clientset.Pods(config.Namespace).List(kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(latestDeploymentName)})
if err != nil {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("failed to list deployer/hook pods for deployment #%d: %v", config.Status.LatestVersion, err)))
continue
}
hasError := false
for _, pod := range pods.Items {
err := o.Clientset.Pods(pod.Namespace).Delete(pod.Name, kapi.NewDeleteOptions(0))
if err != nil {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, fmt.Errorf("failed to delete deployer/hook pod %s for deployment #%d: %v", pod.Name, config.Status.LatestVersion, err)))
hasError = true
}
}
if hasError {
continue
}
patches := set.CalculatePatches([]*resource.Info{{Object: rc, Mapping: mapping}}, o.Encoder, func(*resource.Info) (bool, error) {
rc.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew)
delete(rc.Annotations, deployapi.DeploymentStatusReasonAnnotation)
delete(rc.Annotations, deployapi.DeploymentCancelledAnnotation)
return true, nil
})
if len(patches) == 0 {
kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, "already retried")
continue
}
if _, err := o.Clientset.ReplicationControllers(rc.Namespace).Patch(rc.Name, kapi.StrategicMergePatchType, patches[0].Patch); err != nil {
allErrs = append(allErrs, kcmdutil.AddSourceToErr("retrying", info.Source, err))
continue
}
kcmdutil.PrintSuccess(o.Mapper, false, o.Out, info.Mapping.Resource, info.Name, false, fmt.Sprintf("retried rollout #%d", config.Status.LatestVersion))
}
return utilerrors.NewAggregate(allErrs)
}
// Stop scales a replication controller via its deployment configuration down to
// zero replicas, waits for all of them to get deleted and then deletes both the
// replication controller and its deployment configuration.
func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *kapi.DeleteOptions) error {
// If the config is already deleted, it may still have associated
// deployments which didn't get cleaned up during prior calls to Stop. If
// the config can't be found, still make an attempt to clean up the
// deployments.
//
// It's important to delete the config first to avoid an undesirable side
// effect which can cause the deployment to be re-triggered upon the
// config's deletion. See https://github.com/openshift/origin/issues/2721
// for more details.
err := reaper.oc.DeploymentConfigs(namespace).Delete(name)
configNotFound := kerrors.IsNotFound(err)
if err != nil && !configNotFound {
return err
}
// Clean up deployments related to the config.
options := kapi.ListOptions{LabelSelector: util.ConfigSelector(name)}
rcList, err := reaper.kc.ReplicationControllers(namespace).List(options)
if err != nil {
return err
}
rcReaper, err := kubectl.ReaperFor(kapi.Kind("ReplicationController"), reaper.kc)
if err != nil {
return err
}
// If there is neither a config nor any deployments, nor any deployer pods, we can return NotFound.
deployments := rcList.Items
if configNotFound && len(deployments) == 0 {
return kerrors.NewNotFound(kapi.Resource("deploymentconfig"), name)
}
for _, rc := range deployments {
if err = rcReaper.Stop(rc.Namespace, rc.Name, timeout, gracePeriod); err != nil {
// Better not error out here...
glog.Infof("Cannot delete ReplicationController %s/%s: %v", rc.Namespace, rc.Name, err)
}
// Only remove deployer pods when the deployment was failed. For completed
// deployment the pods should be already deleted.
if !util.IsFailedDeployment(&rc) {
continue
}
// Delete all deployer and hook pods
options = kapi.ListOptions{LabelSelector: util.DeployerPodSelector(rc.Name)}
podList, err := reaper.kc.Pods(rc.Namespace).List(options)
if err != nil {
return err
}
for _, pod := range podList.Items {
err := reaper.kc.Pods(pod.Namespace).Delete(pod.Name, gracePeriod)
if err != nil {
// Better not error out here...
glog.Infof("Cannot delete Pod %s/%s: %v", pod.Namespace, pod.Name, err)
}
}
}
return nil
}
func NewCmdPruneDeployments(f *clientcmd.Factory, parentName, name string, out io.Writer) *cobra.Command {
cfg := &pruneDeploymentConfig{
Confirm: false,
KeepYoungerThan: 60 * time.Minute,
KeepComplete: 5,
KeepFailed: 1,
}
cmd := &cobra.Command{
Use: name,
Short: "Remove old completed and failed deployments",
Long: deploymentsLongDesc,
Example: fmt.Sprintf(deploymentsExample, parentName, name),
SuggestFor: []string{"deployment", "deployments"},
Run: func(cmd *cobra.Command, args []string) {
if len(args) > 0 {
glog.Fatalf("No arguments are allowed to this command")
}
osClient, kclient, err := f.Clients()
if err != nil {
cmdutil.CheckErr(err)
}
deploymentConfigList, err := osClient.DeploymentConfigs(kapi.NamespaceAll).List(kapi.ListOptions{})
if err != nil {
cmdutil.CheckErr(err)
}
deploymentList, err := kclient.ReplicationControllers(kapi.NamespaceAll).List(kapi.ListOptions{})
if err != nil {
cmdutil.CheckErr(err)
}
deploymentConfigs := []*deployapi.DeploymentConfig{}
for i := range deploymentConfigList.Items {
deploymentConfigs = append(deploymentConfigs, &deploymentConfigList.Items[i])
}
deployments := []*kapi.ReplicationController{}
for i := range deploymentList.Items {
deployments = append(deployments, &deploymentList.Items[i])
}
var deploymentPruneFunc prune.PruneFunc
w := tabwriter.NewWriter(out, 10, 4, 3, ' ', 0)
defer w.Flush()
describingPruneDeploymentFunc := func(deployment *kapi.ReplicationController) error {
fmt.Fprintf(w, "%s\t%s\n", deployment.Namespace, deployment.Name)
return nil
}
switch cfg.Confirm {
case true:
deploymentPruneFunc = func(deployment *kapi.ReplicationController) error {
describingPruneDeploymentFunc(deployment)
// If the deployment is failed we need to remove its deployer pods, too.
if deployutil.DeploymentStatusFor(deployment) == deployapi.DeploymentStatusFailed {
dpSelector := deployutil.DeployerPodSelector(deployment.Name)
deployers, err := kclient.Pods(deployment.Namespace).List(kapi.ListOptions{LabelSelector: dpSelector})
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot list deployer pods for %q: %v\n", deployment.Name, err)
} else {
for _, pod := range deployers.Items {
if err := kclient.Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
fmt.Fprintf(os.Stderr, "Cannot remove deployer pod %q: %v\n", pod.Name, err)
}
}
}
}
return kclient.ReplicationControllers(deployment.Namespace).Delete(deployment.Name)
}
default:
fmt.Fprintln(os.Stderr, "Dry run enabled - no modifications will be made. Add --confirm to remove deployments")
deploymentPruneFunc = describingPruneDeploymentFunc
}
fmt.Fprintln(w, "NAMESPACE\tNAME")
pruneTask := prune.NewPruneTasker(deploymentConfigs, deployments, cfg.KeepYoungerThan, cfg.Orphans, cfg.KeepComplete, cfg.KeepFailed, deploymentPruneFunc)
err = pruneTask.PruneTask()
if err != nil {
cmdutil.CheckErr(err)
}
},
}
cmd.Flags().BoolVar(&cfg.Confirm, "confirm", cfg.Confirm, "Specify that deployment pruning should proceed. Defaults to false, displaying what would be deleted but not actually deleting anything.")
cmd.Flags().BoolVar(&cfg.Orphans, "orphans", cfg.Orphans, "Prune all deployments where the associated DeploymentConfig no longer exists, the status is complete or failed, and the replica size is 0.")
cmd.Flags().DurationVar(&cfg.KeepYoungerThan, "keep-younger-than", cfg.KeepYoungerThan, "Specify the minimum age of a deployment for it to be considered a candidate for pruning.")
cmd.Flags().IntVar(&cfg.KeepComplete, "keep-complete", cfg.KeepComplete, "Per DeploymentConfig, specify the number of deployments whose status is complete that will be preserved whose replica size is 0.")
cmd.Flags().IntVar(&cfg.KeepFailed, "keep-failed", cfg.KeepFailed, "Per DeploymentConfig, specify the number of deployments whose status is failed that will be preserved whose replica size is 0.")
return cmd
}
// Handle syncs pod's status with any associated deployment.
func (c *DeployerPodController) Handle(pod *kapi.Pod) error {
// Find the deployment associated with the deployer pod.
deploymentName := deployutil.DeploymentNameFor(pod)
if len(deploymentName) == 0 {
return nil
}
// Reject updates to anything but the main deployer pod
// TODO: Find a way to filter this on the watch side.
if pod.Name != deployutil.DeployerPodNameForDeployment(deploymentName) {
return nil
}
deployment := &kapi.ReplicationController{ObjectMeta: kapi.ObjectMeta{Namespace: pod.Namespace, Name: deploymentName}}
cached, exists, err := c.store.Get(deployment)
if err == nil && exists {
// Try to use the cache first. Trust hits and return them.
deployment = cached.(*kapi.ReplicationController)
} else {
// Double-check with the master for cache misses/errors, since those
// are rare and API calls are expensive but more reliable.
deployment, err = c.kClient.ReplicationControllers(pod.Namespace).Get(deploymentName)
}
// If the deployment for this pod has disappeared, we should clean up this
// and any other deployer pods, then bail out.
if err != nil {
// Some retrieval error occurred. Retry.
if !kerrors.IsNotFound(err) {
return fmt.Errorf("couldn't get deployment %s/%s which owns deployer pod %s/%s", pod.Namespace, deploymentName, pod.Name, pod.Namespace)
}
// Find all the deployer pods for the deployment (including this one).
opts := kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(deploymentName)}
deployers, err := c.kClient.Pods(pod.Namespace).List(opts)
if err != nil {
// Retry.
return fmt.Errorf("couldn't get deployer pods for %s: %v", deployutil.LabelForDeployment(deployment), err)
}
// Delete all deployers.
for _, deployer := range deployers.Items {
err := c.kClient.Pods(deployer.Namespace).Delete(deployer.Name, kapi.NewDeleteOptions(0))
if err != nil {
if !kerrors.IsNotFound(err) {
// TODO: Should this fire an event?
glog.V(2).Infof("Couldn't delete orphaned deployer pod %s/%s: %v", deployer.Namespace, deployer.Name, err)
}
} else {
// TODO: Should this fire an event?
glog.V(2).Infof("Deleted orphaned deployer pod %s/%s", deployer.Namespace, deployer.Name)
}
}
return nil
}
currentStatus := deployutil.DeploymentStatusFor(deployment)
nextStatus := currentStatus
switch pod.Status.Phase {
case kapi.PodRunning:
nextStatus = deployapi.DeploymentStatusRunning
case kapi.PodSucceeded:
nextStatus = deployapi.DeploymentStatusComplete
config, decodeErr := c.decodeConfig(deployment)
// If the deployment was cancelled just prior to the deployer pod succeeding
// then we need to remove the cancel annotations from the complete deployment
// and emit an event letting users know their cancellation failed.
if deployutil.IsDeploymentCancelled(deployment) {
delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation)
delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation)
if decodeErr == nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "FailedCancellation", "Deployment %q succeeded before cancel recorded", deployutil.LabelForDeployment(deployment))
} else {
c.recorder.Event(deployment, kapi.EventTypeWarning, "FailedCancellation", "Succeeded before cancel recorded")
}
}
// Sync the internal replica annotation with the target so that we can
// distinguish deployer updates from other scaling events.
deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = deployment.Annotations[deployapi.DesiredReplicasAnnotation]
if nextStatus == deployapi.DeploymentStatusComplete {
delete(deployment.Annotations, deployapi.DesiredReplicasAnnotation)
}
// reset the size of any test container, since we are the ones updating the RC
if decodeErr == nil && config.Spec.Test {
deployment.Spec.Replicas = 0
}
case kapi.PodFailed:
nextStatus = deployapi.DeploymentStatusFailed
// reset the size of any test container, since we are the ones updating the RC
if config, err := c.decodeConfig(deployment); err == nil && config.Spec.Test {
deployment.Spec.Replicas = 0
}
}
if deployutil.CanTransitionPhase(currentStatus, nextStatus) {
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus)
if _, err := c.kClient.ReplicationControllers(deployment.Namespace).Update(deployment); err != nil {
if kerrors.IsNotFound(err) {
return nil
}
return fmt.Errorf("couldn't update Deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err)
}
glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas)
// If the deployment was canceled, trigger a reconcilation of its deployment config
// so that the latest complete deployment can immediately rollback in place of the
// canceled deployment.
if nextStatus == deployapi.DeploymentStatusFailed && deployutil.IsDeploymentCancelled(deployment) {
// If we are unable to get the deployment config, then the deploymentconfig controller will
// perform its duties once the resync interval forces the deploymentconfig to be reconciled.
name := deployutil.DeploymentConfigNameFor(deployment)
kclient.RetryOnConflict(kclient.DefaultRetry, func() error {
config, err := c.client.DeploymentConfigs(deployment.Namespace).Get(name)
if err != nil {
return err
}
if config.Annotations == nil {
config.Annotations = make(map[string]string)
}
config.Annotations[deployapi.DeploymentCancelledAnnotation] = strconv.Itoa(config.Status.LatestVersion)
_, err = c.client.DeploymentConfigs(config.Namespace).Update(config)
return err
})
}
}
return nil
}