func (t *Tester) testDeleteGracefulImmediate(obj runtime.Object, createFn CreateFunc, getFn GetFunc, expectedGrace int64) { ctx := t.TestContext() foo := copyOrDie(obj) t.setObjectMeta(foo, "foo4") if err := createFn(ctx, foo); err != nil { t.Errorf("unexpected error: %v", err) } objectMeta := t.getObjectMetaOrFail(foo) _, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(expectedGrace)) if err != nil { t.Errorf("unexpected error: %v", err) } if _, err := getFn(ctx, foo); err != nil { t.Fatalf("did not gracefully delete resource: %v", err) } // second delete is immediate, resource is deleted out, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(0)) if err != nil { t.Errorf("unexpected error: %v", err) } _, err = t.storage.(rest.Getter).Get(ctx, objectMeta.Name) if !errors.IsNotFound(err) { t.Errorf("unexpected error, object should be deleted immediately: %v", err) } objectMeta = t.getObjectMetaOrFail(out) // the second delete shouldn't update the object, so the objectMeta.DeletionGracePeriodSeconds should eqaul to the value set in the first delete. if objectMeta.DeletionTimestamp == nil || objectMeta.DeletionGracePeriodSeconds == nil || *objectMeta.DeletionGracePeriodSeconds != 0 { t.Errorf("unexpected deleted meta: %#v", objectMeta) } }
func (t *Tester) testDeleteGracefulExtend(obj runtime.Object, createFn CreateFunc, getFn GetFunc, expectedGrace int64) { ctx := t.TestContext() foo := copyOrDie(obj) t.setObjectMeta(foo, t.namer(3)) if err := createFn(ctx, foo); err != nil { t.Errorf("unexpected error: %v", err) } objectMeta := t.getObjectMetaOrFail(foo) _, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(expectedGrace)) if err != nil { t.Errorf("unexpected error: %v", err) } if _, err := getFn(ctx, foo); err != nil { t.Fatalf("did not gracefully delete resource: %v", err) } // second delete duration is ignored _, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(expectedGrace+2)) if err != nil { t.Errorf("unexpected error: %v", err) } object, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name) if err != nil { t.Errorf("unexpected error, object should exist: %v", err) } objectMeta = t.getObjectMetaOrFail(object) if objectMeta.DeletionTimestamp == nil || objectMeta.DeletionGracePeriodSeconds == nil || *objectMeta.DeletionGracePeriodSeconds != expectedGrace { t.Errorf("unexpected deleted meta: %#v", objectMeta) } }
func (t *Tester) testDeleteNoGraceful(obj runtime.Object, createFn CreateFunc, getFn GetFunc, isNotFoundFn IsErrorFunc) { ctx := t.TestContext() foo := copyOrDie(obj) t.setObjectMeta(foo, t.namer(1)) if err := createFn(ctx, foo); err != nil { t.Errorf("unexpected error: %v", err) } objectMeta := t.getObjectMetaOrFail(foo) obj, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(10)) if err != nil { t.Errorf("unexpected error: %v", err) } if !t.returnDeletedObject { if status, ok := obj.(*unversioned.Status); !ok { t.Errorf("expected status of delete, got %v", status) } else if status.Status != unversioned.StatusSuccess { t.Errorf("expected success, got: %v", status.Status) } } _, err = getFn(ctx, foo) if err == nil || !isNotFoundFn(err) { t.Errorf("unexpected error: %v", err) } }
func New(kubeClient clientset.Interface, resyncPeriod controller.ResyncPeriodFunc, threshold int) *PodGCController { if kubeClient != nil && kubeClient.Core().GetRESTClient().GetRateLimiter() != nil { metrics.RegisterMetricAndTrackRateLimiterUsage("gc_controller", kubeClient.Core().GetRESTClient().GetRateLimiter()) } gcc := &PodGCController{ kubeClient: kubeClient, threshold: threshold, deletePod: func(namespace, name string) error { return kubeClient.Core().Pods(namespace).Delete(name, api.NewDeleteOptions(0)) }, } terminatedSelector := fields.ParseSelectorOrDie("status.phase!=" + string(api.PodPending) + ",status.phase!=" + string(api.PodRunning) + ",status.phase!=" + string(api.PodUnknown)) gcc.podStore.Indexer, gcc.podStoreSyncer = framework.NewIndexerInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { options.FieldSelector = terminatedSelector return gcc.kubeClient.Core().Pods(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { options.FieldSelector = terminatedSelector return gcc.kubeClient.Core().Pods(api.NamespaceAll).Watch(options) }, }, &api.Pod{}, resyncPeriod(), framework.ResourceEventHandlerFuncs{}, // We don't need to build a index for podStore here actually, but build one for consistency. // It will ensure that if people start making use of the podStore in more specific ways, // they'll get the benefits they expect. It will also reserve the name for future refactorings. cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, ) return gcc }
// terminatePods will ensure all pods on the given node that are in terminating state are eventually // cleaned up. Returns true if the node has no pods in terminating state, a duration that indicates how // long before we should check again (the next deadline for a pod to complete), or an error. func (nc *NodeController) terminatePods(nodeName string, since time.Time) (bool, time.Duration, error) { // the time before we should try again nextAttempt := time.Duration(0) // have we deleted all pods complete := true selector := fields.OneTermEqualSelector(api.PodHostField, nodeName) options := api.ListOptions{FieldSelector: selector} pods, err := nc.kubeClient.Core().Pods(api.NamespaceAll).List(options) if err != nil { return false, nextAttempt, err } now := time.Now() elapsed := now.Sub(since) for _, pod := range pods.Items { // Defensive check, also needed for tests. if pod.Spec.NodeName != nodeName { continue } // only clean terminated pods if pod.DeletionGracePeriodSeconds == nil { continue } // the user's requested grace period grace := time.Duration(*pod.DeletionGracePeriodSeconds) * time.Second if grace > nc.maximumGracePeriod { grace = nc.maximumGracePeriod } // the time remaining before the pod should have been deleted remaining := grace - elapsed if remaining < 0 { remaining = 0 glog.V(2).Infof("Removing pod %v after %s grace period", pod.Name, grace) nc.recordNodeEvent(nodeName, api.EventTypeNormal, "TerminatingEvictedPod", fmt.Sprintf("Pod %s has exceeded the grace period for deletion after being evicted from Node %q and is being force killed", pod.Name, nodeName)) if err := nc.kubeClient.Core().Pods(pod.Namespace).Delete(pod.Name, api.NewDeleteOptions(0)); err != nil { glog.Errorf("Error completing deletion of pod %s: %v", pod.Name, err) complete = false } } else { glog.V(2).Infof("Pod %v still terminating, requested grace period %s, %s remaining", pod.Name, grace, remaining) complete = false } if nextAttempt < remaining { nextAttempt = remaining } } return complete, nextAttempt, nil }
func (mc *basicMirrorClient) DeleteMirrorPod(podFullName string) error { if mc.apiserverClient == nil { return nil } name, namespace, err := kubecontainer.ParsePodFullName(podFullName) if err != nil { glog.Errorf("Failed to parse a pod full name %q", podFullName) return err } glog.V(4).Infof("Deleting a mirror pod %q", podFullName) // TODO(random-liu): Delete the mirror pod with uid precondition in mirror pod manager if err := mc.apiserverClient.Core().Pods(namespace).Delete(name, api.NewDeleteOptions(0)); err != nil && !errors.IsNotFound(err) { glog.Errorf("Failed deleting a mirror pod %q: %v", podFullName, err) } return nil }
func ReapResult(r *resource.Result, f *cmdutil.Factory, out io.Writer, isDefaultDelete, ignoreNotFound bool, timeout time.Duration, gracePeriod int, shortOutput bool, mapper meta.RESTMapper) error { found := 0 if ignoreNotFound { r = r.IgnoreErrors(errors.IsNotFound) } err := r.Visit(func(info *resource.Info, err error) error { if err != nil { return err } found++ reaper, err := f.Reaper(info.Mapping) if err != nil { // If there is no reaper for this resources and the user didn't explicitly ask for stop. if kubectl.IsNoSuchReaperError(err) && isDefaultDelete { return deleteResource(info, out, shortOutput, mapper) } return cmdutil.AddSourceToErr("reaping", info.Source, err) } var options *api.DeleteOptions if gracePeriod >= 0 { options = api.NewDeleteOptions(int64(gracePeriod)) } if err := reaper.Stop(info.Namespace, info.Name, timeout, options); err != nil { return cmdutil.AddSourceToErr("stopping", info.Source, err) } cmdutil.PrintSuccess(mapper, shortOutput, out, info.Mapping.Resource, info.Name, "deleted") return nil }) if err != nil { return err } if found == 0 { fmt.Fprintf(out, "No resources found\n") } return nil }
// Delete enforces life-cycle rules for namespace termination func (r *REST) Delete(ctx api.Context, name string, options *api.DeleteOptions) (runtime.Object, error) { nsObj, err := r.Get(ctx, name) if err != nil { return nil, err } namespace := nsObj.(*api.Namespace) // Ensure we have a UID precondition if options == nil { options = api.NewDeleteOptions(0) } if options.Preconditions == nil { options.Preconditions = &api.Preconditions{} } if options.Preconditions.UID == nil { options.Preconditions.UID = &namespace.UID } else if *options.Preconditions.UID != namespace.UID { err = apierrors.NewConflict( api.Resource("namespaces"), name, fmt.Errorf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *options.Preconditions.UID, namespace.UID), ) return nil, err } // upon first request to delete, we switch the phase to start namespace termination // TODO: enhance graceful deletion's calls to DeleteStrategy to allow phase change and finalizer patterns if namespace.DeletionTimestamp.IsZero() { key, err := r.Store.KeyFunc(ctx, name) if err != nil { return nil, err } preconditions := storage.Preconditions{UID: options.Preconditions.UID} out := r.Store.NewFunc() err = r.Store.Storage.GuaranteedUpdate( ctx, key, out, false, &preconditions, storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) { existingNamespace, ok := existing.(*api.Namespace) if !ok { // wrong type return nil, fmt.Errorf("expected *api.Namespace, got %v", existing) } // Set the deletion timestamp if needed if existingNamespace.DeletionTimestamp.IsZero() { now := unversioned.Now() existingNamespace.DeletionTimestamp = &now } // Set the namespace phase to terminating, if needed if existingNamespace.Status.Phase != api.NamespaceTerminating { existingNamespace.Status.Phase = api.NamespaceTerminating } return existingNamespace, nil }), ) if err != nil { err = storageerr.InterpretGetError(err, api.Resource("namespaces"), name) err = storageerr.InterpretUpdateError(err, api.Resource("namespaces"), name) if _, ok := err.(*apierrors.StatusError); !ok { err = apierrors.NewInternalError(err) } return nil, err } return out, nil } // prior to final deletion, we must ensure that finalizers is empty if len(namespace.Spec.Finalizers) != 0 { err = apierrors.NewConflict(api.Resource("namespaces"), namespace.Name, fmt.Errorf("The system is ensuring all content is removed from this namespace. Upon completion, this namespace will automatically be purged by the system.")) return nil, err } return r.Store.Delete(ctx, name, options) }