func (t *Tester) testDeleteGracefulImmediate(obj runtime.Object, createFn CreateFunc, getFn GetFunc, expectedGrace int64) {
ctx := t.TestContext()
foo := copyOrDie(obj)
t.setObjectMeta(foo, "foo4")
if err := createFn(ctx, foo); err != nil {
t.Errorf("unexpected error: %v", err)
}
objectMeta := t.getObjectMetaOrFail(foo)
_, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(expectedGrace))
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if _, err := getFn(ctx, foo); err != nil {
t.Fatalf("did not gracefully delete resource: %v", err)
}
// second delete is immediate, resource is deleted
out, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(0))
if err != nil {
t.Errorf("unexpected error: %v", err)
}
_, err = t.storage.(rest.Getter).Get(ctx, objectMeta.Name)
if !errors.IsNotFound(err) {
t.Errorf("unexpected error, object should be deleted immediately: %v", err)
}
objectMeta = t.getObjectMetaOrFail(out)
// the second delete shouldn't update the object, so the objectMeta.DeletionGracePeriodSeconds should eqaul to the value set in the first delete.
if objectMeta.DeletionTimestamp == nil || objectMeta.DeletionGracePeriodSeconds == nil || *objectMeta.DeletionGracePeriodSeconds != 0 {
t.Errorf("unexpected deleted meta: %#v", objectMeta)
}
}
func (t *Tester) testDeleteGracefulExtend(obj runtime.Object, createFn CreateFunc, getFn GetFunc, expectedGrace int64) {
ctx := t.TestContext()
foo := copyOrDie(obj)
t.setObjectMeta(foo, t.namer(3))
if err := createFn(ctx, foo); err != nil {
t.Errorf("unexpected error: %v", err)
}
objectMeta := t.getObjectMetaOrFail(foo)
_, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(expectedGrace))
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if _, err := getFn(ctx, foo); err != nil {
t.Fatalf("did not gracefully delete resource: %v", err)
}
// second delete duration is ignored
_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(expectedGrace+2))
if err != nil {
t.Errorf("unexpected error: %v", err)
}
object, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name)
if err != nil {
t.Errorf("unexpected error, object should exist: %v", err)
}
objectMeta = t.getObjectMetaOrFail(object)
if objectMeta.DeletionTimestamp == nil || objectMeta.DeletionGracePeriodSeconds == nil || *objectMeta.DeletionGracePeriodSeconds != expectedGrace {
t.Errorf("unexpected deleted meta: %#v", objectMeta)
}
}
func (t *Tester) testDeleteNoGraceful(obj runtime.Object, createFn CreateFunc, getFn GetFunc, isNotFoundFn IsErrorFunc) {
ctx := t.TestContext()
foo := copyOrDie(obj)
t.setObjectMeta(foo, t.namer(1))
if err := createFn(ctx, foo); err != nil {
t.Errorf("unexpected error: %v", err)
}
objectMeta := t.getObjectMetaOrFail(foo)
obj, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(10))
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if !t.returnDeletedObject {
if status, ok := obj.(*unversioned.Status); !ok {
t.Errorf("expected status of delete, got %v", status)
} else if status.Status != unversioned.StatusSuccess {
t.Errorf("expected success, got: %v", status.Status)
}
}
_, err = getFn(ctx, foo)
if err == nil || !isNotFoundFn(err) {
t.Errorf("unexpected error: %v", err)
}
}
func New(kubeClient clientset.Interface, resyncPeriod controller.ResyncPeriodFunc, threshold int) *PodGCController {
if kubeClient != nil && kubeClient.Core().GetRESTClient().GetRateLimiter() != nil {
metrics.RegisterMetricAndTrackRateLimiterUsage("gc_controller", kubeClient.Core().GetRESTClient().GetRateLimiter())
}
gcc := &PodGCController{
kubeClient: kubeClient,
threshold: threshold,
deletePod: func(namespace, name string) error {
return kubeClient.Core().Pods(namespace).Delete(name, api.NewDeleteOptions(0))
},
}
terminatedSelector := fields.ParseSelectorOrDie("status.phase!=" + string(api.PodPending) + ",status.phase!=" + string(api.PodRunning) + ",status.phase!=" + string(api.PodUnknown))
gcc.podStore.Indexer, gcc.podStoreSyncer = framework.NewIndexerInformer(
&cache.ListWatch{
ListFunc: func(options api.ListOptions) (runtime.Object, error) {
options.FieldSelector = terminatedSelector
return gcc.kubeClient.Core().Pods(api.NamespaceAll).List(options)
},
WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
options.FieldSelector = terminatedSelector
return gcc.kubeClient.Core().Pods(api.NamespaceAll).Watch(options)
},
},
&api.Pod{},
resyncPeriod(),
framework.ResourceEventHandlerFuncs{},
// We don't need to build a index for podStore here actually, but build one for consistency.
// It will ensure that if people start making use of the podStore in more specific ways,
// they'll get the benefits they expect. It will also reserve the name for future refactorings.
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
)
return gcc
}
// terminatePods will ensure all pods on the given node that are in terminating state are eventually
// cleaned up. Returns true if the node has no pods in terminating state, a duration that indicates how
// long before we should check again (the next deadline for a pod to complete), or an error.
func (nc *NodeController) terminatePods(nodeName string, since time.Time) (bool, time.Duration, error) {
// the time before we should try again
nextAttempt := time.Duration(0)
// have we deleted all pods
complete := true
selector := fields.OneTermEqualSelector(api.PodHostField, nodeName)
options := api.ListOptions{FieldSelector: selector}
pods, err := nc.kubeClient.Core().Pods(api.NamespaceAll).List(options)
if err != nil {
return false, nextAttempt, err
}
now := time.Now()
elapsed := now.Sub(since)
for _, pod := range pods.Items {
// Defensive check, also needed for tests.
if pod.Spec.NodeName != nodeName {
continue
}
// only clean terminated pods
if pod.DeletionGracePeriodSeconds == nil {
continue
}
// the user's requested grace period
grace := time.Duration(*pod.DeletionGracePeriodSeconds) * time.Second
if grace > nc.maximumGracePeriod {
grace = nc.maximumGracePeriod
}
// the time remaining before the pod should have been deleted
remaining := grace - elapsed
if remaining < 0 {
remaining = 0
glog.V(2).Infof("Removing pod %v after %s grace period", pod.Name, grace)
nc.recordNodeEvent(nodeName, api.EventTypeNormal, "TerminatingEvictedPod", fmt.Sprintf("Pod %s has exceeded the grace period for deletion after being evicted from Node %q and is being force killed", pod.Name, nodeName))
if err := nc.kubeClient.Core().Pods(pod.Namespace).Delete(pod.Name, api.NewDeleteOptions(0)); err != nil {
glog.Errorf("Error completing deletion of pod %s: %v", pod.Name, err)
complete = false
}
} else {
glog.V(2).Infof("Pod %v still terminating, requested grace period %s, %s remaining", pod.Name, grace, remaining)
complete = false
}
if nextAttempt < remaining {
nextAttempt = remaining
}
}
return complete, nextAttempt, nil
}
func (mc *basicMirrorClient) DeleteMirrorPod(podFullName string) error {
if mc.apiserverClient == nil {
return nil
}
name, namespace, err := kubecontainer.ParsePodFullName(podFullName)
if err != nil {
glog.Errorf("Failed to parse a pod full name %q", podFullName)
return err
}
glog.V(4).Infof("Deleting a mirror pod %q", podFullName)
// TODO(random-liu): Delete the mirror pod with uid precondition in mirror pod manager
if err := mc.apiserverClient.Core().Pods(namespace).Delete(name, api.NewDeleteOptions(0)); err != nil && !errors.IsNotFound(err) {
glog.Errorf("Failed deleting a mirror pod %q: %v", podFullName, err)
}
return nil
}
func ReapResult(r *resource.Result, f *cmdutil.Factory, out io.Writer, isDefaultDelete, ignoreNotFound bool, timeout time.Duration, gracePeriod int, shortOutput bool, mapper meta.RESTMapper) error {
found := 0
if ignoreNotFound {
r = r.IgnoreErrors(errors.IsNotFound)
}
err := r.Visit(func(info *resource.Info, err error) error {
if err != nil {
return err
}
found++
reaper, err := f.Reaper(info.Mapping)
if err != nil {
// If there is no reaper for this resources and the user didn't explicitly ask for stop.
if kubectl.IsNoSuchReaperError(err) && isDefaultDelete {
return deleteResource(info, out, shortOutput, mapper)
}
return cmdutil.AddSourceToErr("reaping", info.Source, err)
}
var options *api.DeleteOptions
if gracePeriod >= 0 {
options = api.NewDeleteOptions(int64(gracePeriod))
}
if err := reaper.Stop(info.Namespace, info.Name, timeout, options); err != nil {
return cmdutil.AddSourceToErr("stopping", info.Source, err)
}
cmdutil.PrintSuccess(mapper, shortOutput, out, info.Mapping.Resource, info.Name, "deleted")
return nil
})
if err != nil {
return err
}
if found == 0 {
fmt.Fprintf(out, "No resources found\n")
}
return nil
}
// Delete enforces life-cycle rules for namespace termination
func (r *REST) Delete(ctx api.Context, name string, options *api.DeleteOptions) (runtime.Object, error) {
nsObj, err := r.Get(ctx, name)
if err != nil {
return nil, err
}
namespace := nsObj.(*api.Namespace)
// Ensure we have a UID precondition
if options == nil {
options = api.NewDeleteOptions(0)
}
if options.Preconditions == nil {
options.Preconditions = &api.Preconditions{}
}
if options.Preconditions.UID == nil {
options.Preconditions.UID = &namespace.UID
} else if *options.Preconditions.UID != namespace.UID {
err = apierrors.NewConflict(
api.Resource("namespaces"),
name,
fmt.Errorf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *options.Preconditions.UID, namespace.UID),
)
return nil, err
}
// upon first request to delete, we switch the phase to start namespace termination
// TODO: enhance graceful deletion's calls to DeleteStrategy to allow phase change and finalizer patterns
if namespace.DeletionTimestamp.IsZero() {
key, err := r.Store.KeyFunc(ctx, name)
if err != nil {
return nil, err
}
preconditions := storage.Preconditions{UID: options.Preconditions.UID}
out := r.Store.NewFunc()
err = r.Store.Storage.GuaranteedUpdate(
ctx, key, out, false, &preconditions,
storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
existingNamespace, ok := existing.(*api.Namespace)
if !ok {
// wrong type
return nil, fmt.Errorf("expected *api.Namespace, got %v", existing)
}
// Set the deletion timestamp if needed
if existingNamespace.DeletionTimestamp.IsZero() {
now := unversioned.Now()
existingNamespace.DeletionTimestamp = &now
}
// Set the namespace phase to terminating, if needed
if existingNamespace.Status.Phase != api.NamespaceTerminating {
existingNamespace.Status.Phase = api.NamespaceTerminating
}
return existingNamespace, nil
}),
)
if err != nil {
err = storageerr.InterpretGetError(err, api.Resource("namespaces"), name)
err = storageerr.InterpretUpdateError(err, api.Resource("namespaces"), name)
if _, ok := err.(*apierrors.StatusError); !ok {
err = apierrors.NewInternalError(err)
}
return nil, err
}
return out, nil
}
// prior to final deletion, we must ensure that finalizers is empty
if len(namespace.Spec.Finalizers) != 0 {
err = apierrors.NewConflict(api.Resource("namespaces"), namespace.Name, fmt.Errorf("The system is ensuring all content is removed from this namespace. Upon completion, this namespace will automatically be purged by the system."))
return nil, err
}
return r.Store.Delete(ctx, name, options)
}