// TestGracefulStoreCanDeleteIfExistingGracePeriodZero tests recovery from
// race condition where the graceful delete is unable to complete
// in prior operation, but the pod remains with deletion timestamp
// and grace period set to 0.
func TestGracefulStoreCanDeleteIfExistingGracePeriodZero(t *testing.T) {
deletionTimestamp := metav1.NewTime(time.Now())
deletionGracePeriodSeconds := int64(0)
initialGeneration := int64(1)
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Generation: initialGeneration,
DeletionGracePeriodSeconds: &deletionGracePeriodSeconds,
DeletionTimestamp: &deletionTimestamp,
},
Spec: api.PodSpec{NodeName: "machine"},
}
testContext := api.WithNamespace(api.NewContext(), "test")
destroyFunc, registry := NewTestGenericStoreRegistry(t)
registry.EnableGarbageCollection = false
defaultDeleteStrategy := testRESTStrategy{api.Scheme, api.SimpleNameGenerator, true, false, true}
registry.DeleteStrategy = testGracefulStrategy{defaultDeleteStrategy}
defer destroyFunc()
graceful, gracefulPending, err := rest.BeforeDelete(registry.DeleteStrategy, testContext, pod, api.NewDeleteOptions(0))
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if graceful {
t.Fatalf("graceful should be false if object has DeletionTimestamp and DeletionGracePeriodSeconds is 0")
}
if gracefulPending {
t.Fatalf("gracefulPending should be false if object has DeletionTimestamp and DeletionGracePeriodSeconds is 0")
}
}
// this functions need to be kept synced with updateForGracefulDeletionAndFinalizers.
func (e *Store) updateForGracefulDeletion(ctx api.Context, name, key string, options *api.DeleteOptions, preconditions storage.Preconditions, in runtime.Object) (err error, ignoreNotFound, deleteImmediately bool, out, lastExisting runtime.Object) {
lastGraceful := int64(0)
out = e.NewFunc()
err = e.Storage.GuaranteedUpdate(
ctx, key, out, false, &preconditions,
storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, existing, options)
if err != nil {
return nil, err
}
if pendingGraceful {
return nil, errAlreadyDeleting
}
if !graceful {
return nil, errDeleteNow
}
lastGraceful = *options.GracePeriodSeconds
lastExisting = existing
return existing, nil
}),
)
switch err {
case nil:
if lastGraceful > 0 {
return nil, false, false, out, lastExisting
}
// If we are here, the registry supports grace period mechanism and
// we are intentionally delete gracelessly. In this case, we may
// enter a race with other k8s components. If other component wins
// the race, the object will not be found, and we should tolerate
// the NotFound error. See
// https://github.com/kubernetes/kubernetes/issues/19403 for
// details.
return nil, true, true, out, lastExisting
case errDeleteNow:
// we've updated the object to have a zero grace period, or it's already at 0, so
// we should fall through and truly delete the object.
return nil, false, true, out, lastExisting
case errAlreadyDeleting:
out, err = e.finalizeDelete(in, true)
return err, false, false, out, lastExisting
default:
return storeerr.InterpretUpdateError(err, e.QualifiedResource, name), false, false, out, lastExisting
}
}
// Delete removes the item from etcd.
func (e *Etcd) Delete(ctx api.Context, name string, options *api.DeleteOptions) (runtime.Object, error) {
key, err := e.KeyFunc(ctx, name)
if err != nil {
return nil, err
}
obj := e.NewFunc()
trace := util.NewTrace("Delete " + reflect.TypeOf(obj).String())
defer trace.LogIfLong(time.Second)
trace.Step("About to read object")
if err := e.Storage.Get(key, obj, false); err != nil {
return nil, etcderr.InterpretDeleteError(err, e.EndpointName, name)
}
// support older consumers of delete by treating "nil" as delete immediately
if options == nil {
options = api.NewDeleteOptions(0)
}
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, obj, options)
if err != nil {
return nil, err
}
if pendingGraceful {
return e.finalizeDelete(obj, false)
}
if graceful && *options.GracePeriodSeconds != 0 {
trace.Step("Graceful deletion")
out := e.NewFunc()
if err := e.Storage.Set(key, obj, out, uint64(*options.GracePeriodSeconds)); err != nil {
return nil, etcderr.InterpretUpdateError(err, e.EndpointName, name)
}
return e.finalizeDelete(out, true)
}
// delete immediately, or no graceful deletion supported
out := e.NewFunc()
trace.Step("About to delete object")
if err := e.Storage.Delete(key, out); err != nil {
return nil, etcderr.InterpretDeleteError(err, e.EndpointName, name)
}
return e.finalizeDelete(out, true)
}
// Delete removes the item from storage.
func (e *Store) Delete(ctx api.Context, name string, options *api.DeleteOptions) (runtime.Object, error) {
key, err := e.KeyFunc(ctx, name)
if err != nil {
return nil, err
}
obj := e.NewFunc()
if err := e.Storage.Get(ctx, key, obj, false); err != nil {
return nil, storeerr.InterpretDeleteError(err, e.QualifiedResource, name)
}
// support older consumers of delete by treating "nil" as delete immediately
if options == nil {
options = api.NewDeleteOptions(0)
}
var preconditions storage.Preconditions
if options.Preconditions != nil {
preconditions.UID = options.Preconditions.UID
}
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, obj, options)
if err != nil {
return nil, err
}
// this means finalizers cannot be updated via DeleteOptions if a deletion is already pending
if pendingGraceful {
return e.finalizeDelete(obj, false)
}
// check if obj has pending finalizers
accessor, err := meta.Accessor(obj)
if err != nil {
return nil, kubeerr.NewInternalError(err)
}
pendingFinalizers := len(accessor.GetFinalizers()) != 0
var ignoreNotFound bool
var deleteImmediately bool = true
var lastExisting, out runtime.Object
if !EnableGarbageCollector {
// TODO: remove the check on graceful, because we support no-op updates now.
if graceful {
err, ignoreNotFound, deleteImmediately, out, lastExisting = e.updateForGracefulDeletion(ctx, name, key, options, preconditions, obj)
}
} else {
shouldUpdateFinalizers, _ := shouldUpdateFinalizers(accessor, options)
// TODO: remove the check, because we support no-op updates now.
if graceful || pendingFinalizers || shouldUpdateFinalizers {
err, ignoreNotFound, deleteImmediately, out, lastExisting = e.updateForGracefulDeletionAndFinalizers(ctx, name, key, options, preconditions, obj)
}
}
// !deleteImmediately covers all cases where err != nil. We keep both to be future-proof.
if !deleteImmediately || err != nil {
return out, err
}
// delete immediately, or no graceful deletion supported
glog.V(6).Infof("going to delete %s from regitry: ", name)
out = e.NewFunc()
if err := e.Storage.Delete(ctx, key, out, &preconditions); err != nil {
// Please refer to the place where we set ignoreNotFound for the reason
// why we ignore the NotFound error .
if storage.IsNotFound(err) && ignoreNotFound && lastExisting != nil {
// The lastExisting object may not be the last state of the object
// before its deletion, but it's the best approximation.
return e.finalizeDelete(lastExisting, true)
}
return nil, storeerr.InterpretDeleteError(err, e.QualifiedResource, name)
}
return e.finalizeDelete(out, true)
}
// this functions need to be kept synced with updateForGracefulDeletion.
func (e *Store) updateForGracefulDeletionAndFinalizers(ctx api.Context, name, key string, options *api.DeleteOptions, preconditions storage.Preconditions, in runtime.Object) (err error, ignoreNotFound, deleteImmediately bool, out, lastExisting runtime.Object) {
lastGraceful := int64(0)
var pendingFinalizers bool
out = e.NewFunc()
err = e.Storage.GuaranteedUpdate(
ctx, key, out, false, &preconditions,
storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, existing, options)
if err != nil {
return nil, err
}
if pendingGraceful {
return nil, errAlreadyDeleting
}
// Add/remove the orphan finalizer as the options dictates.
// Note that this occurs after checking pendingGraceufl, so
// finalizers cannot be updated via DeleteOptions if deletion has
// started.
existingAccessor, err := meta.Accessor(existing)
if err != nil {
return nil, err
}
shouldUpdate, newFinalizers := shouldUpdateFinalizers(existingAccessor, options)
if shouldUpdate {
existingAccessor.SetFinalizers(newFinalizers)
}
if !graceful {
// set the DeleteGracePeriods to 0 if the object has pendingFinalizers but not supporting graceful deletion
pendingFinalizers = len(existingAccessor.GetFinalizers()) != 0
if pendingFinalizers {
glog.V(6).Infof("update the DeletionTimestamp to \"now\" and GracePeriodSeconds to 0 for object %s, because it has pending finalizers", name)
err = markAsDeleting(existing)
if err != nil {
return nil, err
}
return existing, nil
}
return nil, errDeleteNow
}
lastGraceful = *options.GracePeriodSeconds
lastExisting = existing
return existing, nil
}),
)
switch err {
case nil:
// If there are pending finalizers, we never delete the object immediately.
if pendingFinalizers {
return nil, false, false, out, lastExisting
}
if lastGraceful > 0 {
return nil, false, false, out, lastExisting
}
// If we are here, the registry supports grace period mechanism and
// we are intentionally delete gracelessly. In this case, we may
// enter a race with other k8s components. If other component wins
// the race, the object will not be found, and we should tolerate
// the NotFound error. See
// https://github.com/kubernetes/kubernetes/issues/19403 for
// details.
return nil, true, true, out, lastExisting
case errDeleteNow:
// we've updated the object to have a zero grace period, or it's already at 0, so
// we should fall through and truly delete the object.
return nil, false, true, out, lastExisting
case errAlreadyDeleting:
out, err = e.finalizeDelete(in, true)
return err, false, false, out, lastExisting
default:
return storeerr.InterpretUpdateError(err, e.QualifiedResource, name), false, false, out, lastExisting
}
}
// Delete removes the item from etcd.
func (e *Etcd) Delete(ctx api.Context, name string, options *api.DeleteOptions) (runtime.Object, error) {
key, err := e.KeyFunc(ctx, name)
if err != nil {
return nil, err
}
obj := e.NewFunc()
trace := util.NewTrace("Delete " + reflect.TypeOf(obj).String())
defer trace.LogIfLong(time.Second)
trace.Step("About to read object")
if err := e.Storage.Get(ctx, key, obj, false); err != nil {
return nil, etcderr.InterpretDeleteError(err, e.EndpointName, name)
}
// support older consumers of delete by treating "nil" as delete immediately
if options == nil {
options = api.NewDeleteOptions(0)
}
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, obj, options)
if err != nil {
return nil, err
}
if pendingGraceful {
return e.finalizeDelete(obj, false)
}
if graceful {
trace.Step("Graceful deletion")
out := e.NewFunc()
lastGraceful := int64(0)
err := e.Storage.GuaranteedUpdate(
ctx, key, out, false,
storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, existing, options)
if err != nil {
return nil, err
}
if pendingGraceful {
return nil, errAlreadyDeleting
}
if !graceful {
return nil, errDeleteNow
}
lastGraceful = *options.GracePeriodSeconds
return existing, nil
}),
)
switch err {
case nil:
if lastGraceful > 0 {
return out, nil
}
// fall through and delete immediately
case errDeleteNow:
// we've updated the object to have a zero grace period, or it's already at 0, so
// we should fall through and truly delete the object.
case errAlreadyDeleting:
return e.finalizeDelete(obj, true)
default:
return nil, etcderr.InterpretUpdateError(err, e.EndpointName, name)
}
}
// delete immediately, or no graceful deletion supported
out := e.NewFunc()
trace.Step("About to delete object")
if err := e.Storage.Delete(ctx, key, out); err != nil {
return nil, etcderr.InterpretDeleteError(err, e.EndpointName, name)
}
return e.finalizeDelete(out, true)
}
// Delete removes the item from storage.
func (e *Store) Delete(ctx api.Context, name string, options *api.DeleteOptions) (runtime.Object, error) {
key, err := e.KeyFunc(ctx, name)
if err != nil {
return nil, err
}
obj := e.NewFunc()
if err := e.Storage.Get(ctx, key, obj, false); err != nil {
return nil, storeerr.InterpretDeleteError(err, e.QualifiedResource, name)
}
// support older consumers of delete by treating "nil" as delete immediately
if options == nil {
options = api.NewDeleteOptions(0)
}
var preconditions storage.Preconditions
if options.Preconditions != nil {
preconditions.UID = options.Preconditions.UID
}
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, obj, options)
if err != nil {
return nil, err
}
if pendingGraceful {
return e.finalizeDelete(obj, false)
}
var ignoreNotFound bool = false
var lastExisting runtime.Object = nil
if graceful {
out := e.NewFunc()
lastGraceful := int64(0)
err := e.Storage.GuaranteedUpdate(
ctx, key, out, false, &preconditions,
storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, existing, options)
if err != nil {
return nil, err
}
if pendingGraceful {
return nil, errAlreadyDeleting
}
if !graceful {
return nil, errDeleteNow
}
lastGraceful = *options.GracePeriodSeconds
lastExisting = existing
return existing, nil
}),
)
switch err {
case nil:
if lastGraceful > 0 {
return out, nil
}
// If we are here, the registry supports grace period mechanism and
// we are intentionally delete gracelessly. In this case, we may
// enter a race with other k8s components. If other component wins
// the race, the object will not be found, and we should tolerate
// the NotFound error. See
// https://github.com/kubernetes/kubernetes/issues/19403 for
// details.
ignoreNotFound = true
// exit the switch and delete immediately
case errDeleteNow:
// we've updated the object to have a zero grace period, or it's already at 0, so
// we should fall through and truly delete the object.
case errAlreadyDeleting:
return e.finalizeDelete(obj, true)
default:
return nil, storeerr.InterpretUpdateError(err, e.QualifiedResource, name)
}
}
// delete immediately, or no graceful deletion supported
out := e.NewFunc()
if err := e.Storage.Delete(ctx, key, out, &preconditions); err != nil {
// Please refer to the place where we set ignoreNotFound for the reason
// why we ignore the NotFound error .
if storage.IsNotFound(err) && ignoreNotFound && lastExisting != nil {
// The lastExisting object may not be the last state of the object
// before its deletion, but it's the best approximation.
return e.finalizeDelete(lastExisting, true)
}
return nil, storeerr.InterpretDeleteError(err, e.QualifiedResource, name)
}
return e.finalizeDelete(out, true)
}