// This is to emulate the case where another party updates the object when
// etcdHelper.Delete has verified the preconditions, but hasn't carried out the
// deletion yet. Etcd will fail the deletion and report the conflict. etcdHelper
// should retry until there is no conflict.
func TestDeleteWithRetry(t *testing.T) {
server := etcdtesting.NewEtcdTestClientServer(t)
defer server.Terminate(t)
prefix := path.Join("/", etcdtest.PathPrefix())
obj := &api.Pod{ObjectMeta: api.ObjectMeta{Name: "foo", UID: "A"}}
// fakeGet returns a large ModifiedIndex to emulate the case that another
// party has updated the object.
fakeGet := func(ctx context.Context, key string, opts *etcd.GetOptions) (*etcd.Response, error) {
data, _ := runtime.Encode(testapi.Default.Codec(), obj)
return &etcd.Response{Node: &etcd.Node{Value: string(data), ModifiedIndex: 99}}, nil
}
expectedRetries := 3
helper := newEtcdHelper(server.Client, testapi.Default.Codec(), prefix)
fake := &fakeDeleteKeysAPI{KeysAPI: helper.etcdKeysAPI, fakeGetCap: expectedRetries, fakeGetFunc: fakeGet}
helper.etcdKeysAPI = fake
returnedObj := &api.Pod{}
err := helper.Create(context.TODO(), "/some/key", obj, returnedObj, 0)
if err != nil {
t.Errorf("Unexpected error %#v", err)
}
err = helper.Delete(context.TODO(), "/some/key", obj, storage.NewUIDPreconditions("A"))
if err != nil {
t.Errorf("Unexpected error %#v", err)
}
if fake.getCount != expectedRetries {
t.Errorf("Expect %d retries, got %d", expectedRetries, fake.getCount)
}
err = helper.Get(context.TODO(), "/some/key", obj, false)
if !storage.IsNotFound(err) {
t.Errorf("Expect an NotFound error, got %v", err)
}
}
// InterpretListError converts a generic error on a retrieval
// operation into the appropriate API error.
func InterpretListError(err error, qualifiedResource unversioned.GroupResource) error {
switch {
case storage.IsNotFound(err):
return errors.NewNotFound(qualifiedResource, "")
case storage.IsUnreachable(err):
return errors.NewServerTimeout(qualifiedResource, "list", 2) // TODO: make configurable or handled at a higher level
default:
return err
}
}
// InterpretDeleteError converts a generic error on a delete
// operation into the appropriate API error.
func InterpretDeleteError(err error, kind, name string) error {
switch {
case storage.IsNotFound(err):
return errors.NewNotFound(kind, name)
case storage.IsUnreachable(err):
return errors.NewServerTimeout(kind, "delete", 2) // TODO: make configurable or handled at a higher level
default:
return err
}
}
// InterpretGetError converts a generic error on a retrieval
// operation into the appropriate API error.
func InterpretGetError(err error, qualifiedResource schema.GroupResource, name string) error {
switch {
case storage.IsNotFound(err):
return errors.NewNotFound(qualifiedResource, name)
case storage.IsUnreachable(err):
return errors.NewServerTimeout(qualifiedResource, "get", 2) // TODO: make configurable or handled at a higher level
default:
return err
}
}
// InterpretUpdateError converts a generic error on a update
// operation into the appropriate API error.
func InterpretUpdateError(err error, qualifiedResource unversioned.GroupResource, name string) error {
switch {
case storage.IsTestFailed(err), storage.IsNodeExist(err):
return errors.NewConflict(qualifiedResource, name, err)
case storage.IsUnreachable(err):
return errors.NewServerTimeout(qualifiedResource, "update", 2) // TODO: make configurable or handled at a higher level
case storage.IsNotFound(err):
return errors.NewNotFound(qualifiedResource, name)
default:
return err
}
}
func TestGetNotFoundErr(t *testing.T) {
server := etcdtesting.NewEtcdTestClientServer(t)
defer server.Terminate(t)
boguskey := "/some/boguskey"
helper := newEtcdHelper(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix())
var got api.Pod
err := helper.Get(context.TODO(), boguskey, "", &got, false)
if !storage.IsNotFound(err) {
t.Errorf("Unexpected reponse on key=%v, err=%v", boguskey, err)
}
}
// InterpretDeleteError converts a generic error on a delete
// operation into the appropriate API error.
func InterpretDeleteError(err error, qualifiedResource schema.GroupResource, name string) error {
switch {
case storage.IsNotFound(err):
return errors.NewNotFound(qualifiedResource, name)
case storage.IsUnreachable(err):
return errors.NewServerTimeout(qualifiedResource, "delete", 2) // TODO: make configurable or handled at a higher level
case storage.IsConflict(err), storage.IsNodeExist(err), storage.IsInvalidObj(err):
return errors.NewConflict(qualifiedResource, name, err)
case storage.IsInternalError(err):
return errors.NewInternalError(err)
default:
return err
}
}
// Refresh reloads the RangeAllocation from etcd.
func (e *Etcd) Refresh() (*api.RangeAllocation, error) {
e.lock.Lock()
defer e.lock.Unlock()
existing := &api.RangeAllocation{}
if err := e.storage.Get(context.TODO(), e.baseKey, existing, false); err != nil {
if storage.IsNotFound(err) {
return nil, nil
}
return nil, etcderr.InterpretGetError(err, e.kind, "")
}
return existing, nil
}
func TestGet(t *testing.T) {
ctx, store, cluster := testSetup(t)
defer cluster.Terminate(t)
key, storedObj := testPropogateStore(t, store, ctx, &api.Pod{ObjectMeta: api.ObjectMeta{Name: "foo"}})
tests := []struct {
key string
ignoreNotFound bool
expectNotFoundErr bool
expectedOut *api.Pod
}{{ // test get on existing item
key: key,
ignoreNotFound: false,
expectNotFoundErr: false,
expectedOut: storedObj,
}, { // test get on non-existing item with ignoreNotFound=false
key: "/non-existing",
ignoreNotFound: false,
expectNotFoundErr: true,
}, { // test get on non-existing item with ignoreNotFound=true
key: "/non-existing",
ignoreNotFound: true,
expectNotFoundErr: false,
expectedOut: &api.Pod{},
}}
for i, tt := range tests {
out := &api.Pod{}
err := store.Get(ctx, tt.key, out, tt.ignoreNotFound)
if tt.expectNotFoundErr {
if err == nil || !storage.IsNotFound(err) {
t.Errorf("#%d: expecting not found error, but get: %s", i, err)
}
continue
}
if err != nil {
t.Fatalf("Get failed: %v", err)
}
if !reflect.DeepEqual(tt.expectedOut, out) {
t.Errorf("#%d: pod want=%#v, get=%#v", i, tt.expectedOut, out)
}
}
}
func (e *Store) deleteForEmptyFinalizers(ctx api.Context, name, key string, obj runtime.Object, preconditions *storage.Preconditions) (runtime.Object, bool, error) {
out := e.NewFunc()
glog.V(6).Infof("going to delete %s from regitry, triggered by update", name)
if err := e.Storage.Delete(ctx, key, out, preconditions); err != nil {
// Deletion is racy, i.e., there could be multiple update
// requests to remove all finalizers from the object, so we
// ignore the NotFound error.
if storage.IsNotFound(err) {
_, err := e.finalizeDelete(obj, true)
// clients are expecting an updated object if a PUT succeeded,
// but finalizeDelete returns a unversioned.Status, so return
// the object in the request instead.
return obj, false, err
}
return nil, false, storeerr.InterpretDeleteError(err, e.QualifiedResource, name)
}
_, err := e.finalizeDelete(out, true)
// clients are expecting an updated object if a PUT succeeded, but
// finalizeDelete returns a unversioned.Status, so return the object in
// the request instead.
return obj, false, err
}
func TestUnconditionalDelete(t *testing.T) {
ctx, store, cluster := testSetup(t)
defer cluster.Terminate(t)
key, storedObj := testPropogateStore(t, store, ctx, &api.Pod{ObjectMeta: api.ObjectMeta{Name: "foo"}})
tests := []struct {
key string
expectedObj *api.Pod
expectNotFoundErr bool
}{{ // test unconditional delete on existing key
key: key,
expectedObj: storedObj,
expectNotFoundErr: false,
}, { // test unconditional delete on non-existing key
key: "/non-existing",
expectedObj: nil,
expectNotFoundErr: true,
}}
for i, tt := range tests {
out := &api.Pod{} // reset
err := store.Delete(ctx, tt.key, out, nil)
if tt.expectNotFoundErr {
if err == nil || !storage.IsNotFound(err) {
t.Errorf("#%d: expecting not found error, but get: %s", i, err)
}
continue
}
if err != nil {
t.Fatalf("Delete failed: %v", err)
}
if !reflect.DeepEqual(tt.expectedObj, out) {
t.Errorf("#%d: pod want=%#v, get=%#v", i, tt.expectedObj, out)
}
}
}
func testInstallThirdPartyResourceRemove(t *testing.T, version string) {
master, etcdserver, server, assert := initThirdParty(t, version, "foo.company.com")
defer server.Close()
defer etcdserver.Terminate(t)
expectedObj := Foo{
ObjectMeta: api.ObjectMeta{
Name: "test",
},
TypeMeta: unversioned.TypeMeta{
Kind: "Foo",
},
SomeField: "test field",
OtherField: 10,
}
if !assert.NoError(createThirdPartyObject(master.thirdPartyStorage, "/ThirdPartyResourceData/company.com/foos/default/test", "test", expectedObj)) {
t.FailNow()
return
}
secondObj := expectedObj
secondObj.Name = "bar"
if !assert.NoError(createThirdPartyObject(master.thirdPartyStorage, "/ThirdPartyResourceData/company.com/foos/default/bar", "bar", secondObj)) {
t.FailNow()
return
}
resp, err := http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test")
if !assert.NoError(err) {
t.FailNow()
return
}
if resp.StatusCode != http.StatusOK {
t.Errorf("unexpected status: %v", resp)
}
item := Foo{}
if err := decodeResponse(resp, &item); err != nil {
t.Errorf("unexpected error: %v", err)
}
// TODO: validate etcd set things here
item.ObjectMeta = expectedObj.ObjectMeta
if !assert.True(reflect.DeepEqual(item, expectedObj)) {
t.Errorf("expected:\n%v\nsaw:\n%v\n", expectedObj, item)
}
path := makeThirdPartyPath("company.com")
master.RemoveThirdPartyResource(path)
resp, err = http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test")
if !assert.NoError(err) {
return
}
if resp.StatusCode != http.StatusNotFound {
t.Errorf("unexpected status: %v", resp)
}
expectedDeletedKeys := []string{
etcdtest.AddPrefix("/ThirdPartyResourceData/company.com/foos/default/test"),
etcdtest.AddPrefix("/ThirdPartyResourceData/company.com/foos/default/bar"),
}
for _, key := range expectedDeletedKeys {
thirdPartyObj := extensions.ThirdPartyResourceData{}
err := master.thirdPartyStorage.Get(context.TODO(), key, &thirdPartyObj, false)
if !storage.IsNotFound(err) {
t.Errorf("expected deletion didn't happen: %v", err)
}
}
installed := master.ListThirdPartyResources()
if len(installed) != 0 {
t.Errorf("Resource(s) still installed: %v", installed)
}
services := master.HandlerContainer.RegisteredWebServices()
for ix := range services {
if strings.HasPrefix(services[ix].RootPath(), "/apis/company.com") {
t.Errorf("Web service still installed at %s: %#v", services[ix].RootPath(), services[ix])
}
}
}
func testInstallThirdPartyAPIDeleteVersion(t *testing.T, version string) {
master, etcdserver, server, assert := initThirdParty(t, version, "foo.company.com")
defer server.Close()
defer etcdserver.Terminate(t)
expectedObj := Foo{
ObjectMeta: api.ObjectMeta{
Name: "test",
Namespace: "default",
},
TypeMeta: unversioned.TypeMeta{
Kind: "Foo",
},
SomeField: "test field",
OtherField: 10,
}
if !assert.NoError(createThirdPartyObject(master.thirdPartyStorage, "/ThirdPartyResourceData/company.com/foos/default/test", "test", expectedObj)) {
t.FailNow()
return
}
resp, err := http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test")
if !assert.NoError(err) {
return
}
assert.Equal(http.StatusOK, resp.StatusCode)
item := Foo{}
assert.NoError(decodeResponse(resp, &item))
// Fill in fields set by the apiserver
expectedObj.SelfLink = item.SelfLink
expectedObj.ResourceVersion = item.ResourceVersion
expectedObj.Namespace = item.Namespace
if !assert.True(reflect.DeepEqual(item, expectedObj)) {
t.Errorf("expected:\n%v\nsaw:\n%v\n", expectedObj, item)
}
resp, err = httpDelete(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test")
if !assert.NoError(err) {
return
}
assert.Equal(http.StatusOK, resp.StatusCode)
resp, err = http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test")
if !assert.NoError(err) {
return
}
assert.Equal(http.StatusNotFound, resp.StatusCode)
expectedDeletedKey := etcdtest.AddPrefix("ThirdPartyResourceData/company.com/foos/default/test")
thirdPartyObj := extensions.ThirdPartyResourceData{}
err = master.thirdPartyStorage.Get(
context.TODO(), expectedDeletedKey, &thirdPartyObj, false)
if !storage.IsNotFound(err) {
t.Errorf("expected deletion didn't happen: %v", err)
}
}
func TestGuaranteedUpdate(t *testing.T) {
ctx, store, cluster := testSetup(t)
defer cluster.Terminate(t)
key, storeObj := testPropogateStore(t, store, ctx, &api.Pod{ObjectMeta: api.ObjectMeta{Name: "foo", UID: "A"}})
tests := []struct {
key string
name string
ignoreNotFound bool
precondition *storage.Preconditions
expectNotFoundErr bool
expectInvalidObjErr bool
expectNoUpdate bool
}{{ // GuaranteedUpdate on non-existing key with ignoreNotFound=false
key: "/non-existing",
ignoreNotFound: false,
precondition: nil,
expectNotFoundErr: true,
expectInvalidObjErr: false,
expectNoUpdate: false,
}, { // GuaranteedUpdate on non-existing key with ignoreNotFound=true
key: "/non-existing",
ignoreNotFound: true,
precondition: nil,
expectNotFoundErr: false,
expectInvalidObjErr: false,
expectNoUpdate: false,
}, { // GuaranteedUpdate on existing key
key: key,
ignoreNotFound: false,
precondition: nil,
expectNotFoundErr: false,
expectInvalidObjErr: false,
expectNoUpdate: false,
}, { // GuaranteedUpdate with same data
key: key,
ignoreNotFound: false,
precondition: nil,
expectNotFoundErr: false,
expectInvalidObjErr: false,
expectNoUpdate: true,
}, { // GuaranteedUpdate with UID match
key: key,
ignoreNotFound: false,
precondition: storage.NewUIDPreconditions("A"),
expectNotFoundErr: false,
expectInvalidObjErr: false,
expectNoUpdate: true,
}, { // GuaranteedUpdate with UID mismatch
key: key,
ignoreNotFound: false,
precondition: storage.NewUIDPreconditions("B"),
expectNotFoundErr: false,
expectInvalidObjErr: true,
expectNoUpdate: true,
}}
for i, tt := range tests {
out := &api.Pod{}
name := fmt.Sprintf("foo-%d", i)
if tt.expectNoUpdate {
name = storeObj.Name
}
version := storeObj.ResourceVersion
err := store.GuaranteedUpdate(ctx, tt.key, out, tt.ignoreNotFound, tt.precondition,
storage.SimpleUpdate(func(obj runtime.Object) (runtime.Object, error) {
if tt.expectNotFoundErr && tt.ignoreNotFound {
if pod := obj.(*api.Pod); pod.Name != "" {
t.Errorf("#%d: expecting zero value, but get=%#v", i, pod)
}
}
pod := *storeObj
pod.Name = name
return &pod, nil
}))
if tt.expectNotFoundErr {
if err == nil || !storage.IsNotFound(err) {
t.Errorf("#%d: expecting not found error, but get: %v", i, err)
}
continue
}
if tt.expectInvalidObjErr {
if err == nil || !storage.IsInvalidObj(err) {
t.Errorf("#%d: expecting invalid UID error, but get: %s", i, err)
}
continue
}
if err != nil {
t.Fatalf("GuaranteedUpdate failed: %v", err)
}
if out.ObjectMeta.Name != name {
t.Errorf("#%d: pod name want=%s, get=%s", i, name, out.ObjectMeta.Name)
}
switch tt.expectNoUpdate {
case true:
if version != out.ResourceVersion {
t.Errorf("#%d: expect no version change, before=%s, after=%s", i, version, out.ResourceVersion)
}
case false:
if version == out.ResourceVersion {
t.Errorf("#%d: expect version change, but get the same version=%s", i, version)
}
}
storeObj = out
}
}
// Delete removes the item from storage.
func (e *Store) Delete(ctx api.Context, name string, options *api.DeleteOptions) (runtime.Object, error) {
key, err := e.KeyFunc(ctx, name)
if err != nil {
return nil, err
}
obj := e.NewFunc()
if err := e.Storage.Get(ctx, key, obj, false); err != nil {
return nil, storeerr.InterpretDeleteError(err, e.QualifiedResource, name)
}
// support older consumers of delete by treating "nil" as delete immediately
if options == nil {
options = api.NewDeleteOptions(0)
}
var preconditions storage.Preconditions
if options.Preconditions != nil {
preconditions.UID = options.Preconditions.UID
}
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, obj, options)
if err != nil {
return nil, err
}
if pendingGraceful {
return e.finalizeDelete(obj, false)
}
var ignoreNotFound bool = false
var lastExisting runtime.Object = nil
if graceful {
out := e.NewFunc()
lastGraceful := int64(0)
err := e.Storage.GuaranteedUpdate(
ctx, key, out, false, &preconditions,
storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, existing, options)
if err != nil {
return nil, err
}
if pendingGraceful {
return nil, errAlreadyDeleting
}
if !graceful {
return nil, errDeleteNow
}
lastGraceful = *options.GracePeriodSeconds
lastExisting = existing
return existing, nil
}),
)
switch err {
case nil:
if lastGraceful > 0 {
return out, nil
}
// If we are here, the registry supports grace period mechanism and
// we are intentionally delete gracelessly. In this case, we may
// enter a race with other k8s components. If other component wins
// the race, the object will not be found, and we should tolerate
// the NotFound error. See
// https://github.com/kubernetes/kubernetes/issues/19403 for
// details.
ignoreNotFound = true
// exit the switch and delete immediately
case errDeleteNow:
// we've updated the object to have a zero grace period, or it's already at 0, so
// we should fall through and truly delete the object.
case errAlreadyDeleting:
return e.finalizeDelete(obj, true)
default:
return nil, storeerr.InterpretUpdateError(err, e.QualifiedResource, name)
}
}
// delete immediately, or no graceful deletion supported
out := e.NewFunc()
if err := e.Storage.Delete(ctx, key, out, &preconditions); err != nil {
// Please refer to the place where we set ignoreNotFound for the reason
// why we ignore the NotFound error .
if storage.IsNotFound(err) && ignoreNotFound && lastExisting != nil {
// The lastExisting object may not be the last state of the object
// before its deletion, but it's the best approximation.
return e.finalizeDelete(lastExisting, true)
}
return nil, storeerr.InterpretDeleteError(err, e.QualifiedResource, name)
}
return e.finalizeDelete(out, true)
}
// Delete removes the item from storage.
func (e *Store) Delete(ctx api.Context, name string, options *api.DeleteOptions) (runtime.Object, error) {
key, err := e.KeyFunc(ctx, name)
if err != nil {
return nil, err
}
obj := e.NewFunc()
if err := e.Storage.Get(ctx, key, obj, false); err != nil {
return nil, storeerr.InterpretDeleteError(err, e.QualifiedResource, name)
}
// support older consumers of delete by treating "nil" as delete immediately
if options == nil {
options = api.NewDeleteOptions(0)
}
var preconditions storage.Preconditions
if options.Preconditions != nil {
preconditions.UID = options.Preconditions.UID
}
graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, obj, options)
if err != nil {
return nil, err
}
// this means finalizers cannot be updated via DeleteOptions if a deletion is already pending
if pendingGraceful {
return e.finalizeDelete(obj, false)
}
// check if obj has pending finalizers
accessor, err := meta.Accessor(obj)
if err != nil {
return nil, kubeerr.NewInternalError(err)
}
pendingFinalizers := len(accessor.GetFinalizers()) != 0
var ignoreNotFound bool
var deleteImmediately bool = true
var lastExisting, out runtime.Object
if !EnableGarbageCollector {
// TODO: remove the check on graceful, because we support no-op updates now.
if graceful {
err, ignoreNotFound, deleteImmediately, out, lastExisting = e.updateForGracefulDeletion(ctx, name, key, options, preconditions, obj)
}
} else {
shouldUpdateFinalizers, _ := shouldUpdateFinalizers(accessor, options)
// TODO: remove the check, because we support no-op updates now.
if graceful || pendingFinalizers || shouldUpdateFinalizers {
err, ignoreNotFound, deleteImmediately, out, lastExisting = e.updateForGracefulDeletionAndFinalizers(ctx, name, key, options, preconditions, obj)
}
}
// !deleteImmediately covers all cases where err != nil. We keep both to be future-proof.
if !deleteImmediately || err != nil {
return out, err
}
// delete immediately, or no graceful deletion supported
glog.V(6).Infof("going to delete %s from regitry: ", name)
out = e.NewFunc()
if err := e.Storage.Delete(ctx, key, out, &preconditions); err != nil {
// Please refer to the place where we set ignoreNotFound for the reason
// why we ignore the NotFound error .
if storage.IsNotFound(err) && ignoreNotFound && lastExisting != nil {
// The lastExisting object may not be the last state of the object
// before its deletion, but it's the best approximation.
return e.finalizeDelete(lastExisting, true)
}
return nil, storeerr.InterpretDeleteError(err, e.QualifiedResource, name)
}
return e.finalizeDelete(out, true)
}
func TestGet(t *testing.T) {
server, etcdStorage := newEtcdTestStorage(t, testapi.Default.Codec(), etcdtest.PathPrefix())
defer server.Terminate(t)
cacher := newTestCacher(etcdStorage, 10)
defer cacher.Stop()
podFoo := makeTestPod("foo")
fooCreated := updatePod(t, etcdStorage, podFoo, nil)
// We pass the ResourceVersion from the above Create() operation.
result := &api.Pod{}
if err := cacher.Get(context.TODO(), "pods/ns/foo", fooCreated.ResourceVersion, result, true); err != nil {
t.Errorf("Unexpected error: %v", err)
}
if e, a := *fooCreated, *result; !reflect.DeepEqual(e, a) {
t.Errorf("Expected: %#v, got: %#v", e, a)
}
if err := cacher.Get(context.TODO(), "pods/ns/bar", fooCreated.ResourceVersion, result, true); err != nil {
t.Errorf("Unexpected error: %v", err)
}
emptyPod := api.Pod{}
if e, a := emptyPod, *result; !reflect.DeepEqual(e, a) {
t.Errorf("Expected: %#v, got: %#v", e, a)
}
if err := cacher.Get(context.TODO(), "pods/ns/bar", fooCreated.ResourceVersion, result, false); !storage.IsNotFound(err) {
t.Errorf("Unexpected error: %v", err)
}
}