// handleMaster performs one loop of master locking. // on success it returns <master>, nil // on error it returns "", err // in situations where you should try again due to concurrent state changes (e.g. another actor simultaneously acquiring the lock) // it returns "", nil func (e *etcdMasterElector) handleMaster(path, id string, ttl uint64) (string, error) { res, err := e.etcd.Get(context.TODO(), path, nil) // Unexpected error, bail out if err != nil && !etcdutil.IsEtcdNotFound(err) { return "", err } // There is no master, try to become the master. if err != nil && etcdutil.IsEtcdNotFound(err) { return e.becomeMaster(path, id, ttl) } // This should never happen. if res.Node == nil { return "", fmt.Errorf("unexpected response: %#v", res) } // We're not the master, just return the current value if res.Node.Value != id { return res.Node.Value, nil } // We are the master, try to extend out lease return e.extendMaster(path, id, ttl, res) }
func Store(api etcd.KeysAPI, path string, ttl time.Duration) frameworkid.Storage { // TODO(jdef) validate Config return &storage{ LookupFunc: func(ctx context.Context) (string, error) { if response, err := api.Get(ctx, path, nil); err != nil { if !etcdutil.IsEtcdNotFound(err) { return "", fmt.Errorf("unexpected failure attempting to load framework ID from etcd: %v", err) } } else { return response.Node.Value, nil } return "", nil }, RemoveFunc: func(ctx context.Context) (err error) { if _, err = api.Delete(ctx, path, &etcd.DeleteOptions{Recursive: true}); err != nil { if !etcdutil.IsEtcdNotFound(err) { return fmt.Errorf("failed to delete framework ID from etcd: %v", err) } } return }, StoreFunc: func(ctx context.Context, id string) (err error) { _, err = api.Set(ctx, path, id, &etcd.SetOptions{TTL: ttl}) return }, } }
// Implements storage.Interface. func (h *etcdHelper) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions) error { if ctx == nil { glog.Errorf("Context is nil") } key = h.prefixEtcdKey(key) v, err := conversion.EnforcePtr(out) if err != nil { panic("unable to convert output object to pointer") } if preconditions == nil { startTime := time.Now() response, err := h.etcdKeysAPI.Delete(ctx, key, nil) metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime) if !etcdutil.IsEtcdNotFound(err) { // if the object that existed prior to the delete is returned by etcd, update the out object. if err != nil || response.PrevNode != nil { _, _, err = h.extractObj(response, err, out, false, true) } } return toStorageErr(err, key, 0) } // Check the preconditions match. obj := reflect.New(v.Type()).Interface().(runtime.Object) for { _, node, res, err := h.bodyAndExtractObj(ctx, key, obj, false) if err != nil { return toStorageErr(err, key, 0) } if err := checkPreconditions(key, preconditions, obj); err != nil { return toStorageErr(err, key, 0) } index := uint64(0) if node != nil { index = node.ModifiedIndex } else if res != nil { index = res.Index } opt := etcd.DeleteOptions{PrevIndex: index} startTime := time.Now() response, err := h.etcdKeysAPI.Delete(ctx, key, &opt) metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime) if etcdutil.IsEtcdTestFailed(err) { glog.Infof("deletion of %s failed because of a conflict, going to retry", key) } else { if !etcdutil.IsEtcdNotFound(err) { // if the object that existed prior to the delete is returned by etcd, update the out object. if err != nil || response.PrevNode != nil { _, _, err = h.extractObj(response, err, out, false, true) } } return toStorageErr(err, key, 0) } } }
// InterpretDeleteError converts a generic etcd error on a delete // operation into the appropriate API error. func InterpretDeleteError(err error, kind, name string) error { switch { case etcdutil.IsEtcdNotFound(err): return errors.NewNotFound(kind, name) case etcdutil.IsEtcdUnreachable(err): return errors.NewServerTimeout(kind, "delete", 2) // TODO: make configurable or handled at a higher level default: return err } }
// TestEtcdClient verifies a client is functional. It will attempt to // connect to the etcd server and block until the server responds at least once, or return an // error if the server never responded. func TestEtcdClient(etcdClient etcdclient.Client) error { for i := 0; ; i++ { _, err := etcdclient.NewKeysAPI(etcdClient).Get(context.Background(), "/", nil) if err == nil || etcdutil.IsEtcdNotFound(err) { break } if i > 100 { return fmt.Errorf("could not reach etcd: %v", err) } time.Sleep(50 * time.Millisecond) } return nil }
func TestGetNotFoundErr(t *testing.T) { server := etcdtesting.NewEtcdTestClientServer(t) defer server.Terminate(t) key := etcdtest.AddPrefix("/some/key") boguskey := etcdtest.AddPrefix("/some/boguskey") helper := newEtcdHelper(server.Client, testapi.Default.Codec(), key) var got api.Pod err := helper.Get(context.TODO(), boguskey, &got, false) if !etcdutil.IsEtcdNotFound(err) { t.Errorf("Unexpected reponse on key=%v, err=%v", key, err) } }
// TestEtcdClient verifies a client is functional. It will attempt to // connect to the etcd server and block until the server responds at least once, or return an // error if the server never responded. func TestEtcdClient(etcdClient *etcdclient.Client) error { for i := 0; ; i++ { _, err := etcdClient.Get("/", false, false) if err == nil || etcdutil.IsEtcdNotFound(err) { break } if i > 100 { return fmt.Errorf("could not reach etcd: %v", err) } time.Sleep(50 * time.Millisecond) } return nil }
func (s *SchedulerServer) fetchFrameworkID(client tools.EtcdClient) (*mesos.FrameworkID, error) { if s.failoverTimeout > 0 { if response, err := client.Get(meta.FrameworkIDKey, false, false); err != nil { if !etcdutil.IsEtcdNotFound(err) { return nil, fmt.Errorf("unexpected failure attempting to load framework ID from etcd: %v", err) } log.V(1).Infof("did not find framework ID in etcd") } else if response.Node.Value != "" { log.Infof("configuring FrameworkInfo with Id found in etcd: '%s'", response.Node.Value) return mutil.NewFrameworkID(response.Node.Value), nil } } else { //TODO(jdef) this seems like a totally hackish way to clean up the framework ID if _, err := client.Delete(meta.FrameworkIDKey, true); err != nil { if !etcdutil.IsEtcdNotFound(err) { return nil, fmt.Errorf("failed to delete framework ID from etcd: %v", err) } log.V(1).Infof("nothing to delete: did not find framework ID in etcd") } } return nil, nil }
// Release tries to delete the leader lock. func (e *Etcd) Release() { for i := 0; i < e.maxRetries; i++ { _, err := e.client.Delete(context.Background(), e.key, &etcdclient.DeleteOptions{PrevValue: e.value}) if err == nil { break } // If the value has changed, we don't hold the lease. If the key is missing we don't // hold the lease. if etcdutil.IsEtcdTestFailed(err) || etcdutil.IsEtcdNotFound(err) { break } utilruntime.HandleError(fmt.Errorf("unable to release %s: %v", e.key, err)) } }
// Refresh reloads the RangeAllocation from etcd. func (e *Etcd) Refresh() (*api.RangeAllocation, error) { e.lock.Lock() defer e.lock.Unlock() existing := &api.RangeAllocation{} if err := e.storage.Get(context.TODO(), e.baseKey, existing, false); err != nil { if etcdutil.IsEtcdNotFound(err) { return nil, nil } return nil, etcderr.InterpretGetError(err, e.kind, "") } return existing, nil }
// bodyAndExtractObj performs the normal Get path to etcd, returning the parsed node and response for additional information // about the response, like the current etcd index and the ttl. func (h *etcdHelper) bodyAndExtractObj(ctx context.Context, key string, objPtr runtime.Object, ignoreNotFound bool) (body string, node *etcd.Node, res *etcd.Response, err error) { if ctx == nil { glog.Errorf("Context is nil") } startTime := time.Now() response, err := h.client.Get(ctx, key, nil) metrics.RecordEtcdRequestLatency("get", getTypeName(objPtr), startTime) if err != nil && !etcdutil.IsEtcdNotFound(err) { return "", nil, nil, err } body, node, err = h.extractObj(response, err, objPtr, ignoreNotFound, false) return body, node, response, err }
// Release tries to delete the leader lock. func (e *Etcd) Release() { for i := 0; i < e.maxRetries; i++ { _, err := e.client.CompareAndDelete(e.key, e.value, 0) if err == nil { break } // If the value has changed, we don't hold the lease. If the key is missing we don't // hold the lease. if etcdutil.IsEtcdTestFailed(err) || etcdutil.IsEtcdNotFound(err) { break } utilruntime.HandleError(fmt.Errorf("unable to release %s: %v", e.key, err)) } }
// etcdGetInitialWatchState turns an etcd Get request into a watch equivalent func etcdGetInitialWatchState(client *etcd.Client, key string, recursive bool, incoming chan<- *etcd.Response) (resourceVersion uint64, err error) { resp, err := client.Get(key, false, recursive) if err != nil { if !etcdutil.IsEtcdNotFound(err) { glog.Errorf("watch was unable to retrieve the current index for the provided key (%q): %v", key, err) return resourceVersion, err } if etcdError, ok := err.(*etcd.EtcdError); ok { resourceVersion = etcdError.Index } return resourceVersion, nil } resourceVersion = resp.EtcdIndex convertRecursiveResponse(resp.Node, resp, incoming) return }
func toStorageErr(err error, key string, rv int64) error { if err == nil { return nil } switch { case etcdutil.IsEtcdNotFound(err): return storage.NewKeyNotFoundError(key, rv) case etcdutil.IsEtcdNodeExist(err): return storage.NewKeyExistsError(key, rv) case etcdutil.IsEtcdTestFailed(err): return storage.NewResourceVersionConflictsError(key, rv) case etcdutil.IsEtcdUnreachable(err): return storage.NewUnreachableError(key, rv) default: return err } }
// bodyAndExtractObj performs the normal Get path to etcd, returning the parsed node and response for additional information // about the response, like the current etcd index and the ttl. func (h *etcdHelper) bodyAndExtractObj(ctx context.Context, key string, objPtr runtime.Object, ignoreNotFound bool) (body string, node *etcd.Node, res *etcd.Response, err error) { if ctx == nil { glog.Errorf("Context is nil") } startTime := time.Now() opts := &etcd.GetOptions{ Quorum: h.quorum, } response, err := h.etcdKeysAPI.Get(ctx, key, opts) metrics.RecordEtcdRequestLatency("get", getTypeName(objPtr), startTime) if err != nil && !etcdutil.IsEtcdNotFound(err) { return "", nil, nil, toStorageErr(err, key, 0) } body, node, err = h.extractObj(response, err, objPtr, ignoreNotFound, false) return body, node, response, toStorageErr(err, key, 0) }
func (h *etcdHelper) listEtcdNode(ctx context.Context, key string) ([]*etcd.Node, uint64, error) { if ctx == nil { glog.Errorf("Context is nil") } result, err := h.client.Get(key, true, true) if err != nil { var index uint64 if etcdError, ok := err.(*etcd.EtcdError); ok { index = etcdError.Index } nodes := make([]*etcd.Node, 0) if etcdutil.IsEtcdNotFound(err) { return nodes, index, nil } else { return nodes, index, err } } return result.Node.Nodes, result.EtcdIndex, nil }
// etcdGetInitialWatchState turns an etcd Get request into a watch equivalent func etcdGetInitialWatchState(ctx context.Context, client etcd.KeysAPI, key string, recursive bool, incoming chan<- *etcd.Response) (resourceVersion uint64, err error) { opts := etcd.GetOptions{ Recursive: recursive, Sort: false, } resp, err := client.Get(ctx, key, &opts) if err != nil { if !etcdutil.IsEtcdNotFound(err) { glog.Errorf("watch was unable to retrieve the current index for the provided key (%q): %v", key, err) return resourceVersion, err } if etcdError, ok := err.(etcd.Error); ok { resourceVersion = etcdError.Index } return resourceVersion, nil } resourceVersion = resp.Index convertRecursiveResponse(resp.Node, resp, incoming) return }
// Implements storage.Interface. func (h *etcdHelper) Delete(ctx context.Context, key string, out runtime.Object) error { if ctx == nil { glog.Errorf("Context is nil") } key = h.prefixEtcdKey(key) if _, err := conversion.EnforcePtr(out); err != nil { panic("unable to convert output object to pointer") } startTime := time.Now() response, err := h.etcdKeysAPI.Delete(ctx, key, nil) metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime) if !etcdutil.IsEtcdNotFound(err) { // if the object that existed prior to the delete is returned by etcd, update out. if err != nil || response.PrevNode != nil { _, _, err = h.extractObj(response, err, out, false, true) } } return err }
// Implements storage.Interface. func (h *etcdHelper) GetToList(ctx context.Context, key string, filter storage.FilterFunc, listObj runtime.Object) error { if ctx == nil { glog.Errorf("Context is nil") } trace := util.NewTrace("GetToList " + getTypeName(listObj)) listPtr, err := meta.GetItemsPtr(listObj) if err != nil { return err } key = h.prefixEtcdKey(key) startTime := time.Now() trace.Step("About to read etcd node") opts := &etcd.GetOptions{ Quorum: h.quorum, } response, err := h.client.Get(ctx, key, opts) metrics.RecordEtcdRequestLatency("get", getTypeName(listPtr), startTime) trace.Step("Etcd node read") if err != nil { if etcdutil.IsEtcdNotFound(err) { return nil } return err } nodes := make([]*etcd.Node, 0) nodes = append(nodes, response.Node) if err := h.decodeNodeList(nodes, filter, listPtr); err != nil { return err } trace.Step("Object decoded") if h.versioner != nil { if err := h.versioner.UpdateList(listObj, response.Index); err != nil { return err } } return nil }
// acquireOrRenewLease either races to acquire a new master lease, or update the existing master's lease // returns true if we have the lease, and an error if one occurs. // TODO: use the master election utility once it is merged in. func (c *config) acquireOrRenewLease(etcdClient *etcd.Client) (bool, error) { keysAPI := etcd.NewKeysAPI(*etcdClient) resp, err := keysAPI.Get(context.TODO(), c.key, nil) if err != nil { if etcdutil.IsEtcdNotFound(err) { // there is no current master, try to become master, create will fail if the key already exists opts := etcd.SetOptions{ TTL: time.Duration(c.ttl) * time.Second, PrevExist: "", } _, err := keysAPI.Set(context.TODO(), c.key, c.whoami, &opts) if err != nil { return false, err } c.lastLease = time.Now() return true, nil } return false, err } if resp.Node.Value == c.whoami { glog.Infof("key already exists, we are the master (%s)", resp.Node.Value) // we extend our lease @ 1/2 of the existing TTL, this ensures the master doesn't flap around if resp.Node.Expiration.Sub(time.Now()) < time.Duration(c.ttl/2)*time.Second { opts := etcd.SetOptions{ TTL: time.Duration(c.ttl) * time.Second, PrevValue: c.whoami, PrevIndex: resp.Node.ModifiedIndex, } _, err := keysAPI.Set(context.TODO(), c.key, c.whoami, &opts) if err != nil { return false, err } } c.lastLease = time.Now() return true, nil } glog.Infof("key already exists, the master is %s, sleeping.", resp.Node.Value) return false, nil }
// Implements storage.Interface. func (h *etcdHelper) GetToList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error { if ctx == nil { glog.Errorf("Context is nil") } trace := util.NewTrace("GetToList " + getTypeName(listObj)) listPtr, err := meta.GetItemsPtr(listObj) if err != nil { return err } key = path.Join(h.pathPrefix, key) startTime := time.Now() trace.Step("About to read etcd node") opts := &etcd.GetOptions{ Quorum: h.quorum, } response, err := h.etcdKeysAPI.Get(ctx, key, opts) trace.Step("Etcd node read") metrics.RecordEtcdRequestLatency("get", getTypeName(listPtr), startTime) if err != nil { if etcdutil.IsEtcdNotFound(err) { return nil } return toStorageErr(err, key, 0) } nodes := make([]*etcd.Node, 0) nodes = append(nodes, response.Node) if err := h.decodeNodeList(nodes, storage.SimpleFilter(pred), listPtr); err != nil { return err } trace.Step("Object decoded") if err := h.versioner.UpdateList(listObj, response.Index); err != nil { return err } return nil }
func (h *etcdHelper) listEtcdNode(ctx context.Context, key string) ([]*etcd.Node, uint64, error) { if ctx == nil { glog.Errorf("Context is nil") } opts := etcd.GetOptions{ Recursive: true, Sort: true, Quorum: h.quorum, } result, err := h.etcdKeysAPI.Get(ctx, key, &opts) if err != nil { var index uint64 if etcdError, ok := err.(etcd.Error); ok { index = etcdError.Index } nodes := make([]*etcd.Node, 0) if etcdutil.IsEtcdNotFound(err) { return nodes, index, nil } else { return nodes, index, err } } return result.Node.Nodes, result.Index, nil }
// IsNotFound returns true if and only if err is "key" not found error. func IsNotFound(err error) bool { // TODO: add alternate storage error here return etcdutil.IsEtcdNotFound(err) }
func TestUserInitialization(t *testing.T) { testutil.RequireEtcd(t) defer testutil.DumpEtcdOnFailure(t) masterConfig, clusterAdminKubeConfig, err := testserver.StartTestMasterAPI() if err != nil { t.Fatalf("unexpected error: %v", err) } clusterAdminClient, err := testutil.GetClusterAdminClient(clusterAdminKubeConfig) if err != nil { t.Fatalf("unexpected error: %v", err) } optsGetter := originrest.StorageOptions(*masterConfig) userStorage, err := useretcd.NewREST(optsGetter) if err != nil { t.Fatalf("unexpected error: %v", err) } userRegistry := userregistry.NewRegistry(userStorage) identityStorage, err := identityetcd.NewREST(optsGetter) if err != nil { t.Fatalf("unexpected error: %v", err) } identityRegistry := identityregistry.NewRegistry(identityStorage) lookup, err := identitymapper.NewIdentityUserMapper(identityRegistry, userRegistry, identitymapper.MappingMethodLookup) if err != nil { t.Fatalf("unexpected error: %v", err) } generate, err := identitymapper.NewIdentityUserMapper(identityRegistry, userRegistry, identitymapper.MappingMethodGenerate) if err != nil { t.Fatalf("unexpected error: %v", err) } add, err := identitymapper.NewIdentityUserMapper(identityRegistry, userRegistry, identitymapper.MappingMethodAdd) if err != nil { t.Fatalf("unexpected error: %v", err) } claim, err := identitymapper.NewIdentityUserMapper(identityRegistry, userRegistry, identitymapper.MappingMethodClaim) if err != nil { t.Fatalf("unexpected error: %v", err) } testcases := map[string]struct { Identity authapi.UserIdentityInfo Mapper authapi.UserIdentityMapper CreateIdentity *api.Identity CreateUser *api.User CreateMapping *api.UserIdentityMapping UpdateUser *api.User ExpectedErr error ExpectedUserName string ExpectedFullName string ExpectedIdentities []string }{ "lookup missing identity": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: lookup, ExpectedErr: identitymapper.NewLookupError(makeIdentityInfo("idp", "bob", nil), kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob")), }, "lookup existing identity": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: lookup, CreateUser: makeUser("mappeduser"), CreateIdentity: makeIdentity("idp", "bob"), CreateMapping: makeMapping("mappeduser", "idp:bob"), ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "generate missing identity and user": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: generate, ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "generate missing identity and user with preferred username and display name": { Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityDisplayNameKey: "Bob, Sr.", authapi.IdentityPreferredUsernameKey: "admin"}), Mapper: generate, ExpectedUserName: "******", ExpectedFullName: "Bob, Sr.", ExpectedIdentities: []string{"idp:bob"}, }, "generate missing identity for existing user": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: generate, CreateUser: makeUser("bob", "idp:bob"), ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "generate missing identity with conflicting user": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: generate, CreateUser: makeUser("bob"), ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "generate missing identity with conflicting user and preferred username": { Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityPreferredUsernameKey: "admin"}), Mapper: generate, CreateUser: makeUser("admin"), ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "generate with existing unmapped identity": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: generate, CreateIdentity: makeIdentity("idp", "bob"), ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"), }, "generate with existing mapped identity with invalid user UID": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: generate, CreateUser: makeUser("mappeduser"), CreateIdentity: makeIdentityWithUserReference("idp", "bob", "mappeduser", "invalidUID"), ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"), ExpectedIdentities: []string{"idp:bob"}, }, "generate with existing mapped identity without user backreference": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: generate, CreateUser: makeUser("mappeduser"), CreateIdentity: makeIdentity("idp", "bob"), CreateMapping: makeMapping("mappeduser", "idp:bob"), // Update user to a version which does not reference the identity UpdateUser: makeUser("mappeduser"), ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"), }, "generate returns existing mapping": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: generate, CreateUser: makeUser("mappeduser"), CreateIdentity: makeIdentity("idp", "bob"), CreateMapping: makeMapping("mappeduser", "idp:bob"), ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "add missing identity and user": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: add, ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "add missing identity and user with preferred username and display name": { Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityDisplayNameKey: "Bob, Sr.", authapi.IdentityPreferredUsernameKey: "admin"}), Mapper: add, ExpectedUserName: "******", ExpectedFullName: "Bob, Sr.", ExpectedIdentities: []string{"idp:bob"}, }, "add missing identity for existing user": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: add, CreateUser: makeUser("bob", "idp:bob"), ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "add missing identity with conflicting user": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: add, CreateUser: makeUser("bob", "otheridp:otheruser"), ExpectedUserName: "******", ExpectedIdentities: []string{"otheridp:otheruser", "idp:bob"}, }, "add missing identity with conflicting user and preferred username": { Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityPreferredUsernameKey: "admin"}), Mapper: add, CreateUser: makeUser("admin", "otheridp:otheruser"), ExpectedUserName: "******", ExpectedIdentities: []string{"otheridp:otheruser", "idp:bob"}, }, "add with existing unmapped identity": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: add, CreateIdentity: makeIdentity("idp", "bob"), ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"), }, "add with existing mapped identity with invalid user UID": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: add, CreateUser: makeUser("mappeduser"), CreateIdentity: makeIdentityWithUserReference("idp", "bob", "mappeduser", "invalidUID"), ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"), }, "add with existing mapped identity without user backreference": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: add, CreateUser: makeUser("mappeduser"), CreateIdentity: makeIdentity("idp", "bob"), CreateMapping: makeMapping("mappeduser", "idp:bob"), // Update user to a version which does not reference the identity UpdateUser: makeUser("mappeduser"), ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"), }, "add returns existing mapping": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: add, CreateUser: makeUser("mappeduser"), CreateIdentity: makeIdentity("idp", "bob"), CreateMapping: makeMapping("mappeduser", "idp:bob"), ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "claim missing identity and user": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: claim, ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "claim missing identity and user with preferred username and display name": { Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityDisplayNameKey: "Bob, Sr.", authapi.IdentityPreferredUsernameKey: "admin"}), Mapper: claim, ExpectedUserName: "******", ExpectedFullName: "Bob, Sr.", ExpectedIdentities: []string{"idp:bob"}, }, "claim missing identity for existing user": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: claim, CreateUser: makeUser("bob", "idp:bob"), ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "claim missing identity with existing available user": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: claim, CreateUser: makeUser("bob"), ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, "claim missing identity with conflicting user": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: claim, CreateUser: makeUser("bob", "otheridp:otheruser"), ExpectedErr: identitymapper.NewClaimError(makeUser("bob", "otheridp:otheruser"), makeIdentity("idp", "bob")), }, "claim missing identity with conflicting user and preferred username": { Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityPreferredUsernameKey: "admin"}), Mapper: claim, CreateUser: makeUser("admin", "otheridp:otheruser"), ExpectedErr: identitymapper.NewClaimError(makeUser("admin", "otheridp:otheruser"), makeIdentity("idp", "bob")), }, "claim with existing unmapped identity": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: claim, CreateIdentity: makeIdentity("idp", "bob"), ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"), }, "claim with existing mapped identity with invalid user UID": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: claim, CreateUser: makeUser("mappeduser"), CreateIdentity: makeIdentityWithUserReference("idp", "bob", "mappeduser", "invalidUID"), ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"), }, "claim with existing mapped identity without user backreference": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: claim, CreateUser: makeUser("mappeduser"), CreateIdentity: makeIdentity("idp", "bob"), CreateMapping: makeMapping("mappeduser", "idp:bob"), // Update user to a version which does not reference the identity UpdateUser: makeUser("mappeduser"), ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"), }, "claim returns existing mapping": { Identity: makeIdentityInfo("idp", "bob", nil), Mapper: claim, CreateUser: makeUser("mappeduser"), CreateIdentity: makeIdentity("idp", "bob"), CreateMapping: makeMapping("mappeduser", "idp:bob"), ExpectedUserName: "******", ExpectedIdentities: []string{"idp:bob"}, }, } oldEtcdClient, err := etcd.MakeNewEtcdClient(masterConfig.EtcdClientInfo) if err != nil { t.Fatalf("unexpected error: %v", err) } etcdClient := etcdclient.NewKeysAPI(oldEtcdClient) for k, testcase := range testcases { // Cleanup if _, err := etcdClient.Delete(context.Background(), path.Join(masterConfig.EtcdStorageConfig.OpenShiftStoragePrefix, "/users"), &etcdclient.DeleteOptions{Recursive: true}); err != nil && !etcdutil.IsEtcdNotFound(err) { t.Fatalf("Could not clean up users: %v", err) } if _, err := etcdClient.Delete(context.Background(), path.Join(masterConfig.EtcdStorageConfig.OpenShiftStoragePrefix, "/identities"), &etcdclient.DeleteOptions{Recursive: true}); err != nil && !etcdutil.IsEtcdNotFound(err) { t.Fatalf("Could not clean up identities: %v", err) } // Pre-create items if testcase.CreateUser != nil { _, err := clusterAdminClient.Users().Create(testcase.CreateUser) if err != nil { t.Errorf("%s: Could not create user: %v", k, err) continue } } if testcase.CreateIdentity != nil { _, err := clusterAdminClient.Identities().Create(testcase.CreateIdentity) if err != nil { t.Errorf("%s: Could not create identity: %v", k, err) continue } } if testcase.CreateMapping != nil { _, err := clusterAdminClient.UserIdentityMappings().Update(testcase.CreateMapping) if err != nil { t.Errorf("%s: Could not create mapping: %v", k, err) continue } } if testcase.UpdateUser != nil { if testcase.UpdateUser.ResourceVersion == "" { existingUser, err := clusterAdminClient.Users().Get(testcase.UpdateUser.Name) if err != nil { t.Errorf("%s: Could not get user to update: %v", k, err) continue } testcase.UpdateUser.ResourceVersion = existingUser.ResourceVersion } _, err := clusterAdminClient.Users().Update(testcase.UpdateUser) if err != nil { t.Errorf("%s: Could not update user: %v", k, err) continue } } // Spawn 5 simultaneous mappers to test race conditions var wg sync.WaitGroup for i := 0; i < 5; i++ { wg.Add(1) go func() { defer wg.Done() userInfo, err := testcase.Mapper.UserFor(testcase.Identity) if err != nil { if testcase.ExpectedErr == nil { t.Errorf("%s: Expected success, got error '%v'", k, err) } else if err.Error() != testcase.ExpectedErr.Error() { t.Errorf("%s: Expected error %v, got '%v'", k, testcase.ExpectedErr.Error(), err) } return } if err == nil && testcase.ExpectedErr != nil { t.Errorf("%s: Expected error '%v', got none", k, testcase.ExpectedErr) return } if userInfo.GetName() != testcase.ExpectedUserName { t.Errorf("%s: Expected username %s, got %s", k, testcase.ExpectedUserName, userInfo.GetName()) return } user, err := clusterAdminClient.Users().Get(userInfo.GetName()) if err != nil { t.Errorf("%s: Error getting user: %v", k, err) } if user.FullName != testcase.ExpectedFullName { t.Errorf("%s: Expected full name %s, got %s", k, testcase.ExpectedFullName, user.FullName) } if !reflect.DeepEqual(user.Identities, testcase.ExpectedIdentities) { t.Errorf("%s: Expected identities %v, got %v", k, testcase.ExpectedIdentities, user.Identities) } }() } wg.Wait() } }
func testInstallThirdPartyAPIDeleteVersion(t *testing.T, version string) { master, etcdserver, server, assert := initThirdParty(t, version) defer server.Close() defer etcdserver.Terminate(t) expectedObj := Foo{ ObjectMeta: api.ObjectMeta{ Name: "test", Namespace: "default", }, TypeMeta: unversioned.TypeMeta{ Kind: "Foo", }, SomeField: "test field", OtherField: 10, } if !assert.NoError(storeThirdPartyObject(master.thirdPartyStorage, "/ThirdPartyResourceData/company.com/foos/default/test", "test", expectedObj)) { t.FailNow() return } resp, err := http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test") if !assert.NoError(err) { return } assert.Equal(http.StatusOK, resp.StatusCode) item := Foo{} assert.NoError(decodeResponse(resp, &item)) // Fill in fields set by the apiserver expectedObj.SelfLink = item.SelfLink expectedObj.ResourceVersion = item.ResourceVersion expectedObj.Namespace = item.Namespace if !assert.True(reflect.DeepEqual(item, expectedObj)) { t.Errorf("expected:\n%v\nsaw:\n%v\n", expectedObj, item) } resp, err = httpDelete(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test") if !assert.NoError(err) { return } assert.Equal(http.StatusOK, resp.StatusCode) resp, err = http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test") if !assert.NoError(err) { return } assert.Equal(http.StatusNotFound, resp.StatusCode) expectedDeletedKey := etcdtest.AddPrefix("ThirdPartyResourceData/company.com/foos/default/test") thirdPartyObj := extensions.ThirdPartyResourceData{} err = master.thirdPartyStorage.Get( context.TODO(), expectedDeletedKey, &thirdPartyObj, false) if !etcdutil.IsEtcdNotFound(err) { t.Errorf("expected deletion didn't happen: %v", err) } }
func testInstallThirdPartyResourceRemove(t *testing.T, version string) { master, etcdserver, server, assert := initThirdParty(t, version) defer server.Close() defer etcdserver.Terminate(t) expectedObj := Foo{ ObjectMeta: api.ObjectMeta{ Name: "test", }, TypeMeta: unversioned.TypeMeta{ Kind: "Foo", }, SomeField: "test field", OtherField: 10, } if !assert.NoError(storeThirdPartyObject(master.thirdPartyStorage, "/ThirdPartyResourceData/company.com/foos/default/test", "test", expectedObj)) { t.FailNow() return } secondObj := expectedObj secondObj.Name = "bar" if !assert.NoError(storeThirdPartyObject(master.thirdPartyStorage, "/ThirdPartyResourceData/company.com/foos/default/bar", "bar", secondObj)) { t.FailNow() return } resp, err := http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test") if !assert.NoError(err) { t.FailNow() return } if resp.StatusCode != http.StatusOK { t.Errorf("unexpected status: %v", resp) } item := Foo{} if err := decodeResponse(resp, &item); err != nil { t.Errorf("unexpected error: %v", err) } // TODO: validate etcd set things here item.ObjectMeta = expectedObj.ObjectMeta if !assert.True(reflect.DeepEqual(item, expectedObj)) { t.Errorf("expected:\n%v\nsaw:\n%v\n", expectedObj, item) } path := makeThirdPartyPath("company.com") master.RemoveThirdPartyResource(path) resp, err = http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test") if !assert.NoError(err) { return } if resp.StatusCode != http.StatusNotFound { t.Errorf("unexpected status: %v", resp) } expectedDeletedKeys := []string{ etcdtest.AddPrefix("/ThirdPartyResourceData/company.com/foos/default/test"), etcdtest.AddPrefix("/ThirdPartyResourceData/company.com/foos/default/bar"), } for _, key := range expectedDeletedKeys { thirdPartyObj := extensions.ThirdPartyResourceData{} err := master.thirdPartyStorage.Get(context.TODO(), key, &thirdPartyObj, false) if !etcdutil.IsEtcdNotFound(err) { t.Errorf("expected deletion didn't happen: %v", err) } } installed := master.ListThirdPartyResources() if len(installed) != 0 { t.Errorf("Resource(s) still installed: %v", installed) } services := master.handlerContainer.RegisteredWebServices() for ix := range services { if strings.HasPrefix(services[ix].RootPath(), "/apis/company.com") { t.Errorf("Web service still installed at %s: %#v", services[ix].RootPath(), services[ix]) } } }
// tryHold attempts to hold on to the lease by repeatedly refreshing its TTL. // If the lease hold fails, is deleted, or changed to another user. The provided // index is used to watch from. // TODO: currently if we miss the watch window, we will error and try to recreate // the lock. It's likely we will lose the lease due to that. func (e *Etcd) tryHold(ttl, index uint64) error { // watch for termination stop := make(chan struct{}) lost := make(chan struct{}) closedLost := false watchIndex := index go utilwait.Until(func() { index, err := e.waitForExpiration(true, watchIndex, stop) watchIndex = index if err != nil { utilruntime.HandleError(fmt.Errorf("error watching for lease expiration %s: %v", e.key, err)) return } glog.V(4).Infof("Lease %s lost due to deletion at %d", e.key, watchIndex) if !closedLost { closedLost = true close(lost) } }, 100*time.Millisecond, stop) defer close(stop) duration := time.Duration(ttl) * time.Second after := time.Duration(float32(duration) * e.waitFraction) last := duration - after interval := last / time.Duration(e.maxRetries) if interval < e.minimumRetryInterval { interval = e.minimumRetryInterval } // as long as we can renew the lease, loop for { select { case <-time.After(after): err := wait.Poll(interval, last, func() (bool, error) { glog.V(4).Infof("Renewing lease %s at %d", e.key, index-1) resp, err := e.client.CompareAndSwap(e.key, e.value, e.ttl, e.value, index-1) switch { case err == nil: index = eventIndexFor(resp) return true, nil case etcdutil.IsEtcdTestFailed(err): return false, fmt.Errorf("another client has taken the lease %s: %v", e.key, err) case etcdutil.IsEtcdNotFound(err): return false, fmt.Errorf("another client has revoked the lease %s", e.key) default: utilruntime.HandleError(fmt.Errorf("unexpected error renewing lease %s: %v", e.key, err)) index = etcdIndexFor(err, index) // try again return false, nil } }) switch err { case nil: // wait again glog.V(4).Infof("Lease %s renewed at %d", e.key, index-1) case wait.ErrWaitTimeout: return fmt.Errorf("unable to renew lease %s at %d: %v", e.key, index, err) default: return fmt.Errorf("lost lease %s at %d: %v", e.key, index, err) } case <-lost: return fmt.Errorf("the lease has been lost %s at %d", e.key, index) } } }