// handleMaster performs one loop of master locking.
// on success it returns <master>, nil
// on error it returns "", err
// in situations where you should try again due to concurrent state changes (e.g. another actor simultaneously acquiring the lock)
// it returns "", nil
func (e *etcdMasterElector) handleMaster(path, id string, ttl uint64) (string, error) {
res, err := e.etcd.Get(context.TODO(), path, nil)
// Unexpected error, bail out
if err != nil && !etcdutil.IsEtcdNotFound(err) {
return "", err
}
// There is no master, try to become the master.
if err != nil && etcdutil.IsEtcdNotFound(err) {
return e.becomeMaster(path, id, ttl)
}
// This should never happen.
if res.Node == nil {
return "", fmt.Errorf("unexpected response: %#v", res)
}
// We're not the master, just return the current value
if res.Node.Value != id {
return res.Node.Value, nil
}
// We are the master, try to extend out lease
return e.extendMaster(path, id, ttl, res)
}
func Store(api etcd.KeysAPI, path string, ttl time.Duration) frameworkid.Storage {
// TODO(jdef) validate Config
return &storage{
LookupFunc: func(ctx context.Context) (string, error) {
if response, err := api.Get(ctx, path, nil); err != nil {
if !etcdutil.IsEtcdNotFound(err) {
return "", fmt.Errorf("unexpected failure attempting to load framework ID from etcd: %v", err)
}
} else {
return response.Node.Value, nil
}
return "", nil
},
RemoveFunc: func(ctx context.Context) (err error) {
if _, err = api.Delete(ctx, path, &etcd.DeleteOptions{Recursive: true}); err != nil {
if !etcdutil.IsEtcdNotFound(err) {
return fmt.Errorf("failed to delete framework ID from etcd: %v", err)
}
}
return
},
StoreFunc: func(ctx context.Context, id string) (err error) {
_, err = api.Set(ctx, path, id, &etcd.SetOptions{TTL: ttl})
return
},
}
}
// Implements storage.Interface.
func (h *etcdHelper) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions) error {
if ctx == nil {
glog.Errorf("Context is nil")
}
key = h.prefixEtcdKey(key)
v, err := conversion.EnforcePtr(out)
if err != nil {
panic("unable to convert output object to pointer")
}
if preconditions == nil {
startTime := time.Now()
response, err := h.etcdKeysAPI.Delete(ctx, key, nil)
metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime)
if !etcdutil.IsEtcdNotFound(err) {
// if the object that existed prior to the delete is returned by etcd, update the out object.
if err != nil || response.PrevNode != nil {
_, _, err = h.extractObj(response, err, out, false, true)
}
}
return toStorageErr(err, key, 0)
}
// Check the preconditions match.
obj := reflect.New(v.Type()).Interface().(runtime.Object)
for {
_, node, res, err := h.bodyAndExtractObj(ctx, key, obj, false)
if err != nil {
return toStorageErr(err, key, 0)
}
if err := checkPreconditions(key, preconditions, obj); err != nil {
return toStorageErr(err, key, 0)
}
index := uint64(0)
if node != nil {
index = node.ModifiedIndex
} else if res != nil {
index = res.Index
}
opt := etcd.DeleteOptions{PrevIndex: index}
startTime := time.Now()
response, err := h.etcdKeysAPI.Delete(ctx, key, &opt)
metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime)
if etcdutil.IsEtcdTestFailed(err) {
glog.Infof("deletion of %s failed because of a conflict, going to retry", key)
} else {
if !etcdutil.IsEtcdNotFound(err) {
// if the object that existed prior to the delete is returned by etcd, update the out object.
if err != nil || response.PrevNode != nil {
_, _, err = h.extractObj(response, err, out, false, true)
}
}
return toStorageErr(err, key, 0)
}
}
}
// InterpretDeleteError converts a generic etcd error on a delete
// operation into the appropriate API error.
func InterpretDeleteError(err error, kind, name string) error {
switch {
case etcdutil.IsEtcdNotFound(err):
return errors.NewNotFound(kind, name)
case etcdutil.IsEtcdUnreachable(err):
return errors.NewServerTimeout(kind, "delete", 2) // TODO: make configurable or handled at a higher level
default:
return err
}
}
// TestEtcdClient verifies a client is functional. It will attempt to
// connect to the etcd server and block until the server responds at least once, or return an
// error if the server never responded.
func TestEtcdClient(etcdClient etcdclient.Client) error {
for i := 0; ; i++ {
_, err := etcdclient.NewKeysAPI(etcdClient).Get(context.Background(), "/", nil)
if err == nil || etcdutil.IsEtcdNotFound(err) {
break
}
if i > 100 {
return fmt.Errorf("could not reach etcd: %v", err)
}
time.Sleep(50 * time.Millisecond)
}
return nil
}
func TestGetNotFoundErr(t *testing.T) {
server := etcdtesting.NewEtcdTestClientServer(t)
defer server.Terminate(t)
key := etcdtest.AddPrefix("/some/key")
boguskey := etcdtest.AddPrefix("/some/boguskey")
helper := newEtcdHelper(server.Client, testapi.Default.Codec(), key)
var got api.Pod
err := helper.Get(context.TODO(), boguskey, &got, false)
if !etcdutil.IsEtcdNotFound(err) {
t.Errorf("Unexpected reponse on key=%v, err=%v", key, err)
}
}
// TestEtcdClient verifies a client is functional. It will attempt to
// connect to the etcd server and block until the server responds at least once, or return an
// error if the server never responded.
func TestEtcdClient(etcdClient *etcdclient.Client) error {
for i := 0; ; i++ {
_, err := etcdClient.Get("/", false, false)
if err == nil || etcdutil.IsEtcdNotFound(err) {
break
}
if i > 100 {
return fmt.Errorf("could not reach etcd: %v", err)
}
time.Sleep(50 * time.Millisecond)
}
return nil
}
func (s *SchedulerServer) fetchFrameworkID(client tools.EtcdClient) (*mesos.FrameworkID, error) {
if s.failoverTimeout > 0 {
if response, err := client.Get(meta.FrameworkIDKey, false, false); err != nil {
if !etcdutil.IsEtcdNotFound(err) {
return nil, fmt.Errorf("unexpected failure attempting to load framework ID from etcd: %v", err)
}
log.V(1).Infof("did not find framework ID in etcd")
} else if response.Node.Value != "" {
log.Infof("configuring FrameworkInfo with Id found in etcd: '%s'", response.Node.Value)
return mutil.NewFrameworkID(response.Node.Value), nil
}
} else {
//TODO(jdef) this seems like a totally hackish way to clean up the framework ID
if _, err := client.Delete(meta.FrameworkIDKey, true); err != nil {
if !etcdutil.IsEtcdNotFound(err) {
return nil, fmt.Errorf("failed to delete framework ID from etcd: %v", err)
}
log.V(1).Infof("nothing to delete: did not find framework ID in etcd")
}
}
return nil, nil
}
// Release tries to delete the leader lock.
func (e *Etcd) Release() {
for i := 0; i < e.maxRetries; i++ {
_, err := e.client.Delete(context.Background(), e.key, &etcdclient.DeleteOptions{PrevValue: e.value})
if err == nil {
break
}
// If the value has changed, we don't hold the lease. If the key is missing we don't
// hold the lease.
if etcdutil.IsEtcdTestFailed(err) || etcdutil.IsEtcdNotFound(err) {
break
}
utilruntime.HandleError(fmt.Errorf("unable to release %s: %v", e.key, err))
}
}
// Refresh reloads the RangeAllocation from etcd.
func (e *Etcd) Refresh() (*api.RangeAllocation, error) {
e.lock.Lock()
defer e.lock.Unlock()
existing := &api.RangeAllocation{}
if err := e.storage.Get(context.TODO(), e.baseKey, existing, false); err != nil {
if etcdutil.IsEtcdNotFound(err) {
return nil, nil
}
return nil, etcderr.InterpretGetError(err, e.kind, "")
}
return existing, nil
}
// bodyAndExtractObj performs the normal Get path to etcd, returning the parsed node and response for additional information
// about the response, like the current etcd index and the ttl.
func (h *etcdHelper) bodyAndExtractObj(ctx context.Context, key string, objPtr runtime.Object, ignoreNotFound bool) (body string, node *etcd.Node, res *etcd.Response, err error) {
if ctx == nil {
glog.Errorf("Context is nil")
}
startTime := time.Now()
response, err := h.client.Get(ctx, key, nil)
metrics.RecordEtcdRequestLatency("get", getTypeName(objPtr), startTime)
if err != nil && !etcdutil.IsEtcdNotFound(err) {
return "", nil, nil, err
}
body, node, err = h.extractObj(response, err, objPtr, ignoreNotFound, false)
return body, node, response, err
}
// Release tries to delete the leader lock.
func (e *Etcd) Release() {
for i := 0; i < e.maxRetries; i++ {
_, err := e.client.CompareAndDelete(e.key, e.value, 0)
if err == nil {
break
}
// If the value has changed, we don't hold the lease. If the key is missing we don't
// hold the lease.
if etcdutil.IsEtcdTestFailed(err) || etcdutil.IsEtcdNotFound(err) {
break
}
utilruntime.HandleError(fmt.Errorf("unable to release %s: %v", e.key, err))
}
}
// etcdGetInitialWatchState turns an etcd Get request into a watch equivalent
func etcdGetInitialWatchState(client *etcd.Client, key string, recursive bool, incoming chan<- *etcd.Response) (resourceVersion uint64, err error) {
resp, err := client.Get(key, false, recursive)
if err != nil {
if !etcdutil.IsEtcdNotFound(err) {
glog.Errorf("watch was unable to retrieve the current index for the provided key (%q): %v", key, err)
return resourceVersion, err
}
if etcdError, ok := err.(*etcd.EtcdError); ok {
resourceVersion = etcdError.Index
}
return resourceVersion, nil
}
resourceVersion = resp.EtcdIndex
convertRecursiveResponse(resp.Node, resp, incoming)
return
}
func toStorageErr(err error, key string, rv int64) error {
if err == nil {
return nil
}
switch {
case etcdutil.IsEtcdNotFound(err):
return storage.NewKeyNotFoundError(key, rv)
case etcdutil.IsEtcdNodeExist(err):
return storage.NewKeyExistsError(key, rv)
case etcdutil.IsEtcdTestFailed(err):
return storage.NewResourceVersionConflictsError(key, rv)
case etcdutil.IsEtcdUnreachable(err):
return storage.NewUnreachableError(key, rv)
default:
return err
}
}
// bodyAndExtractObj performs the normal Get path to etcd, returning the parsed node and response for additional information
// about the response, like the current etcd index and the ttl.
func (h *etcdHelper) bodyAndExtractObj(ctx context.Context, key string, objPtr runtime.Object, ignoreNotFound bool) (body string, node *etcd.Node, res *etcd.Response, err error) {
if ctx == nil {
glog.Errorf("Context is nil")
}
startTime := time.Now()
opts := &etcd.GetOptions{
Quorum: h.quorum,
}
response, err := h.etcdKeysAPI.Get(ctx, key, opts)
metrics.RecordEtcdRequestLatency("get", getTypeName(objPtr), startTime)
if err != nil && !etcdutil.IsEtcdNotFound(err) {
return "", nil, nil, toStorageErr(err, key, 0)
}
body, node, err = h.extractObj(response, err, objPtr, ignoreNotFound, false)
return body, node, response, toStorageErr(err, key, 0)
}
func (h *etcdHelper) listEtcdNode(ctx context.Context, key string) ([]*etcd.Node, uint64, error) {
if ctx == nil {
glog.Errorf("Context is nil")
}
result, err := h.client.Get(key, true, true)
if err != nil {
var index uint64
if etcdError, ok := err.(*etcd.EtcdError); ok {
index = etcdError.Index
}
nodes := make([]*etcd.Node, 0)
if etcdutil.IsEtcdNotFound(err) {
return nodes, index, nil
} else {
return nodes, index, err
}
}
return result.Node.Nodes, result.EtcdIndex, nil
}
// etcdGetInitialWatchState turns an etcd Get request into a watch equivalent
func etcdGetInitialWatchState(ctx context.Context, client etcd.KeysAPI, key string, recursive bool, incoming chan<- *etcd.Response) (resourceVersion uint64, err error) {
opts := etcd.GetOptions{
Recursive: recursive,
Sort: false,
}
resp, err := client.Get(ctx, key, &opts)
if err != nil {
if !etcdutil.IsEtcdNotFound(err) {
glog.Errorf("watch was unable to retrieve the current index for the provided key (%q): %v", key, err)
return resourceVersion, err
}
if etcdError, ok := err.(etcd.Error); ok {
resourceVersion = etcdError.Index
}
return resourceVersion, nil
}
resourceVersion = resp.Index
convertRecursiveResponse(resp.Node, resp, incoming)
return
}
// Implements storage.Interface.
func (h *etcdHelper) Delete(ctx context.Context, key string, out runtime.Object) error {
if ctx == nil {
glog.Errorf("Context is nil")
}
key = h.prefixEtcdKey(key)
if _, err := conversion.EnforcePtr(out); err != nil {
panic("unable to convert output object to pointer")
}
startTime := time.Now()
response, err := h.etcdKeysAPI.Delete(ctx, key, nil)
metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime)
if !etcdutil.IsEtcdNotFound(err) {
// if the object that existed prior to the delete is returned by etcd, update out.
if err != nil || response.PrevNode != nil {
_, _, err = h.extractObj(response, err, out, false, true)
}
}
return err
}
// Implements storage.Interface.
func (h *etcdHelper) GetToList(ctx context.Context, key string, filter storage.FilterFunc, listObj runtime.Object) error {
if ctx == nil {
glog.Errorf("Context is nil")
}
trace := util.NewTrace("GetToList " + getTypeName(listObj))
listPtr, err := meta.GetItemsPtr(listObj)
if err != nil {
return err
}
key = h.prefixEtcdKey(key)
startTime := time.Now()
trace.Step("About to read etcd node")
opts := &etcd.GetOptions{
Quorum: h.quorum,
}
response, err := h.client.Get(ctx, key, opts)
metrics.RecordEtcdRequestLatency("get", getTypeName(listPtr), startTime)
trace.Step("Etcd node read")
if err != nil {
if etcdutil.IsEtcdNotFound(err) {
return nil
}
return err
}
nodes := make([]*etcd.Node, 0)
nodes = append(nodes, response.Node)
if err := h.decodeNodeList(nodes, filter, listPtr); err != nil {
return err
}
trace.Step("Object decoded")
if h.versioner != nil {
if err := h.versioner.UpdateList(listObj, response.Index); err != nil {
return err
}
}
return nil
}
// acquireOrRenewLease either races to acquire a new master lease, or update the existing master's lease
// returns true if we have the lease, and an error if one occurs.
// TODO: use the master election utility once it is merged in.
func (c *config) acquireOrRenewLease(etcdClient *etcd.Client) (bool, error) {
keysAPI := etcd.NewKeysAPI(*etcdClient)
resp, err := keysAPI.Get(context.TODO(), c.key, nil)
if err != nil {
if etcdutil.IsEtcdNotFound(err) {
// there is no current master, try to become master, create will fail if the key already exists
opts := etcd.SetOptions{
TTL: time.Duration(c.ttl) * time.Second,
PrevExist: "",
}
_, err := keysAPI.Set(context.TODO(), c.key, c.whoami, &opts)
if err != nil {
return false, err
}
c.lastLease = time.Now()
return true, nil
}
return false, err
}
if resp.Node.Value == c.whoami {
glog.Infof("key already exists, we are the master (%s)", resp.Node.Value)
// we extend our lease @ 1/2 of the existing TTL, this ensures the master doesn't flap around
if resp.Node.Expiration.Sub(time.Now()) < time.Duration(c.ttl/2)*time.Second {
opts := etcd.SetOptions{
TTL: time.Duration(c.ttl) * time.Second,
PrevValue: c.whoami,
PrevIndex: resp.Node.ModifiedIndex,
}
_, err := keysAPI.Set(context.TODO(), c.key, c.whoami, &opts)
if err != nil {
return false, err
}
}
c.lastLease = time.Now()
return true, nil
}
glog.Infof("key already exists, the master is %s, sleeping.", resp.Node.Value)
return false, nil
}
// Implements storage.Interface.
func (h *etcdHelper) GetToList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
if ctx == nil {
glog.Errorf("Context is nil")
}
trace := util.NewTrace("GetToList " + getTypeName(listObj))
listPtr, err := meta.GetItemsPtr(listObj)
if err != nil {
return err
}
key = path.Join(h.pathPrefix, key)
startTime := time.Now()
trace.Step("About to read etcd node")
opts := &etcd.GetOptions{
Quorum: h.quorum,
}
response, err := h.etcdKeysAPI.Get(ctx, key, opts)
trace.Step("Etcd node read")
metrics.RecordEtcdRequestLatency("get", getTypeName(listPtr), startTime)
if err != nil {
if etcdutil.IsEtcdNotFound(err) {
return nil
}
return toStorageErr(err, key, 0)
}
nodes := make([]*etcd.Node, 0)
nodes = append(nodes, response.Node)
if err := h.decodeNodeList(nodes, storage.SimpleFilter(pred), listPtr); err != nil {
return err
}
trace.Step("Object decoded")
if err := h.versioner.UpdateList(listObj, response.Index); err != nil {
return err
}
return nil
}
func (h *etcdHelper) listEtcdNode(ctx context.Context, key string) ([]*etcd.Node, uint64, error) {
if ctx == nil {
glog.Errorf("Context is nil")
}
opts := etcd.GetOptions{
Recursive: true,
Sort: true,
Quorum: h.quorum,
}
result, err := h.etcdKeysAPI.Get(ctx, key, &opts)
if err != nil {
var index uint64
if etcdError, ok := err.(etcd.Error); ok {
index = etcdError.Index
}
nodes := make([]*etcd.Node, 0)
if etcdutil.IsEtcdNotFound(err) {
return nodes, index, nil
} else {
return nodes, index, err
}
}
return result.Node.Nodes, result.Index, nil
}
// IsNotFound returns true if and only if err is "key" not found error.
func IsNotFound(err error) bool {
// TODO: add alternate storage error here
return etcdutil.IsEtcdNotFound(err)
}
func TestUserInitialization(t *testing.T) {
testutil.RequireEtcd(t)
defer testutil.DumpEtcdOnFailure(t)
masterConfig, clusterAdminKubeConfig, err := testserver.StartTestMasterAPI()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
clusterAdminClient, err := testutil.GetClusterAdminClient(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
optsGetter := originrest.StorageOptions(*masterConfig)
userStorage, err := useretcd.NewREST(optsGetter)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
userRegistry := userregistry.NewRegistry(userStorage)
identityStorage, err := identityetcd.NewREST(optsGetter)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
identityRegistry := identityregistry.NewRegistry(identityStorage)
lookup, err := identitymapper.NewIdentityUserMapper(identityRegistry, userRegistry, identitymapper.MappingMethodLookup)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
generate, err := identitymapper.NewIdentityUserMapper(identityRegistry, userRegistry, identitymapper.MappingMethodGenerate)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
add, err := identitymapper.NewIdentityUserMapper(identityRegistry, userRegistry, identitymapper.MappingMethodAdd)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
claim, err := identitymapper.NewIdentityUserMapper(identityRegistry, userRegistry, identitymapper.MappingMethodClaim)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
testcases := map[string]struct {
Identity authapi.UserIdentityInfo
Mapper authapi.UserIdentityMapper
CreateIdentity *api.Identity
CreateUser *api.User
CreateMapping *api.UserIdentityMapping
UpdateUser *api.User
ExpectedErr error
ExpectedUserName string
ExpectedFullName string
ExpectedIdentities []string
}{
"lookup missing identity": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: lookup,
ExpectedErr: identitymapper.NewLookupError(makeIdentityInfo("idp", "bob", nil), kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob")),
},
"lookup existing identity": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: lookup,
CreateUser: makeUser("mappeduser"),
CreateIdentity: makeIdentity("idp", "bob"),
CreateMapping: makeMapping("mappeduser", "idp:bob"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"generate missing identity and user": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: generate,
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"generate missing identity and user with preferred username and display name": {
Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityDisplayNameKey: "Bob, Sr.", authapi.IdentityPreferredUsernameKey: "admin"}),
Mapper: generate,
ExpectedUserName: "******",
ExpectedFullName: "Bob, Sr.",
ExpectedIdentities: []string{"idp:bob"},
},
"generate missing identity for existing user": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: generate,
CreateUser: makeUser("bob", "idp:bob"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"generate missing identity with conflicting user": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: generate,
CreateUser: makeUser("bob"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"generate missing identity with conflicting user and preferred username": {
Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityPreferredUsernameKey: "admin"}),
Mapper: generate,
CreateUser: makeUser("admin"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"generate with existing unmapped identity": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: generate,
CreateIdentity: makeIdentity("idp", "bob"),
ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"),
},
"generate with existing mapped identity with invalid user UID": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: generate,
CreateUser: makeUser("mappeduser"),
CreateIdentity: makeIdentityWithUserReference("idp", "bob", "mappeduser", "invalidUID"),
ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"),
ExpectedIdentities: []string{"idp:bob"},
},
"generate with existing mapped identity without user backreference": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: generate,
CreateUser: makeUser("mappeduser"),
CreateIdentity: makeIdentity("idp", "bob"),
CreateMapping: makeMapping("mappeduser", "idp:bob"),
// Update user to a version which does not reference the identity
UpdateUser: makeUser("mappeduser"),
ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"),
},
"generate returns existing mapping": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: generate,
CreateUser: makeUser("mappeduser"),
CreateIdentity: makeIdentity("idp", "bob"),
CreateMapping: makeMapping("mappeduser", "idp:bob"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"add missing identity and user": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: add,
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"add missing identity and user with preferred username and display name": {
Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityDisplayNameKey: "Bob, Sr.", authapi.IdentityPreferredUsernameKey: "admin"}),
Mapper: add,
ExpectedUserName: "******",
ExpectedFullName: "Bob, Sr.",
ExpectedIdentities: []string{"idp:bob"},
},
"add missing identity for existing user": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: add,
CreateUser: makeUser("bob", "idp:bob"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"add missing identity with conflicting user": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: add,
CreateUser: makeUser("bob", "otheridp:otheruser"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"otheridp:otheruser", "idp:bob"},
},
"add missing identity with conflicting user and preferred username": {
Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityPreferredUsernameKey: "admin"}),
Mapper: add,
CreateUser: makeUser("admin", "otheridp:otheruser"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"otheridp:otheruser", "idp:bob"},
},
"add with existing unmapped identity": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: add,
CreateIdentity: makeIdentity("idp", "bob"),
ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"),
},
"add with existing mapped identity with invalid user UID": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: add,
CreateUser: makeUser("mappeduser"),
CreateIdentity: makeIdentityWithUserReference("idp", "bob", "mappeduser", "invalidUID"),
ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"),
},
"add with existing mapped identity without user backreference": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: add,
CreateUser: makeUser("mappeduser"),
CreateIdentity: makeIdentity("idp", "bob"),
CreateMapping: makeMapping("mappeduser", "idp:bob"),
// Update user to a version which does not reference the identity
UpdateUser: makeUser("mappeduser"),
ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"),
},
"add returns existing mapping": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: add,
CreateUser: makeUser("mappeduser"),
CreateIdentity: makeIdentity("idp", "bob"),
CreateMapping: makeMapping("mappeduser", "idp:bob"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"claim missing identity and user": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: claim,
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"claim missing identity and user with preferred username and display name": {
Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityDisplayNameKey: "Bob, Sr.", authapi.IdentityPreferredUsernameKey: "admin"}),
Mapper: claim,
ExpectedUserName: "******",
ExpectedFullName: "Bob, Sr.",
ExpectedIdentities: []string{"idp:bob"},
},
"claim missing identity for existing user": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: claim,
CreateUser: makeUser("bob", "idp:bob"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"claim missing identity with existing available user": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: claim,
CreateUser: makeUser("bob"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
"claim missing identity with conflicting user": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: claim,
CreateUser: makeUser("bob", "otheridp:otheruser"),
ExpectedErr: identitymapper.NewClaimError(makeUser("bob", "otheridp:otheruser"), makeIdentity("idp", "bob")),
},
"claim missing identity with conflicting user and preferred username": {
Identity: makeIdentityInfo("idp", "bob", map[string]string{authapi.IdentityPreferredUsernameKey: "admin"}),
Mapper: claim,
CreateUser: makeUser("admin", "otheridp:otheruser"),
ExpectedErr: identitymapper.NewClaimError(makeUser("admin", "otheridp:otheruser"), makeIdentity("idp", "bob")),
},
"claim with existing unmapped identity": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: claim,
CreateIdentity: makeIdentity("idp", "bob"),
ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"),
},
"claim with existing mapped identity with invalid user UID": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: claim,
CreateUser: makeUser("mappeduser"),
CreateIdentity: makeIdentityWithUserReference("idp", "bob", "mappeduser", "invalidUID"),
ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"),
},
"claim with existing mapped identity without user backreference": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: claim,
CreateUser: makeUser("mappeduser"),
CreateIdentity: makeIdentity("idp", "bob"),
CreateMapping: makeMapping("mappeduser", "idp:bob"),
// Update user to a version which does not reference the identity
UpdateUser: makeUser("mappeduser"),
ExpectedErr: kerrs.NewNotFound(api.Resource("useridentitymapping"), "idp:bob"),
},
"claim returns existing mapping": {
Identity: makeIdentityInfo("idp", "bob", nil),
Mapper: claim,
CreateUser: makeUser("mappeduser"),
CreateIdentity: makeIdentity("idp", "bob"),
CreateMapping: makeMapping("mappeduser", "idp:bob"),
ExpectedUserName: "******",
ExpectedIdentities: []string{"idp:bob"},
},
}
oldEtcdClient, err := etcd.MakeNewEtcdClient(masterConfig.EtcdClientInfo)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
etcdClient := etcdclient.NewKeysAPI(oldEtcdClient)
for k, testcase := range testcases {
// Cleanup
if _, err := etcdClient.Delete(context.Background(), path.Join(masterConfig.EtcdStorageConfig.OpenShiftStoragePrefix, "/users"), &etcdclient.DeleteOptions{Recursive: true}); err != nil && !etcdutil.IsEtcdNotFound(err) {
t.Fatalf("Could not clean up users: %v", err)
}
if _, err := etcdClient.Delete(context.Background(), path.Join(masterConfig.EtcdStorageConfig.OpenShiftStoragePrefix, "/identities"), &etcdclient.DeleteOptions{Recursive: true}); err != nil && !etcdutil.IsEtcdNotFound(err) {
t.Fatalf("Could not clean up identities: %v", err)
}
// Pre-create items
if testcase.CreateUser != nil {
_, err := clusterAdminClient.Users().Create(testcase.CreateUser)
if err != nil {
t.Errorf("%s: Could not create user: %v", k, err)
continue
}
}
if testcase.CreateIdentity != nil {
_, err := clusterAdminClient.Identities().Create(testcase.CreateIdentity)
if err != nil {
t.Errorf("%s: Could not create identity: %v", k, err)
continue
}
}
if testcase.CreateMapping != nil {
_, err := clusterAdminClient.UserIdentityMappings().Update(testcase.CreateMapping)
if err != nil {
t.Errorf("%s: Could not create mapping: %v", k, err)
continue
}
}
if testcase.UpdateUser != nil {
if testcase.UpdateUser.ResourceVersion == "" {
existingUser, err := clusterAdminClient.Users().Get(testcase.UpdateUser.Name)
if err != nil {
t.Errorf("%s: Could not get user to update: %v", k, err)
continue
}
testcase.UpdateUser.ResourceVersion = existingUser.ResourceVersion
}
_, err := clusterAdminClient.Users().Update(testcase.UpdateUser)
if err != nil {
t.Errorf("%s: Could not update user: %v", k, err)
continue
}
}
// Spawn 5 simultaneous mappers to test race conditions
var wg sync.WaitGroup
for i := 0; i < 5; i++ {
wg.Add(1)
go func() {
defer wg.Done()
userInfo, err := testcase.Mapper.UserFor(testcase.Identity)
if err != nil {
if testcase.ExpectedErr == nil {
t.Errorf("%s: Expected success, got error '%v'", k, err)
} else if err.Error() != testcase.ExpectedErr.Error() {
t.Errorf("%s: Expected error %v, got '%v'", k, testcase.ExpectedErr.Error(), err)
}
return
}
if err == nil && testcase.ExpectedErr != nil {
t.Errorf("%s: Expected error '%v', got none", k, testcase.ExpectedErr)
return
}
if userInfo.GetName() != testcase.ExpectedUserName {
t.Errorf("%s: Expected username %s, got %s", k, testcase.ExpectedUserName, userInfo.GetName())
return
}
user, err := clusterAdminClient.Users().Get(userInfo.GetName())
if err != nil {
t.Errorf("%s: Error getting user: %v", k, err)
}
if user.FullName != testcase.ExpectedFullName {
t.Errorf("%s: Expected full name %s, got %s", k, testcase.ExpectedFullName, user.FullName)
}
if !reflect.DeepEqual(user.Identities, testcase.ExpectedIdentities) {
t.Errorf("%s: Expected identities %v, got %v", k, testcase.ExpectedIdentities, user.Identities)
}
}()
}
wg.Wait()
}
}
func testInstallThirdPartyAPIDeleteVersion(t *testing.T, version string) {
master, etcdserver, server, assert := initThirdParty(t, version)
defer server.Close()
defer etcdserver.Terminate(t)
expectedObj := Foo{
ObjectMeta: api.ObjectMeta{
Name: "test",
Namespace: "default",
},
TypeMeta: unversioned.TypeMeta{
Kind: "Foo",
},
SomeField: "test field",
OtherField: 10,
}
if !assert.NoError(storeThirdPartyObject(master.thirdPartyStorage, "/ThirdPartyResourceData/company.com/foos/default/test", "test", expectedObj)) {
t.FailNow()
return
}
resp, err := http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test")
if !assert.NoError(err) {
return
}
assert.Equal(http.StatusOK, resp.StatusCode)
item := Foo{}
assert.NoError(decodeResponse(resp, &item))
// Fill in fields set by the apiserver
expectedObj.SelfLink = item.SelfLink
expectedObj.ResourceVersion = item.ResourceVersion
expectedObj.Namespace = item.Namespace
if !assert.True(reflect.DeepEqual(item, expectedObj)) {
t.Errorf("expected:\n%v\nsaw:\n%v\n", expectedObj, item)
}
resp, err = httpDelete(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test")
if !assert.NoError(err) {
return
}
assert.Equal(http.StatusOK, resp.StatusCode)
resp, err = http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test")
if !assert.NoError(err) {
return
}
assert.Equal(http.StatusNotFound, resp.StatusCode)
expectedDeletedKey := etcdtest.AddPrefix("ThirdPartyResourceData/company.com/foos/default/test")
thirdPartyObj := extensions.ThirdPartyResourceData{}
err = master.thirdPartyStorage.Get(
context.TODO(), expectedDeletedKey, &thirdPartyObj, false)
if !etcdutil.IsEtcdNotFound(err) {
t.Errorf("expected deletion didn't happen: %v", err)
}
}
func testInstallThirdPartyResourceRemove(t *testing.T, version string) {
master, etcdserver, server, assert := initThirdParty(t, version)
defer server.Close()
defer etcdserver.Terminate(t)
expectedObj := Foo{
ObjectMeta: api.ObjectMeta{
Name: "test",
},
TypeMeta: unversioned.TypeMeta{
Kind: "Foo",
},
SomeField: "test field",
OtherField: 10,
}
if !assert.NoError(storeThirdPartyObject(master.thirdPartyStorage, "/ThirdPartyResourceData/company.com/foos/default/test", "test", expectedObj)) {
t.FailNow()
return
}
secondObj := expectedObj
secondObj.Name = "bar"
if !assert.NoError(storeThirdPartyObject(master.thirdPartyStorage, "/ThirdPartyResourceData/company.com/foos/default/bar", "bar", secondObj)) {
t.FailNow()
return
}
resp, err := http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test")
if !assert.NoError(err) {
t.FailNow()
return
}
if resp.StatusCode != http.StatusOK {
t.Errorf("unexpected status: %v", resp)
}
item := Foo{}
if err := decodeResponse(resp, &item); err != nil {
t.Errorf("unexpected error: %v", err)
}
// TODO: validate etcd set things here
item.ObjectMeta = expectedObj.ObjectMeta
if !assert.True(reflect.DeepEqual(item, expectedObj)) {
t.Errorf("expected:\n%v\nsaw:\n%v\n", expectedObj, item)
}
path := makeThirdPartyPath("company.com")
master.RemoveThirdPartyResource(path)
resp, err = http.Get(server.URL + "/apis/company.com/" + version + "/namespaces/default/foos/test")
if !assert.NoError(err) {
return
}
if resp.StatusCode != http.StatusNotFound {
t.Errorf("unexpected status: %v", resp)
}
expectedDeletedKeys := []string{
etcdtest.AddPrefix("/ThirdPartyResourceData/company.com/foos/default/test"),
etcdtest.AddPrefix("/ThirdPartyResourceData/company.com/foos/default/bar"),
}
for _, key := range expectedDeletedKeys {
thirdPartyObj := extensions.ThirdPartyResourceData{}
err := master.thirdPartyStorage.Get(context.TODO(), key, &thirdPartyObj, false)
if !etcdutil.IsEtcdNotFound(err) {
t.Errorf("expected deletion didn't happen: %v", err)
}
}
installed := master.ListThirdPartyResources()
if len(installed) != 0 {
t.Errorf("Resource(s) still installed: %v", installed)
}
services := master.handlerContainer.RegisteredWebServices()
for ix := range services {
if strings.HasPrefix(services[ix].RootPath(), "/apis/company.com") {
t.Errorf("Web service still installed at %s: %#v", services[ix].RootPath(), services[ix])
}
}
}
// tryHold attempts to hold on to the lease by repeatedly refreshing its TTL.
// If the lease hold fails, is deleted, or changed to another user. The provided
// index is used to watch from.
// TODO: currently if we miss the watch window, we will error and try to recreate
// the lock. It's likely we will lose the lease due to that.
func (e *Etcd) tryHold(ttl, index uint64) error {
// watch for termination
stop := make(chan struct{})
lost := make(chan struct{})
closedLost := false
watchIndex := index
go utilwait.Until(func() {
index, err := e.waitForExpiration(true, watchIndex, stop)
watchIndex = index
if err != nil {
utilruntime.HandleError(fmt.Errorf("error watching for lease expiration %s: %v", e.key, err))
return
}
glog.V(4).Infof("Lease %s lost due to deletion at %d", e.key, watchIndex)
if !closedLost {
closedLost = true
close(lost)
}
}, 100*time.Millisecond, stop)
defer close(stop)
duration := time.Duration(ttl) * time.Second
after := time.Duration(float32(duration) * e.waitFraction)
last := duration - after
interval := last / time.Duration(e.maxRetries)
if interval < e.minimumRetryInterval {
interval = e.minimumRetryInterval
}
// as long as we can renew the lease, loop
for {
select {
case <-time.After(after):
err := wait.Poll(interval, last, func() (bool, error) {
glog.V(4).Infof("Renewing lease %s at %d", e.key, index-1)
resp, err := e.client.CompareAndSwap(e.key, e.value, e.ttl, e.value, index-1)
switch {
case err == nil:
index = eventIndexFor(resp)
return true, nil
case etcdutil.IsEtcdTestFailed(err):
return false, fmt.Errorf("another client has taken the lease %s: %v", e.key, err)
case etcdutil.IsEtcdNotFound(err):
return false, fmt.Errorf("another client has revoked the lease %s", e.key)
default:
utilruntime.HandleError(fmt.Errorf("unexpected error renewing lease %s: %v", e.key, err))
index = etcdIndexFor(err, index)
// try again
return false, nil
}
})
switch err {
case nil:
// wait again
glog.V(4).Infof("Lease %s renewed at %d", e.key, index-1)
case wait.ErrWaitTimeout:
return fmt.Errorf("unable to renew lease %s at %d: %v", e.key, index, err)
default:
return fmt.Errorf("lost lease %s at %d: %v", e.key, index, err)
}
case <-lost:
return fmt.Errorf("the lease has been lost %s at %d", e.key, index)
}
}
}
