// secretDeleted reacts to a Secret being deleted by removing a reference from the corresponding ServiceAccount if needed
func (e *TokensController) secretDeleted(obj interface{}) {
secret, ok := obj.(*api.Secret)
if !ok {
// Unknown type. If we missed a Secret deletion, the corresponding ServiceAccount (if it exists)
// will get a secret recreated (if needed) during the ServiceAccount re-list
return
}
serviceAccount, err := e.getServiceAccount(secret, false)
if err != nil {
glog.Error(err)
return
}
if serviceAccount == nil {
return
}
for i := 1; i <= NumServiceAccountRemoveReferenceRetries; i++ {
if _, err := e.removeSecretReferenceIfNeeded(serviceAccount, secret.Name); err != nil {
if apierrors.IsConflict(err) && i < NumServiceAccountRemoveReferenceRetries {
time.Sleep(wait.Jitter(100*time.Millisecond, 0.0))
continue
}
glog.Error(err)
break
}
break
}
}
func TestEtcdCreateWithConflict(t *testing.T) {
storage, bindingStorage, _, fakeClient := newStorage(t)
ctx := api.NewDefaultContext()
fakeClient.TestIndex = true
key, _ := storage.KeyFunc(ctx, "foo")
fakeClient.ExpectNotFoundGet(key)
_, err := storage.Create(ctx, validNewPod())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Suddenly, a wild scheduler appears:
binding := api.Binding{
ObjectMeta: api.ObjectMeta{
Namespace: api.NamespaceDefault,
Name: "foo",
Annotations: map[string]string{"label1": "value1"},
},
Target: api.ObjectReference{Name: "machine"},
}
_, err = bindingStorage.Create(ctx, &binding)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = bindingStorage.Create(ctx, &binding)
if err == nil || !errors.IsConflict(err) {
t.Fatalf("expected resource conflict error, not: %v", err)
}
}
func (s *ServiceController) persistUpdate(service *api.Service) error {
var err error
for i := 0; i < clientRetryCount; i++ {
_, err = s.kubeClient.Services(service.Namespace).Update(service)
if err == nil {
return nil
}
// If the object no longer exists, we don't want to recreate it. Just bail
// out so that we can process the delete, which we should soon be receiving
// if we haven't already.
if errors.IsNotFound(err) {
glog.Infof("Not persisting update to service that no longer exists: %v", err)
return nil
}
// TODO: Try to resolve the conflict if the change was unrelated to load
// balancer status. For now, just rely on the fact that we'll
// also process the update that caused the resource version to change.
if errors.IsConflict(err) {
glog.Infof("Not persisting update to service that has been changed since we received it: %v", err)
return nil
}
glog.Warningf("Failed to persist updated LoadBalancerStatus to service %s after creating its external load balancer: %v",
service.Name, err)
time.Sleep(clientRetryInterval)
}
return err
}
func (t *Tester) TestUpdateFailsOnVersion(older runtime.Object) {
_, _, err := t.storage.(rest.Updater).Update(t.TestContext(), older)
if err == nil {
t.Errorf("Expected an error, but we didn't get one")
} else if !errors.IsConflict(err) {
t.Errorf("Expected Conflict error, got '%v'", err)
}
}
// retryOnConflictError retries the specified fn if there was a conflict error
// TODO RetryOnConflict should be a generic concept in client code
func retryOnConflictError(kubeClient client.Interface, namespace *api.Namespace, fn updateNamespaceFunc) (result *api.Namespace, err error) {
latestNamespace := namespace
for {
result, err = fn(kubeClient, latestNamespace)
if err == nil {
return result, nil
}
if !errors.IsConflict(err) {
return nil, err
}
latestNamespace, err = kubeClient.Namespaces().Get(latestNamespace.Name)
if err != nil {
return nil, err
}
}
return
}
// updateService fetches a service, calls the update function on it,
// and then attempts to send the updated service. It retries up to 2
// times in the face of timeouts and conflicts.
func updateService(c *client.Client, namespace, serviceName string, update func(*api.Service)) (*api.Service, error) {
var service *api.Service
var err error
for i := 0; i < 3; i++ {
service, err = c.Services(namespace).Get(serviceName)
if err != nil {
return service, err
}
update(service)
service, err = c.Services(namespace).Update(service)
if !errors.IsConflict(err) && !errors.IsServerTimeout(err) {
return service, err
}
}
return service, err
}
func (t *Tester) testUpdateFailsOnVersionTooOld(obj runtime.Object, setFn SetFunc, setRVFn SetRVFunc) {
ctx := t.TestContext()
foo := copyOrDie(obj)
t.setObjectMeta(foo, "foo3")
setRVFn(10)
if err := setFn(ctx, foo); err != nil {
t.Errorf("unexpected error: %v", err)
}
older := copyOrDie(foo)
olderMeta := t.getObjectMetaOrFail(older)
olderMeta.ResourceVersion = "8"
_, _, err := t.storage.(rest.Updater).Update(t.TestContext(), older)
if err == nil {
t.Errorf("Expected an error, but we didn't get one")
} else if !errors.IsConflict(err) {
t.Errorf("Expected Conflict error, got '%v'", err)
}
}
func TestEtcdUpdate(t *testing.T) {
podA := &api.Pod{
ObjectMeta: api.ObjectMeta{Name: "foo", Namespace: "test"},
Spec: api.PodSpec{NodeName: "machine"},
}
podB := &api.Pod{
ObjectMeta: api.ObjectMeta{Name: "foo", Namespace: "test", ResourceVersion: "1"},
Spec: api.PodSpec{NodeName: "machine2"},
}
podAWithResourceVersion := &api.Pod{
ObjectMeta: api.ObjectMeta{Name: "foo", Namespace: "test", ResourceVersion: "3"},
Spec: api.PodSpec{NodeName: "machine"},
}
nodeWithPodA := tools.EtcdResponseWithError{
R: &etcd.Response{
Node: &etcd.Node{
Value: runtime.EncodeOrDie(testapi.Default.Codec(), podA),
ModifiedIndex: 1,
CreatedIndex: 1,
},
},
E: nil,
}
newerNodeWithPodA := tools.EtcdResponseWithError{
R: &etcd.Response{
Node: &etcd.Node{
Value: runtime.EncodeOrDie(testapi.Default.Codec(), podA),
ModifiedIndex: 2,
CreatedIndex: 1,
},
},
E: nil,
}
nodeWithPodB := tools.EtcdResponseWithError{
R: &etcd.Response{
Node: &etcd.Node{
Value: runtime.EncodeOrDie(testapi.Default.Codec(), podB),
ModifiedIndex: 1,
CreatedIndex: 1,
},
},
E: nil,
}
nodeWithPodAWithResourceVersion := tools.EtcdResponseWithError{
R: &etcd.Response{
Node: &etcd.Node{
Value: runtime.EncodeOrDie(testapi.Default.Codec(), podAWithResourceVersion),
ModifiedIndex: 3,
CreatedIndex: 1,
},
},
E: nil,
}
emptyNode := tools.EtcdResponseWithError{
R: &etcd.Response{},
E: tools.EtcdErrorNotFound,
}
testContext := api.WithNamespace(api.NewContext(), "test")
table := map[string]struct {
existing tools.EtcdResponseWithError
expect tools.EtcdResponseWithError
toUpdate runtime.Object
allowCreate bool
allowUnconditionalUpdate bool
objOK func(obj runtime.Object) bool
errOK func(error) bool
}{
"normal": {
existing: nodeWithPodA,
expect: nodeWithPodB,
toUpdate: podB,
errOK: func(err error) bool { return err == nil },
},
"notExisting": {
existing: emptyNode,
expect: emptyNode,
toUpdate: podA,
errOK: func(err error) bool { return errors.IsNotFound(err) },
},
"createIfNotFound": {
existing: emptyNode,
toUpdate: podA,
allowCreate: true,
objOK: hasCreated(t, podA),
errOK: func(err error) bool { return err == nil },
},
"outOfDate": {
existing: newerNodeWithPodA,
expect: newerNodeWithPodA,
toUpdate: podB,
errOK: func(err error) bool { return errors.IsConflict(err) },
},
"unconditionalUpdate": {
existing: nodeWithPodAWithResourceVersion,
allowUnconditionalUpdate: true,
toUpdate: podA,
objOK: func(obj runtime.Object) bool { return true },
errOK: func(err error) bool { return err == nil },
},
}
for name, item := range table {
fakeClient, registry := NewTestGenericEtcdRegistry(t)
registry.UpdateStrategy.(*testRESTStrategy).allowCreateOnUpdate = item.allowCreate
registry.UpdateStrategy.(*testRESTStrategy).allowUnconditionalUpdate = item.allowUnconditionalUpdate
path := etcdtest.AddPrefix("pods/foo")
fakeClient.Data[path] = item.existing
obj, _, err := registry.Update(testContext, item.toUpdate)
if !item.errOK(err) {
t.Errorf("%v: unexpected error: %v", name, err)
}
actual := fakeClient.Data[path]
if item.objOK != nil {
if !item.objOK(obj) {
t.Errorf("%v: unexpected returned: %#v", name, obj)
}
actualObj, err := api.Scheme.Decode([]byte(actual.R.Node.Value))
if err != nil {
t.Errorf("unable to decode stored value for %#v", actual)
continue
}
if !item.objOK(actualObj) {
t.Errorf("%v: unexpected response: %#v", name, actual)
}
} else {
if e, a := item.expect, actual; !api.Semantic.DeepDerivative(e, a) {
t.Errorf("%v:\n%s", name, util.ObjectDiff(e, a))
}
}
}
}
func runAtomicPutTest(c *client.Client) {
svcBody := api.Service{
TypeMeta: unversioned.TypeMeta{
APIVersion: c.APIVersion(),
},
ObjectMeta: api.ObjectMeta{
Name: "atomicservice",
Labels: map[string]string{
"name": "atomicService",
},
},
Spec: api.ServiceSpec{
// This is here because validation requires it.
Selector: map[string]string{
"foo": "bar",
},
Ports: []api.ServicePort{{
Port: 12345,
Protocol: "TCP",
}},
SessionAffinity: "None",
},
}
services := c.Services(api.NamespaceDefault)
svc, err := services.Create(&svcBody)
if err != nil {
glog.Fatalf("Failed creating atomicService: %v", err)
}
glog.Info("Created atomicService")
testLabels := labels.Set{
"foo": "bar",
}
for i := 0; i < 5; i++ {
// a: z, b: y, etc...
testLabels[string([]byte{byte('a' + i)})] = string([]byte{byte('z' - i)})
}
var wg sync.WaitGroup
wg.Add(len(testLabels))
for label, value := range testLabels {
go func(l, v string) {
for {
glog.Infof("Starting to update (%s, %s)", l, v)
tmpSvc, err := services.Get(svc.Name)
if err != nil {
glog.Errorf("Error getting atomicService: %v", err)
continue
}
if tmpSvc.Spec.Selector == nil {
tmpSvc.Spec.Selector = map[string]string{l: v}
} else {
tmpSvc.Spec.Selector[l] = v
}
glog.Infof("Posting update (%s, %s)", l, v)
tmpSvc, err = services.Update(tmpSvc)
if err != nil {
if apierrors.IsConflict(err) {
glog.Infof("Conflict: (%s, %s)", l, v)
// This is what we expect.
continue
}
glog.Errorf("Unexpected error putting atomicService: %v", err)
continue
}
break
}
glog.Infof("Done update (%s, %s)", l, v)
wg.Done()
}(label, value)
}
wg.Wait()
svc, err = services.Get(svc.Name)
if err != nil {
glog.Fatalf("Failed getting atomicService after writers are complete: %v", err)
}
if !reflect.DeepEqual(testLabels, labels.Set(svc.Spec.Selector)) {
glog.Fatalf("Selector PUTs were not atomic: wanted %v, got %v", testLabels, svc.Spec.Selector)
}
glog.Info("Atomic PUTs work.")
}
// createSecret creates a secret of type ServiceAccountToken for the given ServiceAccount
func (e *TokensController) createSecret(serviceAccount *api.ServiceAccount) error {
// We don't want to update the cache's copy of the service account
// so add the secret to a freshly retrieved copy of the service account
serviceAccounts := e.client.ServiceAccounts(serviceAccount.Namespace)
liveServiceAccount, err := serviceAccounts.Get(serviceAccount.Name)
if err != nil {
return err
}
if liveServiceAccount.ResourceVersion != serviceAccount.ResourceVersion {
// our view of the service account is not up to date
// we'll get notified of an update event later and get to try again
// this only prevent interactions between successive runs of this controller's event handlers, but that is useful
glog.V(2).Infof("View of ServiceAccount %s/%s is not up to date, skipping token creation", serviceAccount.Namespace, serviceAccount.Name)
return nil
}
// Build the secret
secret := &api.Secret{
ObjectMeta: api.ObjectMeta{
Name: secret.Strategy.GenerateName(fmt.Sprintf("%s-token-", serviceAccount.Name)),
Namespace: serviceAccount.Namespace,
Annotations: map[string]string{
api.ServiceAccountNameKey: serviceAccount.Name,
api.ServiceAccountUIDKey: string(serviceAccount.UID),
},
},
Type: api.SecretTypeServiceAccountToken,
Data: map[string][]byte{},
}
// Generate the token
token, err := e.token.GenerateToken(*serviceAccount, *secret)
if err != nil {
return err
}
secret.Data[api.ServiceAccountTokenKey] = []byte(token)
if e.rootCA != nil && len(e.rootCA) > 0 {
secret.Data[api.ServiceAccountRootCAKey] = e.rootCA
}
// Save the secret
if _, err := e.client.Secrets(serviceAccount.Namespace).Create(secret); err != nil {
return err
}
liveServiceAccount.Secrets = append(liveServiceAccount.Secrets, api.ObjectReference{Name: secret.Name})
_, err = serviceAccounts.Update(liveServiceAccount)
if err != nil {
// we weren't able to use the token, try to clean it up.
glog.V(2).Infof("Deleting secret %s/%s because reference couldn't be added (%v)", secret.Namespace, secret.Name, err)
if err := e.client.Secrets(secret.Namespace).Delete(secret.Name); err != nil {
glog.Error(err) // if we fail, just log it
}
}
if apierrors.IsConflict(err) {
// nothing to do. We got a conflict, that means that the service account was updated. We simply need to return because we'll get an update notification later
return nil
}
return err
}
expectNoError(wait.Poll(time.Millisecond*500, time.Second*30, func() (bool, error) {
By("updating the pod")
value = strconv.Itoa(time.Now().Nanosecond())
if pod == nil { // on retries we need to re-get
pod, err = podClient.Get(name)
if err != nil {
return false, fmt.Errorf("failed to get pod: %v", err)
}
}
pod.Labels["time"] = value
pod, err = podClient.Update(pod)
if err == nil {
Logf("Successfully updated pod")
return true, nil
}
if errors.IsConflict(err) {
Logf("Conflicting update to pod, re-get and re-update: %v", err)
pod = nil // re-get it when we retry
return false, nil
}
return false, fmt.Errorf("failed to update pod: %v", err)
}))
expectNoError(framework.WaitForPodRunning(pod.Name))
By("verifying the updated pod is in kubernetes")
pods, err = podClient.List(labels.SelectorFromSet(labels.Set(map[string]string{"time": value})), fields.Everything())
Expect(len(pods.Items)).To(Equal(1))
Logf("Pod update OK")
})