// CreateOrUpdate attempts to update the current etcd state with the provided
// allocation.
func (e *Etcd) CreateOrUpdate(snapshot *api.RangeAllocation) error {
e.lock.Lock()
defer e.lock.Unlock()
last := ""
err := e.storage.GuaranteedUpdate(context.TODO(), e.baseKey, &api.RangeAllocation{}, true, nil,
storage.SimpleUpdate(func(input runtime.Object) (output runtime.Object, err error) {
existing := input.(*api.RangeAllocation)
switch {
case len(snapshot.ResourceVersion) != 0 && len(existing.ResourceVersion) != 0:
if snapshot.ResourceVersion != existing.ResourceVersion {
return nil, k8serr.NewConflict(e.resource, "", fmt.Errorf("the provided resource version does not match"))
}
case len(existing.ResourceVersion) != 0:
return nil, k8serr.NewConflict(e.resource, "", fmt.Errorf("another caller has already initialized the resource"))
}
last = snapshot.ResourceVersion
return snapshot, nil
}),
)
if err != nil {
return storeerr.InterpretUpdateError(err, e.resource, "")
}
err = e.alloc.Restore(snapshot.Range, snapshot.Data)
if err == nil {
e.last = last
}
return err
}
func (r *RollbackREST) rollbackDeployment(ctx api.Context, deploymentID string, config *extensions.RollbackConfig, annotations map[string]string) (err error) {
if _, err = r.setDeploymentRollback(ctx, deploymentID, config, annotations); err != nil {
err = storeerr.InterpretGetError(err, extensions.Resource("deployments"), deploymentID)
err = storeerr.InterpretUpdateError(err, extensions.Resource("deployments"), deploymentID)
if _, ok := err.(*errors.StatusError); !ok {
err = errors.NewConflict(extensions.Resource("deployments/rollback"), deploymentID, err)
}
}
return
}
func TestApplyRetry(t *testing.T) {
initTestErrorHandler(t)
nameRC, currentRC := readAndAnnotateReplicationController(t, filenameRC)
pathRC := "/namespaces/test/replicationcontrollers/" + nameRC
firstPatch := true
retry := false
getCount := 0
f, tf, codec := NewAPIFactory()
tf.Printer = &testPrinter{}
tf.Client = &fake.RESTClient{
Codec: codec,
Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
switch p, m := req.URL.Path, req.Method; {
case p == pathRC && m == "GET":
getCount++
bodyRC := ioutil.NopCloser(bytes.NewReader(currentRC))
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: bodyRC}, nil
case p == pathRC && m == "PATCH":
if firstPatch {
firstPatch = false
statusErr := kubeerr.NewConflict(unversioned.GroupResource{Group: "", Resource: "rc"}, "test-rc", fmt.Errorf("the object has been modified. Please apply at first."))
bodyBytes, _ := json.Marshal(statusErr)
bodyErr := ioutil.NopCloser(bytes.NewReader(bodyBytes))
return &http.Response{StatusCode: http.StatusConflict, Header: defaultHeader(), Body: bodyErr}, nil
}
retry = true
validatePatchApplication(t, req)
bodyRC := ioutil.NopCloser(bytes.NewReader(currentRC))
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: bodyRC}, nil
default:
t.Fatalf("unexpected request: %#v\n%#v", req.URL, req)
return nil, nil
}
}),
}
tf.Namespace = "test"
buf := bytes.NewBuffer([]byte{})
cmd := NewCmdApply(f, buf)
cmd.Flags().Set("filename", filenameRC)
cmd.Flags().Set("output", "name")
cmd.Run(cmd, []string{})
if !retry || getCount != 2 {
t.Fatalf("apply didn't retry when get conflict error")
}
// uses the name from the file, not the response
expectRC := "replicationcontroller/" + nameRC + "\n"
if buf.String() != expectRC {
t.Fatalf("unexpected output: %s\nexpected: %s", buf.String(), expectRC)
}
}
// InterpretDeleteError converts a generic error on a delete
// operation into the appropriate API error.
func InterpretDeleteError(err error, qualifiedResource unversioned.GroupResource, name string) error {
switch {
case storage.IsNotFound(err):
return errors.NewNotFound(qualifiedResource, name)
case storage.IsUnreachable(err):
return errors.NewServerTimeout(qualifiedResource, "delete", 2) // TODO: make configurable or handled at a higher level
case storage.IsTestFailed(err), storage.IsNodeExist(err):
return errors.NewConflict(qualifiedResource, name, err)
case storage.IsInternalError(err):
return errors.NewInternalError(err)
default:
return err
}
}
// BeforeDelete tests whether the object can be gracefully deleted. If graceful is set the object
// should be gracefully deleted, if gracefulPending is set the object has already been gracefully deleted
// (and the provided grace period is longer than the time to deletion), and an error is returned if the
// condition cannot be checked or the gracePeriodSeconds is invalid. The options argument may be updated with
// default values if graceful is true.
func BeforeDelete(strategy RESTDeleteStrategy, ctx api.Context, obj runtime.Object, options *api.DeleteOptions) (graceful, gracefulPending bool, err error) {
objectMeta, gvk, kerr := objectMetaAndKind(strategy, obj)
if kerr != nil {
return false, false, kerr
}
// Checking the Preconditions here to fail early. They'll be enforced later on when we actually do the deletion, too.
if options.Preconditions != nil && options.Preconditions.UID != nil && *options.Preconditions.UID != objectMeta.UID {
return false, false, errors.NewConflict(unversioned.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.Name, fmt.Errorf("the UID in the precondition (%s) does not match the UID in record (%s). The object might have been deleted and then recreated", *options.Preconditions.UID, objectMeta.UID))
}
gracefulStrategy, ok := strategy.(RESTGracefulDeleteStrategy)
if !ok {
return false, false, nil
}
// if the object is already being deleted
if objectMeta.DeletionTimestamp != nil {
// if we are already being deleted, we may only shorten the deletion grace period
// this means the object was gracefully deleted previously but deletionGracePeriodSeconds was not set,
// so we force deletion immediately
if objectMeta.DeletionGracePeriodSeconds == nil {
return false, false, nil
}
// only a shorter grace period may be provided by a user
if options.GracePeriodSeconds != nil {
period := int64(*options.GracePeriodSeconds)
if period > *objectMeta.DeletionGracePeriodSeconds {
return false, true, nil
}
now := unversioned.NewTime(unversioned.Now().Add(time.Second * time.Duration(*options.GracePeriodSeconds)))
objectMeta.DeletionTimestamp = &now
objectMeta.DeletionGracePeriodSeconds = &period
options.GracePeriodSeconds = &period
return true, false, nil
}
// graceful deletion is pending, do nothing
options.GracePeriodSeconds = objectMeta.DeletionGracePeriodSeconds
return false, true, nil
}
if !gracefulStrategy.CheckGracefulDelete(obj, options) {
return false, false, nil
}
now := unversioned.NewTime(unversioned.Now().Add(time.Second * time.Duration(*options.GracePeriodSeconds)))
objectMeta.DeletionTimestamp = &now
objectMeta.DeletionGracePeriodSeconds = options.GracePeriodSeconds
return true, false, nil
}
func (p *testPatcher) Update(ctx api.Context, name string, objInfo rest.UpdatedObjectInfo) (runtime.Object, bool, error) {
currentPod := p.startingPod
if p.numUpdates > 0 {
currentPod = p.updatePod
}
p.numUpdates++
obj, err := objInfo.UpdatedObject(ctx, currentPod)
if err != nil {
return nil, false, err
}
inPod := obj.(*api.Pod)
if inPod.ResourceVersion != p.updatePod.ResourceVersion {
return nil, false, apierrors.NewConflict(api.Resource("pods"), inPod.Name, fmt.Errorf("existing %v, new %v", p.updatePod.ResourceVersion, inPod.ResourceVersion))
}
return inPod, false, nil
}
func (rs *REST) Update(ctx api.Context, name string, objInfo rest.UpdatedObjectInfo) (runtime.Object, bool, error) {
oldService, err := rs.registry.GetService(ctx, name)
if err != nil {
return nil, false, err
}
obj, err := objInfo.UpdatedObject(ctx, oldService)
if err != nil {
return nil, false, err
}
service := obj.(*api.Service)
if !api.ValidNamespace(ctx, &service.ObjectMeta) {
return nil, false, errors.NewConflict(api.Resource("services"), service.Namespace, fmt.Errorf("Service.Namespace does not match the provided context"))
}
// Copy over non-user fields
// TODO: make this a merge function
if errs := validation.ValidateServiceUpdate(service, oldService); len(errs) > 0 {
return nil, false, errors.NewInvalid(api.Kind("Service"), service.Name, errs)
}
nodePortOp := portallocator.StartOperation(rs.serviceNodePorts)
defer nodePortOp.Finish()
assignNodePorts := shouldAssignNodePorts(service)
oldNodePorts := CollectServiceNodePorts(oldService)
newNodePorts := []int{}
if assignNodePorts {
for i := range service.Spec.Ports {
servicePort := &service.Spec.Ports[i]
nodePort := int(servicePort.NodePort)
if nodePort != 0 {
if !contains(oldNodePorts, nodePort) {
err := nodePortOp.Allocate(nodePort)
if err != nil {
el := field.ErrorList{field.Invalid(field.NewPath("spec", "ports").Index(i).Child("nodePort"), nodePort, err.Error())}
return nil, false, errors.NewInvalid(api.Kind("Service"), service.Name, el)
}
}
} else {
nodePort, err = nodePortOp.AllocateNext()
if err != nil {
// TODO: what error should be returned here? It's not a
// field-level validation failure (the field is valid), and it's
// not really an internal error.
return nil, false, errors.NewInternalError(fmt.Errorf("failed to allocate a nodePort: %v", err))
}
servicePort.NodePort = int32(nodePort)
}
// Detect duplicate node ports; this should have been caught by validation, so we panic
if contains(newNodePorts, nodePort) {
panic("duplicate node port")
}
newNodePorts = append(newNodePorts, nodePort)
}
} else {
// Validate should have validated that nodePort == 0
}
// The comparison loops are O(N^2), but we don't expect N to be huge
// (there's a hard-limit at 2^16, because they're ports; and even 4 ports would be a lot)
for _, oldNodePort := range oldNodePorts {
if !contains(newNodePorts, oldNodePort) {
continue
}
nodePortOp.ReleaseDeferred(oldNodePort)
}
// Remove any LoadBalancerStatus now if Type != LoadBalancer;
// although loadbalancer delete is actually asynchronous, we don't need to expose the user to that complexity.
if service.Spec.Type != api.ServiceTypeLoadBalancer {
service.Status.LoadBalancer = api.LoadBalancerStatus{}
}
out, err := rs.registry.UpdateService(ctx, service)
if err == nil {
el := nodePortOp.Commit()
if el != nil {
// problems should be fixed by an eventual reconciliation / restart
glog.Errorf("error(s) committing NodePorts changes: %v", el)
}
}
return out, false, err
}
// patchResource divides PatchResource for easier unit testing
func patchResource(
ctx api.Context,
admit updateAdmissionFunc,
timeout time.Duration,
versionedObj runtime.Object,
patcher rest.Patcher,
name string,
patchType api.PatchType,
patchJS []byte,
namer ScopeNamer,
copier runtime.ObjectCopier,
resource unversioned.GroupVersionResource,
codec runtime.Codec,
) (runtime.Object, error) {
namespace := api.NamespaceValue(ctx)
var (
originalObjJS []byte
originalPatchedObjJS []byte
lastConflictErr error
)
// applyPatch is called every time GuaranteedUpdate asks for the updated object,
// and is given the currently persisted object as input.
applyPatch := func(_ api.Context, _, currentObject runtime.Object) (runtime.Object, error) {
// Make sure we actually have a persisted currentObject
if hasUID, err := hasUID(currentObject); err != nil {
return nil, err
} else if !hasUID {
return nil, errors.NewNotFound(resource.GroupResource(), name)
}
switch {
case len(originalObjJS) == 0 || len(originalPatchedObjJS) == 0:
// first time through,
// 1. apply the patch
// 2. save the originalJS and patchedJS to detect whether there were conflicting changes on retries
if js, err := runtime.Encode(codec, currentObject); err != nil {
return nil, err
} else {
originalObjJS = js
}
if js, err := getPatchedJS(patchType, originalObjJS, patchJS, versionedObj); err != nil {
return nil, err
} else {
originalPatchedObjJS = js
}
objToUpdate := patcher.New()
if err := runtime.DecodeInto(codec, originalPatchedObjJS, objToUpdate); err != nil {
return nil, err
}
if err := checkName(objToUpdate, name, namespace, namer); err != nil {
return nil, err
}
return objToUpdate, nil
default:
// on a conflict,
// 1. build a strategic merge patch from originalJS and the patchedJS. Different patch types can
// be specified, but a strategic merge patch should be expressive enough handle them. Build the
// patch with this type to handle those cases.
// 2. build a strategic merge patch from originalJS and the currentJS
// 3. ensure no conflicts between the two patches
// 4. apply the #1 patch to the currentJS object
currentObjectJS, err := runtime.Encode(codec, currentObject)
if err != nil {
return nil, err
}
currentPatch, err := strategicpatch.CreateStrategicMergePatch(originalObjJS, currentObjectJS, versionedObj)
if err != nil {
return nil, err
}
originalPatch, err := strategicpatch.CreateStrategicMergePatch(originalObjJS, originalPatchedObjJS, versionedObj)
if err != nil {
return nil, err
}
diff1 := make(map[string]interface{})
if err := json.Unmarshal(originalPatch, &diff1); err != nil {
return nil, err
}
diff2 := make(map[string]interface{})
if err := json.Unmarshal(currentPatch, &diff2); err != nil {
return nil, err
}
hasConflicts, err := strategicpatch.HasConflicts(diff1, diff2)
if err != nil {
return nil, err
}
if hasConflicts {
glog.V(4).Infof("patchResource failed for resource %s, because there is a meaningful conflict.\n diff1=%v\n, diff2=%v\n", name, diff1, diff2)
// Return the last conflict error we got if we have one
if lastConflictErr != nil {
return nil, lastConflictErr
}
// Otherwise manufacture one of our own
return nil, errors.NewConflict(resource.GroupResource(), name, nil)
}
newlyPatchedObjJS, err := getPatchedJS(api.StrategicMergePatchType, currentObjectJS, originalPatch, versionedObj)
if err != nil {
return nil, err
}
objToUpdate := patcher.New()
if err := runtime.DecodeInto(codec, newlyPatchedObjJS, objToUpdate); err != nil {
return nil, err
}
return objToUpdate, nil
}
}
// applyAdmission is called every time GuaranteedUpdate asks for the updated object,
// and is given the currently persisted object and the patched object as input.
applyAdmission := func(ctx api.Context, patchedObject runtime.Object, currentObject runtime.Object) (runtime.Object, error) {
return patchedObject, admit(patchedObject, currentObject)
}
updatedObjectInfo := rest.DefaultUpdatedObjectInfo(nil, copier, applyPatch, applyAdmission)
return finishRequest(timeout, func() (runtime.Object, error) {
updateObject, _, updateErr := patcher.Update(ctx, name, updatedObjectInfo)
for i := 0; i < MaxPatchConflicts && (errors.IsConflict(updateErr)); i++ {
lastConflictErr = updateErr
updateObject, _, updateErr = patcher.Update(ctx, name, updatedObjectInfo)
}
return updateObject, updateErr
})
}
// Delete enforces life-cycle rules for namespace termination
func (r *REST) Delete(ctx api.Context, name string, options *api.DeleteOptions) (runtime.Object, error) {
nsObj, err := r.Get(ctx, name)
if err != nil {
return nil, err
}
namespace := nsObj.(*api.Namespace)
// Ensure we have a UID precondition
if options == nil {
options = api.NewDeleteOptions(0)
}
if options.Preconditions == nil {
options.Preconditions = &api.Preconditions{}
}
if options.Preconditions.UID == nil {
options.Preconditions.UID = &namespace.UID
} else if *options.Preconditions.UID != namespace.UID {
err = apierrors.NewConflict(
api.Resource("namespaces"),
name,
fmt.Errorf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *options.Preconditions.UID, namespace.UID),
)
return nil, err
}
// upon first request to delete, we switch the phase to start namespace termination
// TODO: enhance graceful deletion's calls to DeleteStrategy to allow phase change and finalizer patterns
if namespace.DeletionTimestamp.IsZero() {
key, err := r.Store.KeyFunc(ctx, name)
if err != nil {
return nil, err
}
preconditions := storage.Preconditions{UID: options.Preconditions.UID}
out := r.Store.NewFunc()
err = r.Store.Storage.GuaranteedUpdate(
ctx, key, out, false, &preconditions,
storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
existingNamespace, ok := existing.(*api.Namespace)
if !ok {
// wrong type
return nil, fmt.Errorf("expected *api.Namespace, got %v", existing)
}
// Set the deletion timestamp if needed
if existingNamespace.DeletionTimestamp.IsZero() {
now := unversioned.Now()
existingNamespace.DeletionTimestamp = &now
}
// Set the namespace phase to terminating, if needed
if existingNamespace.Status.Phase != api.NamespaceTerminating {
existingNamespace.Status.Phase = api.NamespaceTerminating
}
return existingNamespace, nil
}),
)
if err != nil {
err = storageerr.InterpretGetError(err, api.Resource("namespaces"), name)
err = storageerr.InterpretUpdateError(err, api.Resource("namespaces"), name)
if _, ok := err.(*apierrors.StatusError); !ok {
err = apierrors.NewInternalError(err)
}
return nil, err
}
return out, nil
}
// prior to final deletion, we must ensure that finalizers is empty
if len(namespace.Spec.Finalizers) != 0 {
err = apierrors.NewConflict(api.Resource("namespaces"), namespace.Name, fmt.Errorf("The system is ensuring all content is removed from this namespace. Upon completion, this namespace will automatically be purged by the system."))
return nil, err
}
return r.Store.Delete(ctx, name, options)
}
