// extendMaster attempts to extend ownership of a master lock for TTL seconds.
// returns "", nil if extension failed
// returns id, nil if extension succeeded
// returns "", err if an error occurred
func (e *etcdMasterElector) extendMaster(path, id string, ttl uint64, res *etcd.Response) (string, error) {
// If it matches the passed in id, extend the lease by writing a new entry.
// Uses compare and swap, so that if we TTL out in the meantime, the write will fail.
// We don't handle the TTL delete w/o a write case here, it's handled in the next loop
// iteration.
_, err := e.etcd.CompareAndSwap(path, id, ttl, "", res.Node.ModifiedIndex)
if err != nil && !etcdutil.IsEtcdTestFailed(err) {
return "", err
}
if err != nil && etcdutil.IsEtcdTestFailed(err) {
return "", nil
}
return id, nil
}
// Implements storage.Interface.
func (h *etcdHelper) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions) error {
if ctx == nil {
glog.Errorf("Context is nil")
}
key = h.prefixEtcdKey(key)
v, err := conversion.EnforcePtr(out)
if err != nil {
panic("unable to convert output object to pointer")
}
if preconditions == nil {
startTime := time.Now()
response, err := h.etcdKeysAPI.Delete(ctx, key, nil)
metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime)
if !etcdutil.IsEtcdNotFound(err) {
// if the object that existed prior to the delete is returned by etcd, update the out object.
if err != nil || response.PrevNode != nil {
_, _, err = h.extractObj(response, err, out, false, true)
}
}
return toStorageErr(err, key, 0)
}
// Check the preconditions match.
obj := reflect.New(v.Type()).Interface().(runtime.Object)
for {
_, node, res, err := h.bodyAndExtractObj(ctx, key, obj, false)
if err != nil {
return toStorageErr(err, key, 0)
}
if err := checkPreconditions(key, preconditions, obj); err != nil {
return toStorageErr(err, key, 0)
}
index := uint64(0)
if node != nil {
index = node.ModifiedIndex
} else if res != nil {
index = res.Index
}
opt := etcd.DeleteOptions{PrevIndex: index}
startTime := time.Now()
response, err := h.etcdKeysAPI.Delete(ctx, key, &opt)
metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime)
if etcdutil.IsEtcdTestFailed(err) {
glog.Infof("deletion of %s failed because of a conflict, going to retry", key)
} else {
if !etcdutil.IsEtcdNotFound(err) {
// if the object that existed prior to the delete is returned by etcd, update the out object.
if err != nil || response.PrevNode != nil {
_, _, err = h.extractObj(response, err, out, false, true)
}
}
return toStorageErr(err, key, 0)
}
}
}
// InterpretUpdateError converts a generic etcd error on a update
// operation into the appropriate API error.
func InterpretUpdateError(err error, kind, name string) error {
switch {
case etcdutil.IsEtcdTestFailed(err), etcdutil.IsEtcdNodeExist(err):
return errors.NewConflict(kind, name, err)
case etcdutil.IsEtcdUnreachable(err):
return errors.NewServerTimeout(kind, "update", 2) // TODO: make configurable or handled at a higher level
default:
return err
}
}
// extendMaster attempts to extend ownership of a master lock for TTL seconds.
// returns "", nil if extension failed
// returns id, nil if extension succeeded
// returns "", err if an error occurred
func (e *etcdMasterElector) extendMaster(path, id string, ttl uint64, res *etcd.Response) (string, error) {
// If it matches the passed in id, extend the lease by writing a new entry.
// Uses compare and swap, so that if we TTL out in the meantime, the write will fail.
// We don't handle the TTL delete w/o a write case here, it's handled in the next loop
// iteration.
opts := etcd.SetOptions{
TTL: time.Duration(ttl) * time.Second,
PrevValue: "",
PrevIndex: res.Node.ModifiedIndex,
}
_, err := e.etcd.Set(context.TODO(), path, id, &opts)
if err != nil && !etcdutil.IsEtcdTestFailed(err) {
return "", err
}
if err != nil && etcdutil.IsEtcdTestFailed(err) {
return "", nil
}
return id, nil
}
// Release tries to delete the leader lock.
func (e *Etcd) Release() {
for i := 0; i < e.maxRetries; i++ {
_, err := e.client.CompareAndDelete(e.key, e.value, 0)
if err == nil {
break
}
// If the value has changed, we don't hold the lease. If the key is missing we don't
// hold the lease.
if etcdutil.IsEtcdTestFailed(err) || etcdutil.IsEtcdNotFound(err) {
break
}
utilruntime.HandleError(fmt.Errorf("unable to release %s: %v", e.key, err))
}
}
// Release tries to delete the leader lock.
func (e *Etcd) Release() {
for i := 0; i < e.maxRetries; i++ {
_, err := e.client.Delete(context.Background(), e.key, &etcdclient.DeleteOptions{PrevValue: e.value})
if err == nil {
break
}
// If the value has changed, we don't hold the lease. If the key is missing we don't
// hold the lease.
if etcdutil.IsEtcdTestFailed(err) || etcdutil.IsEtcdNotFound(err) {
break
}
utilruntime.HandleError(fmt.Errorf("unable to release %s: %v", e.key, err))
}
}
func toStorageErr(err error, key string, rv int64) error {
if err == nil {
return nil
}
switch {
case etcdutil.IsEtcdNotFound(err):
return storage.NewKeyNotFoundError(key, rv)
case etcdutil.IsEtcdNodeExist(err):
return storage.NewKeyExistsError(key, rv)
case etcdutil.IsEtcdTestFailed(err):
return storage.NewResourceVersionConflictsError(key, rv)
case etcdutil.IsEtcdUnreachable(err):
return storage.NewUnreachableError(key, rv)
default:
return err
}
}
// errToAPIStatus converts an error to an unversioned.Status object.
func errToAPIStatus(err error) *unversioned.Status {
switch t := err.(type) {
case statusError:
status := t.Status()
if len(status.Status) == 0 {
status.Status = unversioned.StatusFailure
}
if status.Code == 0 {
switch status.Status {
case unversioned.StatusSuccess:
status.Code = http.StatusOK
case unversioned.StatusFailure:
status.Code = http.StatusInternalServerError
}
}
//TODO: check for invalid responses
return &status
default:
status := http.StatusInternalServerError
switch {
//TODO: replace me with NewConflictErr
case etcdutil.IsEtcdTestFailed(err):
status = http.StatusConflict
}
// Log errors that were not converted to an error status
// by REST storage - these typically indicate programmer
// error by not using pkg/api/errors, or unexpected failure
// cases.
util.HandleError(fmt.Errorf("apiserver received an error that is not an unversioned.Status: %v", err))
return &unversioned.Status{
Status: unversioned.StatusFailure,
Code: int32(status),
Reason: unversioned.StatusReasonUnknown,
Message: err.Error(),
}
}
}
// Implements storage.Interface.
func (h *etcdHelper) GuaranteedUpdate(ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool, tryUpdate storage.UpdateFunc) error {
if ctx == nil {
glog.Errorf("Context is nil")
}
v, err := conversion.EnforcePtr(ptrToType)
if err != nil {
// Panic is appropriate, because this is a programming error.
panic("need ptr to type")
}
key = h.prefixEtcdKey(key)
for {
obj := reflect.New(v.Type()).Interface().(runtime.Object)
origBody, node, res, err := h.bodyAndExtractObj(ctx, key, obj, ignoreNotFound)
if err != nil {
return err
}
meta := storage.ResponseMeta{}
if node != nil {
meta.TTL = node.TTL
if node.Expiration != nil {
meta.Expiration = node.Expiration
}
meta.ResourceVersion = node.ModifiedIndex
}
// Get the object to be written by calling tryUpdate.
ret, newTTL, err := tryUpdate(obj, meta)
if err != nil {
return err
}
index := uint64(0)
ttl := uint64(0)
if node != nil {
index = node.ModifiedIndex
if node.TTL != 0 {
ttl = uint64(node.TTL)
}
if node.Expiration != nil && ttl == 0 {
ttl = 1
}
} else if res != nil {
index = res.Index
}
if newTTL != nil {
if ttl != 0 && *newTTL == 0 {
// TODO: remove this after we have verified this is no longer an issue
glog.V(4).Infof("GuaranteedUpdate is clearing TTL for %q, may not be intentional", key)
}
ttl = *newTTL
}
// Since update object may have a resourceVersion set, we need to clear it here.
if err := h.versioner.UpdateObject(ret, meta.Expiration, 0); err != nil {
return errors.New("resourceVersion cannot be set on objects store in etcd")
}
data, err := runtime.Encode(h.codec, ret)
if err != nil {
return err
}
// First time this key has been used, try creating new value.
if index == 0 {
startTime := time.Now()
opts := etcd.SetOptions{
TTL: time.Duration(ttl) * time.Second,
PrevExist: etcd.PrevNoExist,
}
response, err := h.etcdKeysAPI.Set(ctx, key, string(data), &opts)
metrics.RecordEtcdRequestLatency("create", getTypeName(ptrToType), startTime)
if etcdutil.IsEtcdNodeExist(err) {
continue
}
_, _, err = h.extractObj(response, err, ptrToType, false, false)
return err
}
if string(data) == origBody {
// If we don't send an update, we simply return the currently existing
// version of the object.
_, _, err := h.extractObj(res, nil, ptrToType, ignoreNotFound, false)
return err
}
startTime := time.Now()
// Swap origBody with data, if origBody is the latest etcd data.
opts := etcd.SetOptions{
PrevValue: origBody,
PrevIndex: index,
TTL: time.Duration(ttl) * time.Second,
}
response, err := h.etcdKeysAPI.Set(ctx, key, string(data), &opts)
metrics.RecordEtcdRequestLatency("compareAndSwap", getTypeName(ptrToType), startTime)
if etcdutil.IsEtcdTestFailed(err) {
// Try again.
continue
}
_, _, err = h.extractObj(response, err, ptrToType, false, false)
return err
}
}
// tryHold attempts to hold on to the lease by repeatedly refreshing its TTL.
// If the lease hold fails, is deleted, or changed to another user. The provided
// index is used to watch from.
// TODO: currently if we miss the watch window, we will error and try to recreate
// the lock. It's likely we will lose the lease due to that.
func (e *Etcd) tryHold(ttl, index uint64) error {
// watch for termination
stop := make(chan struct{})
lost := make(chan struct{})
closedLost := false
watchIndex := index
go utilwait.Until(func() {
index, err := e.waitForExpiration(true, watchIndex, stop)
watchIndex = index
if err != nil {
utilruntime.HandleError(fmt.Errorf("error watching for lease expiration %s: %v", e.key, err))
return
}
glog.V(4).Infof("Lease %s lost due to deletion at %d", e.key, watchIndex)
if !closedLost {
closedLost = true
close(lost)
}
}, 100*time.Millisecond, stop)
defer close(stop)
duration := time.Duration(ttl) * time.Second
after := time.Duration(float32(duration) * e.waitFraction)
last := duration - after
interval := last / time.Duration(e.maxRetries)
if interval < e.minimumRetryInterval {
interval = e.minimumRetryInterval
}
// as long as we can renew the lease, loop
for {
select {
case <-time.After(after):
err := wait.Poll(interval, last, func() (bool, error) {
glog.V(4).Infof("Renewing lease %s at %d", e.key, index-1)
resp, err := e.client.CompareAndSwap(e.key, e.value, e.ttl, e.value, index-1)
switch {
case err == nil:
index = eventIndexFor(resp)
return true, nil
case etcdutil.IsEtcdTestFailed(err):
return false, fmt.Errorf("another client has taken the lease %s: %v", e.key, err)
case etcdutil.IsEtcdNotFound(err):
return false, fmt.Errorf("another client has revoked the lease %s", e.key)
default:
utilruntime.HandleError(fmt.Errorf("unexpected error renewing lease %s: %v", e.key, err))
index = etcdIndexFor(err, index)
// try again
return false, nil
}
})
switch err {
case nil:
// wait again
glog.V(4).Infof("Lease %s renewed at %d", e.key, index-1)
case wait.ErrWaitTimeout:
return fmt.Errorf("unable to renew lease %s at %d: %v", e.key, index, err)
default:
return fmt.Errorf("lost lease %s at %d: %v", e.key, index, err)
}
case <-lost:
return fmt.Errorf("the lease has been lost %s at %d", e.key, index)
}
}
}
// IsTestFailed returns true if and only if err is a write conflict.
func IsTestFailed(err error) bool {
// TODO: add alternate storage error here
return etcdutil.IsEtcdTestFailed(err)
}
// Implements storage.Interface.
func (h *etcdHelper) GuaranteedUpdate(ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool, tryUpdate storage.UpdateFunc) error {
if ctx == nil {
glog.Errorf("Context is nil")
}
v, err := conversion.EnforcePtr(ptrToType)
if err != nil {
// Panic is appropriate, because this is a programming error.
panic("need ptr to type")
}
key = h.prefixEtcdKey(key)
for {
obj := reflect.New(v.Type()).Interface().(runtime.Object)
origBody, node, res, err := h.bodyAndExtractObj(ctx, key, obj, ignoreNotFound)
if err != nil {
return err
}
meta := storage.ResponseMeta{}
if node != nil {
meta.TTL = node.TTL
if node.Expiration != nil {
meta.Expiration = node.Expiration
}
meta.ResourceVersion = node.ModifiedIndex
}
// Get the object to be written by calling tryUpdate.
ret, newTTL, err := tryUpdate(obj, meta)
if err != nil {
return err
}
index := uint64(0)
ttl := uint64(0)
if node != nil {
index = node.ModifiedIndex
if node.TTL != 0 {
ttl = uint64(node.TTL)
}
if node.Expiration != nil && ttl == 0 {
ttl = 1
}
} else if res != nil {
index = res.EtcdIndex
}
if newTTL != nil {
if ttl != 0 && *newTTL == 0 {
// TODO: remove this after we have verified this is no longer an issue
glog.V(4).Infof("GuaranteedUpdate is clearing TTL for %q, may not be intentional", key)
}
ttl = *newTTL
}
data, err := h.codec.Encode(ret)
if err != nil {
return err
}
// First time this key has been used, try creating new value.
if index == 0 {
startTime := time.Now()
response, err := h.client.Create(key, string(data), ttl)
metrics.RecordEtcdRequestLatency("create", getTypeName(ptrToType), startTime)
if etcdutil.IsEtcdNodeExist(err) {
continue
}
_, _, err = h.extractObj(response, err, ptrToType, false, false)
return err
}
if string(data) == origBody {
return nil
}
startTime := time.Now()
// Swap origBody with data, if origBody is the latest etcd data.
response, err := h.client.CompareAndSwap(key, string(data), ttl, origBody, index)
metrics.RecordEtcdRequestLatency("compareAndSwap", getTypeName(ptrToType), startTime)
if etcdutil.IsEtcdTestFailed(err) {
// Try again.
continue
}
_, _, err = h.extractObj(response, err, ptrToType, false, false)
return err
}
}