// extendMaster attempts to extend ownership of a master lock for TTL seconds.
// returns "", nil if extension failed
// returns id, nil if extension succeeded
// returns "", err if an error occurred
func (e *etcdMasterElector) extendMaster(path, id string, ttl uint64, res *etcd.Response) (string, error) {
// If it matches the passed in id, extend the lease by writing a new entry.
// Uses compare and swap, so that if we TTL out in the meantime, the write will fail.
// We don't handle the TTL delete w/o a write case here, it's handled in the next loop
// iteration.
_, err := e.etcd.CompareAndSwap(path, id, ttl, "", res.Node.ModifiedIndex)
if err != nil && !etcdstorage.IsEtcdTestFailed(err) {
return "", err
}
if err != nil && etcdstorage.IsEtcdTestFailed(err) {
return "", nil
}
return id, nil
}
// InterpretUpdateError converts a generic etcd error on a update
// operation into the appropriate API error.
func InterpretUpdateError(err error, kind, name string) error {
switch {
case etcdstorage.IsEtcdTestFailed(err), etcdstorage.IsEtcdNodeExist(err):
return errors.NewConflict(kind, name, err)
default:
return err
}
}
// InterpretUpdateError converts a generic etcd error on a update
// operation into the appropriate API error.
func InterpretUpdateError(err error, kind, name string) error {
switch {
case etcdstorage.IsEtcdTestFailed(err), etcdstorage.IsEtcdNodeExist(err):
return errors.NewConflict(kind, name, err)
case etcdstorage.IsEtcdUnreachable(err):
return errors.NewServerTimeout(kind, "update", 2) // TODO: make configurable or handled at a higher level
default:
return err
}
}
// Release tries to delete the leader lock.
func (e *Etcd) Release() {
for i := 0; i < e.maxRetries; i++ {
_, err := e.client.CompareAndDelete(e.key, e.value, 0)
if err == nil {
break
}
// If the value has changed, we don't hold the lease. If the key is missing we don't
// hold the lease.
if storage.IsEtcdTestFailed(err) || storage.IsEtcdNotFound(err) {
break
}
util.HandleError(fmt.Errorf("unable to release %s: %v", e.key, err))
}
}
// errToAPIStatus converts an error to an unversioned.Status object.
func errToAPIStatus(err error) *unversioned.Status {
switch t := err.(type) {
case statusError:
status := t.Status()
if len(status.Status) == 0 {
status.Status = unversioned.StatusFailure
}
if status.Code == 0 {
switch status.Status {
case unversioned.StatusSuccess:
status.Code = http.StatusOK
case unversioned.StatusFailure:
status.Code = http.StatusInternalServerError
}
}
//TODO: check for invalid responses
return &status
default:
status := http.StatusInternalServerError
switch {
//TODO: replace me with NewConflictErr
case etcdstorage.IsEtcdTestFailed(err):
status = http.StatusConflict
}
// Log errors that were not converted to an error status
// by REST storage - these typically indicate programmer
// error by not using pkg/api/errors, or unexpected failure
// cases.
util.HandleError(fmt.Errorf("apiserver received an error that is not an unversioned.Status: %v", err))
return &unversioned.Status{
Status: unversioned.StatusFailure,
Code: status,
Reason: unversioned.StatusReasonUnknown,
Message: err.Error(),
}
}
}
// tryHold attempts to hold on to the lease by repeatedly refreshing its TTL.
// If the lease hold fails, is deleted, or changed to another user. The provided
// index is used to watch from.
// TODO: currently if we miss the watch window, we will error and try to recreate
// the lock. It's likely we will lose the lease due to that.
func (e *Etcd) tryHold(ttl, index uint64) error {
// watch for termination
stop := make(chan struct{})
lost := make(chan struct{})
watchIndex := index
go util.Until(func() {
index, err := e.waitForExpiration(true, watchIndex, stop)
watchIndex = index
if err != nil {
util.HandleError(fmt.Errorf("error watching for lease expiration %s: %v", e.key, err))
return
}
glog.V(4).Infof("Lease %s lost due to deletion", e.key)
close(lost)
}, 100*time.Millisecond, stop)
defer close(stop)
duration := time.Duration(ttl) * time.Second
after := time.Duration(float32(duration) * e.waitFraction)
last := duration - after
interval := last / time.Duration(e.maxRetries)
if interval < e.minimumRetryInterval {
interval = e.minimumRetryInterval
}
// as long as we can renew the lease, loop
for {
select {
case <-time.After(after):
err := wait.Poll(interval, last, func() (bool, error) {
glog.V(4).Infof("Renewing lease %s", e.key)
resp, err := e.client.CompareAndSwap(e.key, e.value, e.ttl, e.value, index-1)
switch {
case err == nil:
index = eventIndexFor(resp)
return true, nil
case storage.IsEtcdTestFailed(err):
return false, fmt.Errorf("another client has taken the lease %s: %v", e.key, err)
case storage.IsEtcdNotFound(err):
return false, fmt.Errorf("another client has revoked the lease %s", e.key)
default:
util.HandleError(fmt.Errorf("unexpected error renewing lease %s: %v", e.key, err))
index = etcdIndexFor(err, index)
// try again
return false, nil
}
})
switch err {
case nil:
// wait again
glog.V(4).Infof("Lease %s renewed", e.key)
case wait.ErrWaitTimeout:
return fmt.Errorf("unable to renew lease %s: %v", e.key, err)
default:
return fmt.Errorf("lost lease %s: %v", e.key, err)
}
case <-lost:
return fmt.Errorf("the lease has been lost %s", e.key)
}
}
}