// extendMaster attempts to extend ownership of a master lock for TTL seconds. // returns "", nil if extension failed // returns id, nil if extension succeeded // returns "", err if an error occurred func (e *etcdMasterElector) extendMaster(path, id string, ttl uint64, res *etcd.Response) (string, error) { // If it matches the passed in id, extend the lease by writing a new entry. // Uses compare and swap, so that if we TTL out in the meantime, the write will fail. // We don't handle the TTL delete w/o a write case here, it's handled in the next loop // iteration. _, err := e.etcd.CompareAndSwap(path, id, ttl, "", res.Node.ModifiedIndex) if err != nil && !etcdstorage.IsEtcdTestFailed(err) { return "", err } if err != nil && etcdstorage.IsEtcdTestFailed(err) { return "", nil } return id, nil }
// InterpretUpdateError converts a generic etcd error on a update // operation into the appropriate API error. func InterpretUpdateError(err error, kind, name string) error { switch { case etcdstorage.IsEtcdTestFailed(err), etcdstorage.IsEtcdNodeExist(err): return errors.NewConflict(kind, name, err) default: return err } }
// InterpretUpdateError converts a generic etcd error on a update // operation into the appropriate API error. func InterpretUpdateError(err error, kind, name string) error { switch { case etcdstorage.IsEtcdTestFailed(err), etcdstorage.IsEtcdNodeExist(err): return errors.NewConflict(kind, name, err) case etcdstorage.IsEtcdUnreachable(err): return errors.NewServerTimeout(kind, "update", 2) // TODO: make configurable or handled at a higher level default: return err } }
// Release tries to delete the leader lock. func (e *Etcd) Release() { for i := 0; i < e.maxRetries; i++ { _, err := e.client.CompareAndDelete(e.key, e.value, 0) if err == nil { break } // If the value has changed, we don't hold the lease. If the key is missing we don't // hold the lease. if storage.IsEtcdTestFailed(err) || storage.IsEtcdNotFound(err) { break } util.HandleError(fmt.Errorf("unable to release %s: %v", e.key, err)) } }
// errToAPIStatus converts an error to an unversioned.Status object. func errToAPIStatus(err error) *unversioned.Status { switch t := err.(type) { case statusError: status := t.Status() if len(status.Status) == 0 { status.Status = unversioned.StatusFailure } if status.Code == 0 { switch status.Status { case unversioned.StatusSuccess: status.Code = http.StatusOK case unversioned.StatusFailure: status.Code = http.StatusInternalServerError } } //TODO: check for invalid responses return &status default: status := http.StatusInternalServerError switch { //TODO: replace me with NewConflictErr case etcdstorage.IsEtcdTestFailed(err): status = http.StatusConflict } // Log errors that were not converted to an error status // by REST storage - these typically indicate programmer // error by not using pkg/api/errors, or unexpected failure // cases. util.HandleError(fmt.Errorf("apiserver received an error that is not an unversioned.Status: %v", err)) return &unversioned.Status{ Status: unversioned.StatusFailure, Code: status, Reason: unversioned.StatusReasonUnknown, Message: err.Error(), } } }
// tryHold attempts to hold on to the lease by repeatedly refreshing its TTL. // If the lease hold fails, is deleted, or changed to another user. The provided // index is used to watch from. // TODO: currently if we miss the watch window, we will error and try to recreate // the lock. It's likely we will lose the lease due to that. func (e *Etcd) tryHold(ttl, index uint64) error { // watch for termination stop := make(chan struct{}) lost := make(chan struct{}) watchIndex := index go util.Until(func() { index, err := e.waitForExpiration(true, watchIndex, stop) watchIndex = index if err != nil { util.HandleError(fmt.Errorf("error watching for lease expiration %s: %v", e.key, err)) return } glog.V(4).Infof("Lease %s lost due to deletion", e.key) close(lost) }, 100*time.Millisecond, stop) defer close(stop) duration := time.Duration(ttl) * time.Second after := time.Duration(float32(duration) * e.waitFraction) last := duration - after interval := last / time.Duration(e.maxRetries) if interval < e.minimumRetryInterval { interval = e.minimumRetryInterval } // as long as we can renew the lease, loop for { select { case <-time.After(after): err := wait.Poll(interval, last, func() (bool, error) { glog.V(4).Infof("Renewing lease %s", e.key) resp, err := e.client.CompareAndSwap(e.key, e.value, e.ttl, e.value, index-1) switch { case err == nil: index = eventIndexFor(resp) return true, nil case storage.IsEtcdTestFailed(err): return false, fmt.Errorf("another client has taken the lease %s: %v", e.key, err) case storage.IsEtcdNotFound(err): return false, fmt.Errorf("another client has revoked the lease %s", e.key) default: util.HandleError(fmt.Errorf("unexpected error renewing lease %s: %v", e.key, err)) index = etcdIndexFor(err, index) // try again return false, nil } }) switch err { case nil: // wait again glog.V(4).Infof("Lease %s renewed", e.key) case wait.ErrWaitTimeout: return fmt.Errorf("unable to renew lease %s: %v", e.key, err) default: return fmt.Errorf("lost lease %s: %v", e.key, err) } case <-lost: return fmt.Errorf("the lease has been lost %s", e.key) } } }