// becomeMaster attempts to become the master for this lock. // returns "", nil if the attempt failed // returns id, nil if the attempt succeeded // returns "", err if an error occurred func (e *etcdMasterElector) becomeMaster(path, id string, ttl uint64) (string, error) { _, err := e.etcd.Create(path, id, ttl) if err != nil && !etcdutil.IsEtcdNodeExist(err) { // unexpected error return "", err } if err != nil && etcdutil.IsEtcdNodeExist(err) { return "", nil } return id, nil }
// becomeMaster attempts to become the master for this lock. // returns "", nil if the attempt failed // returns id, nil if the attempt succeeded // returns "", err if an error occurred func (e *etcdMasterElector) becomeMaster(path, id string, ttl uint64) (string, error) { opts := etcd.SetOptions{ TTL: time.Duration(ttl) * time.Second, PrevExist: etcd.PrevNoExist, } _, err := e.etcd.Set(context.TODO(), path, id, &opts) if err != nil && !etcdutil.IsEtcdNodeExist(err) { // unexpected error return "", err } if err != nil && etcdutil.IsEtcdNodeExist(err) { return "", nil } return id, nil }
// InterpretUpdateError converts a generic etcd error on a update // operation into the appropriate API error. func InterpretUpdateError(err error, kind, name string) error { switch { case etcdutil.IsEtcdTestFailed(err), etcdutil.IsEtcdNodeExist(err): return errors.NewConflict(kind, name, err) case etcdutil.IsEtcdUnreachable(err): return errors.NewServerTimeout(kind, "update", 2) // TODO: make configurable or handled at a higher level default: return err } }
// InterpretCreateError converts a generic etcd error on a create // operation into the appropriate API error. func InterpretCreateError(err error, kind, name string) error { switch { case etcdutil.IsEtcdNodeExist(err): return errors.NewAlreadyExists(kind, name) case etcdutil.IsEtcdUnreachable(err): return errors.NewServerTimeout(kind, "create", 2) // TODO: make configurable or handled at a higher level default: return err } }
// tryAcquire tries to create the lease key in etcd, or if it already exists // and belongs to another user, to wait until the lease expires or is deleted. // It returns true if the lease was acquired, the current TTL, the nextIndex // to watch from, or an error. func (e *Etcd) tryAcquire() (ok bool, ttl uint64, nextIndex uint64, err error) { ttl = e.ttl resp, err := e.client.Set( context.Background(), e.key, e.value, &etcdclient.SetOptions{ TTL: time.Duration(ttl) * time.Second, PrevExist: etcdclient.PrevNoExist, }, ) if err == nil { // we hold the lease index := resp.Index glog.V(4).Infof("Lease %s acquired at %d, ttl %d seconds", e.key, index, e.ttl) return true, ttl, index + 1, nil } if !etcdutil.IsEtcdNodeExist(err) { return false, 0, 0, fmt.Errorf("unable to check lease %s: %v", e.key, err) } latest, err := e.client.Get(context.Background(), e.key, nil) if err != nil { return false, 0, 0, fmt.Errorf("unable to retrieve lease %s: %v", e.key, err) } nextIndex = eventIndexFor(latest) if latest.Node.TTL > 0 { ttl = uint64(latest.Node.TTL) } if latest.Node.Value != e.value { glog.V(4).Infof("Lease %s owned by %s at %d ttl %d seconds, waiting for expiration", e.key, latest.Node.Value, nextIndex-1, ttl) // waits until the lease expires or changes to us. // TODO: it's possible we were given the lease during the watch, but we just expect to go // through this loop again and let this condition check if _, err := e.waitForExpiration(false, nextIndex, nil); err != nil { return false, 0, 0, fmt.Errorf("unable to wait for lease expiration %s: %v", e.key, err) } return false, 0, 0, nil } glog.V(4).Infof("Lease %s already held, expires in %d seconds", e.key, ttl) return true, ttl, nextIndex, nil }
func toStorageErr(err error, key string, rv int64) error { if err == nil { return nil } switch { case etcdutil.IsEtcdNotFound(err): return storage.NewKeyNotFoundError(key, rv) case etcdutil.IsEtcdNodeExist(err): return storage.NewKeyExistsError(key, rv) case etcdutil.IsEtcdTestFailed(err): return storage.NewResourceVersionConflictsError(key, rv) case etcdutil.IsEtcdUnreachable(err): return storage.NewUnreachableError(key, rv) default: return err } }
// Implements storage.Interface. func (h *etcdHelper) GuaranteedUpdate(ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool, tryUpdate storage.UpdateFunc) error { if ctx == nil { glog.Errorf("Context is nil") } v, err := conversion.EnforcePtr(ptrToType) if err != nil { // Panic is appropriate, because this is a programming error. panic("need ptr to type") } key = h.prefixEtcdKey(key) for { obj := reflect.New(v.Type()).Interface().(runtime.Object) origBody, node, res, err := h.bodyAndExtractObj(ctx, key, obj, ignoreNotFound) if err != nil { return err } meta := storage.ResponseMeta{} if node != nil { meta.TTL = node.TTL if node.Expiration != nil { meta.Expiration = node.Expiration } meta.ResourceVersion = node.ModifiedIndex } // Get the object to be written by calling tryUpdate. ret, newTTL, err := tryUpdate(obj, meta) if err != nil { return err } index := uint64(0) ttl := uint64(0) if node != nil { index = node.ModifiedIndex if node.TTL != 0 { ttl = uint64(node.TTL) } if node.Expiration != nil && ttl == 0 { ttl = 1 } } else if res != nil { index = res.Index } if newTTL != nil { if ttl != 0 && *newTTL == 0 { // TODO: remove this after we have verified this is no longer an issue glog.V(4).Infof("GuaranteedUpdate is clearing TTL for %q, may not be intentional", key) } ttl = *newTTL } // Since update object may have a resourceVersion set, we need to clear it here. if err := h.versioner.UpdateObject(ret, meta.Expiration, 0); err != nil { return errors.New("resourceVersion cannot be set on objects store in etcd") } data, err := runtime.Encode(h.codec, ret) if err != nil { return err } // First time this key has been used, try creating new value. if index == 0 { startTime := time.Now() opts := etcd.SetOptions{ TTL: time.Duration(ttl) * time.Second, PrevExist: etcd.PrevNoExist, } response, err := h.etcdKeysAPI.Set(ctx, key, string(data), &opts) metrics.RecordEtcdRequestLatency("create", getTypeName(ptrToType), startTime) if etcdutil.IsEtcdNodeExist(err) { continue } _, _, err = h.extractObj(response, err, ptrToType, false, false) return err } if string(data) == origBody { // If we don't send an update, we simply return the currently existing // version of the object. _, _, err := h.extractObj(res, nil, ptrToType, ignoreNotFound, false) return err } startTime := time.Now() // Swap origBody with data, if origBody is the latest etcd data. opts := etcd.SetOptions{ PrevValue: origBody, PrevIndex: index, TTL: time.Duration(ttl) * time.Second, } response, err := h.etcdKeysAPI.Set(ctx, key, string(data), &opts) metrics.RecordEtcdRequestLatency("compareAndSwap", getTypeName(ptrToType), startTime) if etcdutil.IsEtcdTestFailed(err) { // Try again. continue } _, _, err = h.extractObj(response, err, ptrToType, false, false) return err } }
// IsNodeExist returns true if and only if err is an node already exist error. func IsNodeExist(err error) bool { // TODO: add alternate storage error here return etcdutil.IsEtcdNodeExist(err) }
// Implements storage.Interface. func (h *etcdHelper) GuaranteedUpdate(ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool, tryUpdate storage.UpdateFunc) error { if ctx == nil { glog.Errorf("Context is nil") } v, err := conversion.EnforcePtr(ptrToType) if err != nil { // Panic is appropriate, because this is a programming error. panic("need ptr to type") } key = h.prefixEtcdKey(key) for { obj := reflect.New(v.Type()).Interface().(runtime.Object) origBody, node, res, err := h.bodyAndExtractObj(ctx, key, obj, ignoreNotFound) if err != nil { return err } meta := storage.ResponseMeta{} if node != nil { meta.TTL = node.TTL if node.Expiration != nil { meta.Expiration = node.Expiration } meta.ResourceVersion = node.ModifiedIndex } // Get the object to be written by calling tryUpdate. ret, newTTL, err := tryUpdate(obj, meta) if err != nil { return err } index := uint64(0) ttl := uint64(0) if node != nil { index = node.ModifiedIndex if node.TTL != 0 { ttl = uint64(node.TTL) } if node.Expiration != nil && ttl == 0 { ttl = 1 } } else if res != nil { index = res.EtcdIndex } if newTTL != nil { if ttl != 0 && *newTTL == 0 { // TODO: remove this after we have verified this is no longer an issue glog.V(4).Infof("GuaranteedUpdate is clearing TTL for %q, may not be intentional", key) } ttl = *newTTL } data, err := h.codec.Encode(ret) if err != nil { return err } // First time this key has been used, try creating new value. if index == 0 { startTime := time.Now() response, err := h.client.Create(key, string(data), ttl) metrics.RecordEtcdRequestLatency("create", getTypeName(ptrToType), startTime) if etcdutil.IsEtcdNodeExist(err) { continue } _, _, err = h.extractObj(response, err, ptrToType, false, false) return err } if string(data) == origBody { return nil } startTime := time.Now() // Swap origBody with data, if origBody is the latest etcd data. response, err := h.client.CompareAndSwap(key, string(data), ttl, origBody, index) metrics.RecordEtcdRequestLatency("compareAndSwap", getTypeName(ptrToType), startTime) if etcdutil.IsEtcdTestFailed(err) { // Try again. continue } _, _, err = h.extractObj(response, err, ptrToType, false, false) return err } }