func createParents(path string) error { parts := strings.Split(path, "/") pth := "" for _, p := range parts[1:] { pth += "/" + p _, err := zookeeper.Create(pth, []byte{}, 0, gozk.WorldACL(gozk.PermAll)) if err != nil && err != gozk.ErrNodeExists { return err } } return nil }
// RegionLeader block indefinitely until this invocation has been elected the "leader" within the local operating region. // It will then return a channel that will eventually be closed when leadership is rescinded. func RegionLeader(id string) Leader { path := fmt.Sprintf(regionLeaderPath, id) prefix := path + "/lock-" var lockNode string for { // create our lock node -- retry until this is done, use exponential backoff // to add some delay between attempts b := backoff.NewExponentialBackOff() b.InitialInterval = backoffInitialInterval b.MaxInterval = backoffMaxInterval b.MaxElapsedTime = 0 // Never stop retrying backoff.RetryNotify(func() (err error) { log.Infof("[Sync:RegionLeader] Attepting to create ephemeral lock node for leadership election") lockNode, err = zookeeper.CreateProtectedEphemeralSequential(prefix, []byte{}, gozk.WorldACL(gozk.PermAll)) return }, b, func(err error, d time.Duration) { if err == gozk.ErrNoNode { createParents(path) } else if err != nil { log.Warnf("[Sync:RegionLeader] ZooKeeper error creating ephemeral lock node for leadership election: %s. Waiting %s", err, d) } }) err := waitForWinner(path, lockNode) if err != nil { // try to cleanup - then go again zookeeper.Delete(lockNode, -1) time.Sleep(time.Second) continue } // we are the leader break } log.Infof("[Sync:RegionLeader] Elected leader of '%v'", id) inst.Counter(1.0, "sync.regionleader.elected") return newRegionLeader(lockNode) }
// RegionTimedLock attempts to achieve a regional lock on `id`, waiting for `waitFor` time in // case of contention (before giving up) and reserving the lock for a maximum `holdFor` // in the event of failing to Unlock() func RegionTimedLock(id []byte, waitFor, holdFor time.Duration) (Lock, error) { // Ensure we are reaping once.Do(startReaper) if int64(holdFor) < int64(minRegionHoldFor) { return nil, ErrRegionHoldFor } lockId := string(id) // we use []byte for function signature compatibility with the global lock // Our locks are namespaced per service path, err := constructLockPath(regionLockNamespace, lockId) if err != nil { return nil, err } log.Tracef("[Sync:RegionTimedLock] Attempting to acquire '%s'; held for %v", path, holdFor) // Create new lock which we will be lock()ed lock := ®ionLock{ zkLock: zk.NewLock(path, gozk.WorldACL(gozk.PermAll)), } lock.zkLock.SetTTL(holdFor) lock.zkLock.SetTimeout(waitFor) // Acquire a lock startTime := time.Now() err = lock.zkLock.Lock() inst.Timing(1.0, "sync.regionlock.acquire", time.Since(startTime)) defaultReaper.addPath(path) // only add path to reaper AFTER we've acquired the lock (or not) if err == nil { log.Tracef("[Sync:RegionTimedLock] Successfully acquired '%s'", path) inst.Counter(1.0, "sync.regionlock.acquire.success") } else { log.Errorf("[Sync:RegionTimedLock] Failed to acquire '%s': %s", path, err.Error()) inst.Counter(1.0, "sync.regionlock.acquire.failure") } return lock, err }
// RegionLeader block indefinitely until this invocation has been elected the "leader" within the local operating region. // It will then return a channel that will eventually be closed when leadership is rescinded. func RegionLeader(id string) Leader { path := fmt.Sprintf(regionLeaderPath, id) prefix := path + "/lock-" var lockNode string for { // create our lock node -- retry until this is done for { var err error lockNode, err = zookeeper.CreateProtectedEphemeralSequential(prefix, []byte{}, gozk.WorldACL(gozk.PermAll)) if err == gozk.ErrNoNode { createParents(path) } else if err == nil { break } else { log.Warnf("[Sync:RegionLeader] ZooKeeper error creating ephemeral lock node for leadership election: %s", err.Error()) } } err := waitForWinner(path, lockNode) if err != nil { // try to cleanup - then go again zookeeper.Delete(lockNode, -1) time.Sleep(time.Second) continue } // we are the leader break } log.Infof("[Sync:RegionLeader] Elected leader of '%v'", id) inst.Counter(1.0, "sync.regionleader.elected") return newRegionLeader(lockNode) }