func createParents(path string) error {
parts := strings.Split(path, "/")
pth := ""
for _, p := range parts[1:] {
pth += "/" + p
_, err := zookeeper.Create(pth, []byte{}, 0, gozk.WorldACL(gozk.PermAll))
if err != nil && err != gozk.ErrNodeExists {
return err
}
}
return nil
}
// RegionLeader block indefinitely until this invocation has been elected the "leader" within the local operating region.
// It will then return a channel that will eventually be closed when leadership is rescinded.
func RegionLeader(id string) Leader {
path := fmt.Sprintf(regionLeaderPath, id)
prefix := path + "/lock-"
var lockNode string
for {
// create our lock node -- retry until this is done, use exponential backoff
// to add some delay between attempts
b := backoff.NewExponentialBackOff()
b.InitialInterval = backoffInitialInterval
b.MaxInterval = backoffMaxInterval
b.MaxElapsedTime = 0 // Never stop retrying
backoff.RetryNotify(func() (err error) {
log.Infof("[Sync:RegionLeader] Attepting to create ephemeral lock node for leadership election")
lockNode, err = zookeeper.CreateProtectedEphemeralSequential(prefix, []byte{}, gozk.WorldACL(gozk.PermAll))
return
}, b, func(err error, d time.Duration) {
if err == gozk.ErrNoNode {
createParents(path)
} else if err != nil {
log.Warnf("[Sync:RegionLeader] ZooKeeper error creating ephemeral lock node for leadership election: %s. Waiting %s", err, d)
}
})
err := waitForWinner(path, lockNode)
if err != nil {
// try to cleanup - then go again
zookeeper.Delete(lockNode, -1)
time.Sleep(time.Second)
continue
}
// we are the leader
break
}
log.Infof("[Sync:RegionLeader] Elected leader of '%v'", id)
inst.Counter(1.0, "sync.regionleader.elected")
return newRegionLeader(lockNode)
}
// RegionTimedLock attempts to achieve a regional lock on `id`, waiting for `waitFor` time in
// case of contention (before giving up) and reserving the lock for a maximum `holdFor`
// in the event of failing to Unlock()
func RegionTimedLock(id []byte, waitFor, holdFor time.Duration) (Lock, error) {
// Ensure we are reaping
once.Do(startReaper)
if int64(holdFor) < int64(minRegionHoldFor) {
return nil, ErrRegionHoldFor
}
lockId := string(id) // we use []byte for function signature compatibility with the global lock
// Our locks are namespaced per service
path, err := constructLockPath(regionLockNamespace, lockId)
if err != nil {
return nil, err
}
log.Tracef("[Sync:RegionTimedLock] Attempting to acquire '%s'; held for %v", path, holdFor)
// Create new lock which we will be lock()ed
lock := ®ionLock{
zkLock: zk.NewLock(path, gozk.WorldACL(gozk.PermAll)),
}
lock.zkLock.SetTTL(holdFor)
lock.zkLock.SetTimeout(waitFor)
// Acquire a lock
startTime := time.Now()
err = lock.zkLock.Lock()
inst.Timing(1.0, "sync.regionlock.acquire", time.Since(startTime))
defaultReaper.addPath(path) // only add path to reaper AFTER we've acquired the lock (or not)
if err == nil {
log.Tracef("[Sync:RegionTimedLock] Successfully acquired '%s'", path)
inst.Counter(1.0, "sync.regionlock.acquire.success")
} else {
log.Errorf("[Sync:RegionTimedLock] Failed to acquire '%s': %s", path, err.Error())
inst.Counter(1.0, "sync.regionlock.acquire.failure")
}
return lock, err
}
// RegionLeader block indefinitely until this invocation has been elected the "leader" within the local operating region.
// It will then return a channel that will eventually be closed when leadership is rescinded.
func RegionLeader(id string) Leader {
path := fmt.Sprintf(regionLeaderPath, id)
prefix := path + "/lock-"
var lockNode string
for {
// create our lock node -- retry until this is done
for {
var err error
lockNode, err = zookeeper.CreateProtectedEphemeralSequential(prefix, []byte{}, gozk.WorldACL(gozk.PermAll))
if err == gozk.ErrNoNode {
createParents(path)
} else if err == nil {
break
} else {
log.Warnf("[Sync:RegionLeader] ZooKeeper error creating ephemeral lock node for leadership election: %s",
err.Error())
}
}
err := waitForWinner(path, lockNode)
if err != nil {
// try to cleanup - then go again
zookeeper.Delete(lockNode, -1)
time.Sleep(time.Second)
continue
}
// we are the leader
break
}
log.Infof("[Sync:RegionLeader] Elected leader of '%v'", id)
inst.Counter(1.0, "sync.regionleader.elected")
return newRegionLeader(lockNode)
}