// Rescind should be called to indicate you no longer wish to be the leader
func (rl *regionLeader) Rescind() {
rl.cleanup.Do(func() {
log.Debugf("[Sync:RegionLeader] Cleaning up leadership of '%v'...", rl.lockNode)
close(rl.rescinded)
// keep trying to delete the ZK node (to release leadership) until we're sure it doesn't exist
for {
err := zookeeper.Delete(rl.lockNode, -1)
if err == nil || err == gozk.ErrNoNode {
log.Debugf("[Sync:RegionLeader] Have deleted leadership node '%v'", rl.lockNode)
inst.Counter(1.0, "sync.regionleader.rescinded")
break
}
log.Warnf("[Sync:RegionLeader] Failed to cleanup/rescind leadership (will retry): %v", err)
time.Sleep(cleanupDelay)
}
// Unregister region leader
mu.Lock()
for i := 0; i < len(rls); i++ {
if rls[i] == rl {
rls = append(rls[:i], rls[i+1:]...)
break
}
}
mu.Unlock()
})
}
// Reserve will reserve an item with the id in the DefaultReservation for the given amount of
// time.
func (dr *DefaultReservation) Reserve(d time.Duration) error {
log.Debugf("[Sync:Reservation] Attempting to reserve '%s' for %s...", dr.id, d)
lockpath := constructPath(dr.path, dr.id)
// Check if the reservation already exists
b, _, err := zookeeper.Get(lockpath)
if err == nil {
// It exists, check if expired
var expired time.Time
expired.GobDecode(b)
log.Debugf("[Sync:Reservation] Read existing node for '%s', expires at %s", dr.id, expired)
if expired.Before(time.Now()) {
log.Debugf("[Sync:Reservation] Deleting expired lock '%s'", dr.id)
// It has expired, delete the node and get the reservation as usual
zookeeper.Delete(lockpath, -1)
} else {
return ErrReserved
}
}
expires := time.Now().Add(d)
expiresBytes, err := expires.GobEncode()
if err != nil {
return err
}
for {
_, err = zookeeper.Create(lockpath, expiresBytes, 0, dr.acl)
if err == gozk.ErrNoNode {
createParents(dr.path)
} else if err == nil {
break
} else {
log.Warnf("[Reservation] ZK error creating ephemeral lock node for reservation: %v", err)
return err
}
}
log.Debugf("[Sync:Reservation] Created lock node for '%s', expires at %s", dr.id, expires)
return nil
}
// RegionLeader block indefinitely until this invocation has been elected the "leader" within the local operating region.
// It will then return a channel that will eventually be closed when leadership is rescinded.
func RegionLeader(id string) Leader {
path := fmt.Sprintf(regionLeaderPath, id)
prefix := path + "/lock-"
var lockNode string
for {
// create our lock node -- retry until this is done, use exponential backoff
// to add some delay between attempts
b := backoff.NewExponentialBackOff()
b.InitialInterval = backoffInitialInterval
b.MaxInterval = backoffMaxInterval
b.MaxElapsedTime = 0 // Never stop retrying
backoff.RetryNotify(func() (err error) {
log.Infof("[Sync:RegionLeader] Attepting to create ephemeral lock node for leadership election")
lockNode, err = zookeeper.CreateProtectedEphemeralSequential(prefix, []byte{}, gozk.WorldACL(gozk.PermAll))
return
}, b, func(err error, d time.Duration) {
if err == gozk.ErrNoNode {
createParents(path)
} else if err != nil {
log.Warnf("[Sync:RegionLeader] ZooKeeper error creating ephemeral lock node for leadership election: %s. Waiting %s", err, d)
}
})
err := waitForWinner(path, lockNode)
if err != nil {
// try to cleanup - then go again
zookeeper.Delete(lockNode, -1)
time.Sleep(time.Second)
continue
}
// we are the leader
break
}
log.Infof("[Sync:RegionLeader] Elected leader of '%v'", id)
inst.Counter(1.0, "sync.regionleader.elected")
return newRegionLeader(lockNode)
}
func (r *reaper) reap() {
r.pathsMtx.RLock()
// snapshot paths
keys := make([]string, len(r.paths))
i := 0
for k := range r.paths {
keys[i] = k
i++
}
r.pathsMtx.RUnlock()
for _, path := range keys {
exists, stat, err := zk.Exists(path)
if !exists {
if err != nil && err != gozk.ErrNoNode {
log.Errorf("[Sync:RegionLock] Error checking path %s %v", path, err)
r.resetPath(path)
// something is still using this
continue
}
// no node
} else {
if stat.NumChildren > 0 {
r.resetPath(path)
continue
}
// node but no children
}
// increment reap number
n := r.incrementPath(path)
if n >= reaperThreshold {
// reaped enough times and it's come out as empty. Delete it
if err := zk.Delete(path, -1); err == nil || err == gozk.ErrNoNode {
// success
r.removePath(path)
} else {
log.Warnf("[Sync:RegionLock] Error reaping path %s. %v", path, err) // debug not error on purpose
// some error, most likely node being used by something else, reset and let it live
r.resetPath(path)
}
}
}
}
// findLowestSequenceNode within a particular lock path
func findLowestSequenceNode(path string, seq int) (lowestSeq int, prevSeqPath string, err error) {
// Read all the children of the node
// This is why we create sequential nodes under a parent node based on the lock ID
// If not, say we stored all ephemeral locks under a higher level parent,
// we would be checking nodes of every lock currently in play, rather than locks
// on this ID
children, _, err := zookeeper.Children(path)
if err != nil {
return -1, "", err
}
var ttl time.Time
lowestSeq = seq
prevSeq := 0
prevSeqPath = ""
for _, p := range children {
// Check if this lock has timed out
data, _, _ := zookeeper.Get(path + "/" + p)
if len(data) > 0 {
ttl.GobDecode(data)
if ttl.Before(time.Now()) {
log.Tracef("[RegionLock] Deleting expired lock '%s'", path+"/"+p)
zookeeper.Delete(path+"/"+p, -1)
continue
}
}
s, err := parseSeq(p)
if err != nil {
return -1, "", err
}
if s < lowestSeq {
lowestSeq = s
}
if s < seq && s > prevSeq {
prevSeq = s
prevSeqPath = p
}
}
return lowestSeq, prevSeqPath, err
}
// Release will release the reservation in the DefaultReservation
func (dr *DefaultReservation) Release() error {
return zookeeper.Delete(constructPath(dr.path, dr.id), -1)
}
// AnonymousRelease will release the reservation of the item with the given id and path
func AnonymousRelease(path, id string) error {
return zookeeper.Delete(constructPath(path, id), -1)
}