func rpcRenewLeadership(lManager lease.Manager, l lease.Lease, ver int, ttl time.Duration) lease.Lease {
err := l.Renew(ttl)
if err != nil {
if eerr, ok := err.(*etcdErr.Error); ok && eerr.ErrorCode == etcdErr.EcodeKeyNotFound {
log.Errorf("Retry renew etcd operation that failed due to %v", err)
l, err = lManager.AcquireLease(engineLeaseName, l.MachineID(), ver, ttl)
if err != nil {
log.Errorf("Engine leadership re-acquisition failed: %v", err)
return nil
} else if l == nil {
log.Infof("Unable to re-acquire engine leadership")
return nil
}
log.Infof("Engine leadership re-acquired")
return l
} else {
log.Errorf("Engine leadership lost, renewal failed: %v", err)
return nil
}
}
log.Debugf("Engine leadership renewed")
return l
}
func rpcAcquireLeadership(reg registry.Registry, lManager lease.Manager, machID string, ver int, ttl time.Duration) lease.Lease {
existing, err := lManager.GetLease(engineLeaseName)
if err != nil {
log.Errorf("Unable to determine current lease: %v", err)
return nil
}
var l lease.Lease
if (existing == nil && reg.UseEtcdRegistry()) || (existing == nil && !reg.IsRegistryReady()) {
l, err = lManager.AcquireLease(engineLeaseName, machID, ver, ttl)
if err != nil {
log.Errorf("Engine leadership acquisition failed: %v", err)
return nil
} else if l == nil {
log.Infof("Unable to acquire engine leadership")
return nil
}
log.Infof("Engine leadership acquired")
return l
}
if existing != nil && existing.Version() >= ver {
log.Debugf("Lease already held by Machine(%s) operating at acceptable version %d", existing.MachineID(), existing.Version())
return existing
}
// TODO(hector): Here we could add a possible SLA to determine when the leader
// is too busy. In such a case, we can trigger a new leader election
if (existing != nil && reg.UseEtcdRegistry()) || (existing != nil && !reg.IsRegistryReady()) {
rem := existing.TimeRemaining()
l, err = lManager.StealLease(engineLeaseName, machID, ver, ttl+rem, existing.Index())
if err != nil {
log.Errorf("Engine leadership steal failed: %v", err)
return nil
} else if l == nil {
log.Infof("Unable to steal engine leadership")
return nil
}
log.Infof("Stole engine leadership from Machine(%s)", existing.MachineID())
if rem > 0 {
log.Infof("Waiting %v for previous lease to expire before continuing reconciliation", rem)
<-time.After(rem)
}
return l
}
log.Infof("Engine leader is BUSY!")
return existing
}
func acquireLeadership(lManager lease.Manager, machID string, ver int, ttl time.Duration) lease.Lease {
existing, err := lManager.GetLease(engineLeaseName)
if err != nil {
log.Errorf("Unable to determine current lease: %v", err)
return nil
}
var l lease.Lease
if existing == nil {
l, err = lManager.AcquireLease(engineLeaseName, machID, ver, ttl)
if err != nil {
log.Errorf("Engine leadership acquisition failed: %v", err)
return nil
} else if l == nil {
log.Debugf("Unable to acquire engine leadership")
return nil
}
log.Infof("Engine leadership acquired")
metrics.ReportEngineLeader()
return l
}
if existing.Version() >= ver {
log.Debugf("Lease already held by Machine(%s) operating at acceptable version %d", existing.MachineID(), existing.Version())
return existing
}
rem := existing.TimeRemaining()
l, err = lManager.StealLease(engineLeaseName, machID, ver, ttl+rem, existing.Index())
if err != nil {
log.Errorf("Engine leadership steal failed: %v", err)
return nil
} else if l == nil {
log.Debugf("Unable to steal engine leadership")
return nil
}
log.Infof("Stole engine leadership from Machine(%s)", existing.MachineID())
metrics.ReportEngineLeader()
if rem > 0 {
log.Infof("Waiting %v for previous lease to expire before continuing reconciliation", rem)
<-time.After(rem)
}
return l
}