func isLeader(l registry.Lease, machID string) bool {
if l == nil {
return false
}
if l.MachineID() != machID {
return false
}
return true
}
func renewLeadership(l registry.Lease, ttl time.Duration) registry.Lease {
err := l.Renew(ttl)
if err != nil {
log.Errorf("Engine leadership lost, renewal failed: %v", err)
return nil
}
log.Debugf("Engine leadership renewed")
return l
}
// ensureLeader will attempt to renew a non-nil Lease, falling back to
// acquiring a new Lease on the lead engine role.
func ensureLeader(prev registry.Lease, reg registry.Registry, machID string) (cur registry.Lease) {
if prev != nil {
err := prev.Renew(engineRoleLeasePeriod)
if err == nil {
cur = prev
return
} else {
log.Errorf("Engine leadership could not be renewed: %v", err)
}
}
var err error
cur, err = reg.LeaseRole(engineRoleName, machID, engineRoleLeasePeriod)
if err != nil {
log.Errorf("Failed acquiring engine leadership: %v", err)
} else if cur == nil {
log.V(1).Infof("Unable to acquire engine leadership")
} else {
log.Infof("Acquired engine leadership")
}
return
}
// ensureLeader will attempt to renew a non-nil Lease, falling back to
// acquiring a new Lease on the lead engine role.
func ensureLeader(prev registry.Lease, reg registry.Registry, machID string, ttl time.Duration) (cur registry.Lease) {
if prev != nil {
err := prev.Renew(ttl)
if err == nil {
log.V(1).Infof("Engine leadership renewed")
cur = prev
return
} else {
log.Errorf("Engine leadership lost, renewal failed: %v", err)
}
}
var err error
cur, err = reg.LeaseRole(engineRoleName, machID, ttl)
if err != nil {
log.Errorf("Engine leadership acquisition failed: %v", err)
} else if cur == nil {
log.V(1).Infof("Unable to acquire engine leadership")
} else {
log.Infof("Engine leadership acquired")
}
return
}
func (e *Engine) Run(ival time.Duration, stop chan bool) {
leaseTTL := ival * 5
machID := e.machine.State().ID
reconcile := func() {
if !ensureEngineVersionMatch(e.cRegistry, engineVersion) {
return
}
var l registry.Lease
if isLeader(e.lease, machID) {
l = renewLeadership(e.lease, leaseTTL)
} else {
l = acquireLeadership(e.lRegistry, machID, engineVersion, leaseTTL)
}
// log all leadership changes
if l != nil && e.lease == nil && l.MachineID() != machID {
log.Infof("Engine leader is %s", l.MachineID())
} else if l != nil && e.lease != nil && l.MachineID() != e.lease.MachineID() {
log.Infof("Engine leadership changed from %s to %s", e.lease.MachineID(), l.MachineID())
}
e.lease = l
if !isLeader(e.lease, machID) {
return
}
// abort is closed when reconciliation must stop prematurely, either
// by a local timeout or the fleet server shutting down
abort := make(chan struct{})
// monitor is used to shut down the following goroutine
monitor := make(chan struct{})
go func() {
select {
case <-monitor:
return
case <-time.After(leaseTTL):
close(abort)
case <-stop:
close(abort)
}
}()
start := time.Now()
e.rec.Reconcile(e, abort)
close(monitor)
elapsed := time.Now().Sub(start)
msg := fmt.Sprintf("Engine completed reconciliation in %s", elapsed)
if elapsed > ival {
log.Warning(msg)
} else {
log.Debug(msg)
}
}
rec := pkg.NewPeriodicReconciler(ival, reconcile, e.rStream)
rec.Run(stop)
}