func convergeRunOnces(bbs Bbs.ExecutorBBS) {
statusChannel, releaseLock, err := bbs.MaintainConvergeLock(*convergenceInterval, *executorID)
if err != nil {
logger.Fatal("executor.converge-lock.acquire-failed", err)
}
tasks.Add(1)
for {
select {
case locked, ok := <-statusChannel:
if !ok {
tasks.Done()
return
}
if locked {
t := time.Now()
logger.Info("converging")
bbs.ConvergeRunOnce(*timeToClaimRunOnce)
logger.Info("converged", time.Since(t))
} else {
logger.Error("lost.convergence.lock")
}
case <-stop:
releaseLock <- nil
}
}
}
func handleRunOnces(bbs Bbs.ExecutorBBS) {
tasks.Add(1)
for {
logger.Info("watch.desired")
runOnces, stopWatching, errors := bbs.WatchForDesiredRunOnce()
INNER:
for {
select {
case runOnce, ok := <-runOnces:
if !ok {
logger.Info("watch.desired.closed")
break INNER
}
tasks.Add(1)
go func() {
handleRunOnce(bbs, runOnce)
tasks.Done()
}()
case err, ok := <-errors:
if ok && err != nil {
logger.Error("watch.desired.error", err)
}
break INNER
case <-stop:
stopWatching <- true
tasks.Done()
}
}
}
}
func maintainPresence(bbs Bbs.ExecutorBBS, ready chan<- bool) error {
p, statusChannel, err := bbs.MaintainExecutorPresence(*heartbeatInterval, *executorID)
if err != nil {
ready <- false
return err
}
tasks.Add(1)
go func() {
for {
select {
case locked, ok := <-statusChannel:
if locked && ready != nil {
ready <- true
ready = nil
}
if !locked && ok {
tasks.Done()
logger.Fatal("maintain.presence.fatal", err)
}
if !ok {
tasks.Done()
return
}
case <-stop:
p.Remove()
tasks.Done()
}
}
}()
return nil
}
func handleRunOnce(bbs Bbs.ExecutorBBS, runOnce *models.RunOnce) {
//hesitate
logger.Info("handling.runonce", runOnce.Guid)
sleepForARandomInterval("sleep.claim", 0, 100)
//reserve memory
ok := reserveMemory(runOnce.MemoryMB)
if !ok {
logger.Info("reserve.memory.failed", runOnce.Guid)
return
}
defer releaseMemory(runOnce.MemoryMB)
//mark claimed
logger.Info("claiming.runonce", runOnce.Guid)
err := bbs.ClaimRunOnce(runOnce, *executorID)
if err != nil {
logger.Info("claim.runonce.failed", runOnce.Guid, err)
return
}
logger.Info("claimed.runonce", runOnce.Guid)
//create container
sleepForContainerCreationInterval()
//mark started
logger.Info("starting.runonce", runOnce.Guid)
err = bbs.StartRunOnce(runOnce, "container")
if err != nil {
logger.Error("start.runonce.failed", runOnce.Guid, err)
return
}
logger.Info("started.runonce", runOnce.Guid)
//run
sleepForRunInterval()
//mark completed
logger.Info("completing.runonce", runOnce.Guid)
err = bbs.CompleteRunOnce(runOnce, false, "", "")
if err != nil {
logger.Error("complete.runonce.failed", runOnce.Guid, err)
return
}
logger.Info("completed.runonce", runOnce.Guid)
}