func convergeRunOnces(bbs Bbs.ExecutorBBS) { statusChannel, releaseLock, err := bbs.MaintainConvergeLock(*convergenceInterval, *executorID) if err != nil { logger.Fatal("executor.converge-lock.acquire-failed", err) } tasks.Add(1) for { select { case locked, ok := <-statusChannel: if !ok { tasks.Done() return } if locked { t := time.Now() logger.Info("converging") bbs.ConvergeRunOnce(*timeToClaimRunOnce) logger.Info("converged", time.Since(t)) } else { logger.Error("lost.convergence.lock") } case <-stop: releaseLock <- nil } } }
func handleRunOnces(bbs Bbs.ExecutorBBS) { tasks.Add(1) for { logger.Info("watch.desired") runOnces, stopWatching, errors := bbs.WatchForDesiredRunOnce() INNER: for { select { case runOnce, ok := <-runOnces: if !ok { logger.Info("watch.desired.closed") break INNER } tasks.Add(1) go func() { handleRunOnce(bbs, runOnce) tasks.Done() }() case err, ok := <-errors: if ok && err != nil { logger.Error("watch.desired.error", err) } break INNER case <-stop: stopWatching <- true tasks.Done() } } } }
func maintainPresence(bbs Bbs.ExecutorBBS, ready chan<- bool) error { p, statusChannel, err := bbs.MaintainExecutorPresence(*heartbeatInterval, *executorID) if err != nil { ready <- false return err } tasks.Add(1) go func() { for { select { case locked, ok := <-statusChannel: if locked && ready != nil { ready <- true ready = nil } if !locked && ok { tasks.Done() logger.Fatal("maintain.presence.fatal", err) } if !ok { tasks.Done() return } case <-stop: p.Remove() tasks.Done() } } }() return nil }
func handleRunOnce(bbs Bbs.ExecutorBBS, runOnce *models.RunOnce) { //hesitate logger.Info("handling.runonce", runOnce.Guid) sleepForARandomInterval("sleep.claim", 0, 100) //reserve memory ok := reserveMemory(runOnce.MemoryMB) if !ok { logger.Info("reserve.memory.failed", runOnce.Guid) return } defer releaseMemory(runOnce.MemoryMB) //mark claimed logger.Info("claiming.runonce", runOnce.Guid) err := bbs.ClaimRunOnce(runOnce, *executorID) if err != nil { logger.Info("claim.runonce.failed", runOnce.Guid, err) return } logger.Info("claimed.runonce", runOnce.Guid) //create container sleepForContainerCreationInterval() //mark started logger.Info("starting.runonce", runOnce.Guid) err = bbs.StartRunOnce(runOnce, "container") if err != nil { logger.Error("start.runonce.failed", runOnce.Guid, err) return } logger.Info("started.runonce", runOnce.Guid) //run sleepForRunInterval() //mark completed logger.Info("completing.runonce", runOnce.Guid) err = bbs.CompleteRunOnce(runOnce, false, "", "") if err != nil { logger.Error("complete.runonce.failed", runOnce.Guid, err) return } logger.Info("completed.runonce", runOnce.Guid) }