func (b *Broker) executeJob(job bs.Job, shellCmd string) (result *JobResult, err error) {
result = &JobResult{JobId: job.Id, Executed: true}
ttr, err := job.TimeLeft()
timer := time.NewTimer(ttr + ttrMargin)
if err != nil {
return
}
cmd, out, err := cmd.NewCommand(shellCmd)
if err != nil {
return
}
if err = cmd.StartWithStdin(job.Body); err != nil {
return
}
// TODO: end loop when stdout closes
stdoutReader:
for {
select {
case <-timer.C:
if err = cmd.Terminate(); err != nil {
return
}
result.TimedOut = true
case data, ok := <-out:
if !ok {
break stdoutReader
}
b.log.Printf("stdout: %s", data)
result.Stdout = append(result.Stdout, data...)
}
}
waitC := cmd.WaitChan()
waitLoop:
for {
select {
case wr := <-waitC:
timer.Stop()
if wr.Err == nil {
err = wr.Err
}
result.ExitStatus = wr.Status
break waitLoop
case <-timer.C:
cmd.Terminate()
result.TimedOut = true
}
}
return
}
func (b *Broker) handleResult(job bs.Job, result *JobResult) (err error) {
if result.TimedOut {
b.log.Printf("job %d timed out", job.Id)
return
}
b.log.Printf("job %d finished with exit(%d)", job.Id, result.ExitStatus)
switch result.ExitStatus {
case 0:
b.log.Printf("deleting job %d", job.Id)
err = job.Delete()
default:
r, err := job.Releases()
if err != nil {
r = ReleaseTries
}
// r*r*r*r means final of 10 tries has 1h49m21s delay, 4h15m33s total.
// See: http://play.golang.org/p/I15lUWoabI
delay := time.Duration(r*r*r*r) * time.Second
b.log.Printf("releasing job %d with %v delay (%d retries)", job.Id, delay, r)
err = job.Release(delay)
}
return
}