func (b *Broker) executeJob(job bs.Job, shellCmd string) (result *JobResult, err error) { result = &JobResult{JobId: job.Id, Executed: true} ttr, err := job.TimeLeft() timer := time.NewTimer(ttr + ttrMargin) if err != nil { return } cmd, out, err := cmd.NewCommand(shellCmd) if err != nil { return } if err = cmd.StartWithStdin(job.Body); err != nil { return } // TODO: end loop when stdout closes stdoutReader: for { select { case <-timer.C: if err = cmd.Terminate(); err != nil { return } result.TimedOut = true case data, ok := <-out: if !ok { break stdoutReader } b.log.Printf("stdout: %s", data) result.Stdout = append(result.Stdout, data...) } } waitC := cmd.WaitChan() waitLoop: for { select { case wr := <-waitC: timer.Stop() if wr.Err == nil { err = wr.Err } result.ExitStatus = wr.Status break waitLoop case <-timer.C: cmd.Terminate() result.TimedOut = true } } return }
func (b *Broker) handleResult(job bs.Job, result *JobResult) (err error) { if result.TimedOut { b.log.Printf("job %d timed out", job.Id) return } b.log.Printf("job %d finished with exit(%d)", job.Id, result.ExitStatus) switch result.ExitStatus { case 0: b.log.Printf("deleting job %d", job.Id) err = job.Delete() default: r, err := job.Releases() if err != nil { r = ReleaseTries } // r*r*r*r means final of 10 tries has 1h49m21s delay, 4h15m33s total. // See: http://play.golang.org/p/I15lUWoabI delay := time.Duration(r*r*r*r) * time.Second b.log.Printf("releasing job %d with %v delay (%d retries)", job.Id, delay, r) err = job.Release(delay) } return }