func ExecCmd(jobname, name string, arg ...string) (int, error) {
cmd := exec.Command(name, arg...)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
return 0, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
log.Fatal(err)
return 0, err
}
if err := cmd.Start(); err != nil {
//log.Fatal(err)
return 0, err
}
errscanner := bufio.NewScanner(stderr)
for errscanner.Scan() {
if len(errscanner.Text()) != 0 {
log.Infof("<%s> %s", jobname, errscanner.Text())
}
}
if err := errscanner.Err(); err != nil {
log.Fatalf("reading standard input: %v", err)
}
outscanner := bufio.NewScanner(stdout)
for outscanner.Scan() {
if len(outscanner.Text()) != 0 {
log.Infof("<%s> %s", jobname, outscanner.Text())
}
}
if err := outscanner.Err(); err != nil {
log.Fatalf("reading standard input: %v", err)
}
if err := cmd.Wait(); err != nil {
if exiterr, ok := err.(*exec.ExitError); ok {
// The program has exited with an exit code != 0
// This works on both Unix and Windows. Although package
// syscall is generally platform dependent, WaitStatus is
// defined for both Unix and Windows and in both cases has
// an ExitStatus() method with the same signature.
status, ok := exiterr.Sys().(syscall.WaitStatus)
if !ok {
return 0, fmt.Errorf("get exit status fail")
}
return status.ExitStatus(), nil
} else {
log.Fatalf("cmd.Wait: %v", err)
return 0, err
}
}
return 0, err
}
func TestAll(t *testing.T) {
l := log.New(os.Stdout, "test_logger ", log.LOG_LEVEL_ALL)
l.Debug("debug")
l.Debugf("%s", "debug")
l.Trace("trace")
l.Tracef("%s", "trace")
l.Info("info")
l.Infof("%s", "info")
l.Warn("warn")
l.Warnf("%s", "warn")
l.Error("error")
l.Errorf("%s", "error")
l.Fatal("fatal")
l.Fatalf("%s", "fatal")
log.Debug("debug")
log.Debugf("%s", "debug")
log.Trace("trace")
log.Tracef("%s", "trace")
log.Info("info")
log.Infof("%s", "info")
log.Warn("warn")
log.Warnf("%s", "warn")
log.Error("error")
log.Errorf("%s", "error")
log.Fatal("fatal")
log.Fatalf("%s", "fatal")
}
func (this *Sched) genRunQueue(d *dag.DAG) []*dag.Job {
queue := []*dag.Job{}
for name, in := range d.InDegrees {
job, ok := d.Jobs[name]
if !ok {
panic(fmt.Errorf("panic: no corresponding job"))
}
if s, err := this.tracker.GetStatus(job); err != nil {
panic(err)
} else {
job.Status = s
}
log.Debugf("check job status: %s -> %s", job.Name, job.Status)
switch job.Status {
case dag.Finished:
if config.ReRun {
if this.tracker.HasReRan(job) {
continue
}
log.Infof("job is already finished, rerun: %s", job.Name)
this.tracker.SetReRan(job)
} else {
log.Infof("job is already finished, skip: %s", job.Name)
continue
}
case dag.Started:
if config.Force {
log.Warnf("job is already started, run still: %s", job.Name)
} else {
log.Warnf("job is already started, skip: %s", job.Name)
continue
}
}
fails := this.tracker.Fails[job.Name]
if in == 0 && fails < config.MaxRetry {
queue = append(queue, job)
} else if fails >= config.MaxRetry {
log.Errorf("job %s failed %d times, reaches max retry times: %d",
job.Name, this.tracker.Fails[job.Name], config.MaxRetry)
}
}
return queue
}
func (this *Sched) runQueue(queue []*dag.Job, d *dag.DAG) error {
var wg sync.WaitGroup
for _, job := range queue {
wg.Add(1)
go func(job *dag.Job, d *dag.DAG) {
// !!! All shared objects need to be thread-safe !!!
defer wg.Done()
log.Infof("run job: %s", job.Name)
if err := d.ResolveJob(job); err != nil {
log.Error(err)
job.Status = dag.Failed
this.updateJobStatus(job, d)
return
}
if job.Type == dag.DummyJob {
job.Status = dag.Finished
this.updateJobStatus(job, d)
} else {
jexec, err := this.getExec(job)
if err != nil {
panic(err)
}
job.Status = dag.Started
this.updateJobStatus(job, d)
if err = jexec.Run(job); err != nil {
log.Error(err)
job.Status = dag.Failed
}
this.updateJobStatus(job, d)
}
}(job, d)
}
wg.Wait()
return nil
}