func (this *HiveExec) Run(job *dag.Job) error {
if !job.ValidateAttr([]string{"output"}) ||
(!job.ValidateAttr([]string{"script"}) && !job.ValidateAttr([]string{"hql"})) {
msg := "invalid job"
log.Error(msg)
return fmt.Errorf(msg)
}
// !!!VERY IMPORTANT!!!
// Many other operations relay on this TrimRight.
job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")
args, err := this.genCmdArgs(job)
if err != nil {
log.Error(args)
return err
}
log.Debugf("CMD: hive %v", args)
retcode, err := util.ExecCmd(job.Name, "hive", args...)
if err != nil {
job.Status = dag.Failed
return err
}
if retcode != 0 {
job.Status = dag.Failed
return fmt.Errorf("script failed: %d", retcode)
}
job.Status = dag.Finished
return nil
}
func (this *ScriptExec) Run(job *dag.Job) error {
if !job.ValidateAttr([]string{"output", "script", "interpreter"}) {
msg := "invalid job: missing output, script or interpreter"
log.Error(msg)
return fmt.Errorf(msg)
}
// !!!VERY IMPORTANT!!!
// Many other operations relay on this TrimRight.
job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")
args := this.genCmdArgs(job)
intepreter := getProp("interpreter", job.Attrs)
log.Debugf("CMD: %s %v", intepreter, args)
retcode, err := util.ExecCmd(job.Name, intepreter, args...)
if err != nil {
job.Status = dag.Failed
return err
}
if retcode != 0 {
job.Status = dag.Failed
return fmt.Errorf("script failed: %d", retcode)
}
job.Status = dag.Finished
return nil
}
func TestAll(t *testing.T) {
l := log.New(os.Stdout, "test_logger ", log.LOG_LEVEL_ALL)
l.Debug("debug")
l.Debugf("%s", "debug")
l.Trace("trace")
l.Tracef("%s", "trace")
l.Info("info")
l.Infof("%s", "info")
l.Warn("warn")
l.Warnf("%s", "warn")
l.Error("error")
l.Errorf("%s", "error")
l.Fatal("fatal")
l.Fatalf("%s", "fatal")
log.Debug("debug")
log.Debugf("%s", "debug")
log.Trace("trace")
log.Tracef("%s", "trace")
log.Info("info")
log.Infof("%s", "info")
log.Warn("warn")
log.Warnf("%s", "warn")
log.Error("error")
log.Errorf("%s", "error")
log.Fatal("fatal")
log.Fatalf("%s", "fatal")
}
func (this *HadoopExec) Run(job *dag.Job) error {
if !job.ValidateAttr([]string{"mapper", "input", "output"}) {
msg := "invalid job"
log.Error(msg)
return fmt.Errorf(msg)
}
// !!!VERY IMPORTANT!!!
// Many other operations relay on this TrimRight.
job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")
this.hdfs.Rm(job.Attrs["output"])
this.createOutput(job)
args := this.genCmdArgs(job)
log.Debugf("CMD: hadoop %v", args)
retcode, err := util.ExecCmd(job.Name, "hadoop", args...)
if err != nil {
job.Status = dag.Failed
return err
}
if retcode != 0 {
job.Status = dag.Failed
return fmt.Errorf("script failed: %d", retcode)
}
job.Status = dag.Finished
return nil
}
func (this *DotSerializer) Deserialize(data []byte) (*DAG, error) {
ast, err := dotparser.ParseBytes(data)
if err != nil {
return nil, err
}
graph := dot.NewGraph()
dot.Analyse(ast, graph)
p := NewDAG(graph.Name)
for src, dests := range graph.Edges.SrcToDsts {
for dest, _ := range dests {
if orig, ok := p.Jobs[src]; !ok {
n := dotNameToDAGJob(graph, src)
n.Post = append(n.Post, dest)
p.Jobs[src] = n
} else {
orig.Post = append(orig.Post, dest)
}
if orig, ok := p.Jobs[dest]; !ok {
n := dotNameToDAGJob(graph, dest)
n.Prev = append(n.Prev, src)
p.Jobs[dest] = n
} else {
orig.Prev = append(orig.Prev, src)
}
if _, ok := p.InDegrees[src]; !ok {
p.InDegrees[src] = 0
}
if orig, ok := p.InDegrees[dest]; !ok {
p.InDegrees[dest] = 1
} else {
p.InDegrees[dest] = orig + 1
}
}
}
for _, edge := range graph.Edges.Edges {
if v, ok := edge.Attrs["nonstrict"]; ok {
nonstrict, err := util.StringToBool(strings.Trim(v, "\""))
if err != nil {
log.Error(err)
return nil, err
}
if len(edge.Src) != 0 && len(edge.Dst) != 0 {
r := NewRelation()
r.NonStrict = nonstrict
if _, ok := p.Relations[edge.Src]; !ok {
p.Relations[edge.Src] = make(map[string]*Relation)
}
p.Relations[edge.Src][edge.Dst] = r
}
}
}
return p, nil
}
func (this *Sched) runQueue(queue []*dag.Job, d *dag.DAG) error {
var wg sync.WaitGroup
for _, job := range queue {
wg.Add(1)
go func(job *dag.Job, d *dag.DAG) {
// !!! All shared objects need to be thread-safe !!!
defer wg.Done()
log.Infof("run job: %s", job.Name)
if err := d.ResolveJob(job); err != nil {
log.Error(err)
job.Status = dag.Failed
this.updateJobStatus(job, d)
return
}
if job.Type == dag.DummyJob {
job.Status = dag.Finished
this.updateJobStatus(job, d)
} else {
jexec, err := this.getExec(job)
if err != nil {
panic(err)
}
job.Status = dag.Started
this.updateJobStatus(job, d)
if err = jexec.Run(job); err != nil {
log.Error(err)
job.Status = dag.Failed
}
this.updateJobStatus(job, d)
}
}(job, d)
}
wg.Wait()
return nil
}