Exemplo n.º 1
0
func (this *HiveExec) Run(job *dag.Job) error {
	if !job.ValidateAttr([]string{"output"}) ||
		(!job.ValidateAttr([]string{"script"}) && !job.ValidateAttr([]string{"hql"})) {
		msg := "invalid job"
		log.Error(msg)
		return fmt.Errorf(msg)
	}

	// !!!VERY IMPORTANT!!!
	// Many other operations relay on this TrimRight.
	job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")

	args, err := this.genCmdArgs(job)
	if err != nil {
		log.Error(args)
		return err
	}
	log.Debugf("CMD: hive %v", args)
	retcode, err := util.ExecCmd(job.Name, "hive", args...)
	if err != nil {
		job.Status = dag.Failed
		return err
	}
	if retcode != 0 {
		job.Status = dag.Failed
		return fmt.Errorf("script failed: %d", retcode)
	}
	job.Status = dag.Finished
	return nil
}
Exemplo n.º 2
0
func (this *ScriptExec) Run(job *dag.Job) error {
	if !job.ValidateAttr([]string{"output", "script", "interpreter"}) {
		msg := "invalid job: missing output, script or interpreter"
		log.Error(msg)
		return fmt.Errorf(msg)
	}

	// !!!VERY IMPORTANT!!!
	// Many other operations relay on this TrimRight.
	job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")

	args := this.genCmdArgs(job)
	intepreter := getProp("interpreter", job.Attrs)
	log.Debugf("CMD: %s %v", intepreter, args)
	retcode, err := util.ExecCmd(job.Name, intepreter, args...)
	if err != nil {
		job.Status = dag.Failed
		return err
	}
	if retcode != 0 {
		job.Status = dag.Failed
		return fmt.Errorf("script failed: %d", retcode)
	}
	job.Status = dag.Finished
	return nil
}
Exemplo n.º 3
0
func TestAll(t *testing.T) {
	l := log.New(os.Stdout, "test_logger ", log.LOG_LEVEL_ALL)
	l.Debug("debug")
	l.Debugf("%s", "debug")
	l.Trace("trace")
	l.Tracef("%s", "trace")
	l.Info("info")
	l.Infof("%s", "info")
	l.Warn("warn")
	l.Warnf("%s", "warn")
	l.Error("error")
	l.Errorf("%s", "error")
	l.Fatal("fatal")
	l.Fatalf("%s", "fatal")
	log.Debug("debug")
	log.Debugf("%s", "debug")
	log.Trace("trace")
	log.Tracef("%s", "trace")
	log.Info("info")
	log.Infof("%s", "info")
	log.Warn("warn")
	log.Warnf("%s", "warn")
	log.Error("error")
	log.Errorf("%s", "error")
	log.Fatal("fatal")
	log.Fatalf("%s", "fatal")
}
Exemplo n.º 4
0
func (this *HadoopExec) Run(job *dag.Job) error {
	if !job.ValidateAttr([]string{"mapper", "input", "output"}) {
		msg := "invalid job"
		log.Error(msg)
		return fmt.Errorf(msg)
	}

	// !!!VERY IMPORTANT!!!
	// Many other operations relay on this TrimRight.
	job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")

	this.hdfs.Rm(job.Attrs["output"])
	this.createOutput(job)

	args := this.genCmdArgs(job)
	log.Debugf("CMD: hadoop %v", args)
	retcode, err := util.ExecCmd(job.Name, "hadoop", args...)
	if err != nil {
		job.Status = dag.Failed
		return err
	}
	if retcode != 0 {
		job.Status = dag.Failed
		return fmt.Errorf("script failed: %d", retcode)
	}
	job.Status = dag.Finished
	return nil
}
Exemplo n.º 5
0
func (this *DotSerializer) Deserialize(data []byte) (*DAG, error) {
	ast, err := dotparser.ParseBytes(data)
	if err != nil {
		return nil, err
	}
	graph := dot.NewGraph()
	dot.Analyse(ast, graph)

	p := NewDAG(graph.Name)

	for src, dests := range graph.Edges.SrcToDsts {
		for dest, _ := range dests {

			if orig, ok := p.Jobs[src]; !ok {
				n := dotNameToDAGJob(graph, src)
				n.Post = append(n.Post, dest)
				p.Jobs[src] = n
			} else {
				orig.Post = append(orig.Post, dest)
			}
			if orig, ok := p.Jobs[dest]; !ok {
				n := dotNameToDAGJob(graph, dest)
				n.Prev = append(n.Prev, src)
				p.Jobs[dest] = n
			} else {
				orig.Prev = append(orig.Prev, src)
			}

			if _, ok := p.InDegrees[src]; !ok {
				p.InDegrees[src] = 0
			}
			if orig, ok := p.InDegrees[dest]; !ok {
				p.InDegrees[dest] = 1
			} else {
				p.InDegrees[dest] = orig + 1
			}
		}
	}

	for _, edge := range graph.Edges.Edges {
		if v, ok := edge.Attrs["nonstrict"]; ok {
			nonstrict, err := util.StringToBool(strings.Trim(v, "\""))
			if err != nil {
				log.Error(err)
				return nil, err
			}
			if len(edge.Src) != 0 && len(edge.Dst) != 0 {
				r := NewRelation()
				r.NonStrict = nonstrict
				if _, ok := p.Relations[edge.Src]; !ok {
					p.Relations[edge.Src] = make(map[string]*Relation)
				}
				p.Relations[edge.Src][edge.Dst] = r
			}
		}
	}

	return p, nil
}
Exemplo n.º 6
0
func (this *Sched) runQueue(queue []*dag.Job, d *dag.DAG) error {
	var wg sync.WaitGroup
	for _, job := range queue {
		wg.Add(1)
		go func(job *dag.Job, d *dag.DAG) {
			// !!! All shared objects need to be thread-safe !!!
			defer wg.Done()

			log.Infof("run job: %s", job.Name)

			if err := d.ResolveJob(job); err != nil {
				log.Error(err)

				job.Status = dag.Failed
				this.updateJobStatus(job, d)

				return
			}

			if job.Type == dag.DummyJob {
				job.Status = dag.Finished
				this.updateJobStatus(job, d)
			} else {
				jexec, err := this.getExec(job)
				if err != nil {
					panic(err)
				}

				job.Status = dag.Started
				this.updateJobStatus(job, d)
				if err = jexec.Run(job); err != nil {
					log.Error(err)
					job.Status = dag.Failed
				}
				this.updateJobStatus(job, d)
			}
		}(job, d)
	}
	wg.Wait()
	return nil
}