コード例 #1
0
ファイル: odps.go プロジェクト: yonglehou/hpipe
func (this *OdpsExec) Run(job *dag.Job) error {
	if !job.ValidateAttr([]string{"script", "output"}) {
		msg := "invalid job"
		log.Error(msg)
		return fmt.Errorf(msg)
	}

	// !!!VERY IMPORTANT!!!
	// Many other operations relay on this TrimRight.
	job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")

	args := this.genCmdArgs(job)
	log.Debugf("CMD: odpscmd %s", strings.Join(args, " "))
	retcode, err := util.ExecCmd(job.Name, "odpscmd", args...)
	if err != nil {
		job.Status = dag.Failed
		return err
	}
	if retcode != 0 {
		job.Status = dag.Failed
		return fmt.Errorf("script failed: %d", retcode)
	}
	job.Status = dag.Finished
	return nil
}
コード例 #2
0
ファイル: script.go プロジェクト: yonglehou/hpipe
func (this *ScriptExec) Run(job *dag.Job) error {
	if !job.ValidateAttr([]string{"output", "script", "interpreter"}) {
		msg := "invalid job: missing output, script or interpreter"
		log.Error(msg)
		return fmt.Errorf(msg)
	}

	// !!!VERY IMPORTANT!!!
	// Many other operations relay on this TrimRight.
	job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")

	args := this.genCmdArgs(job)
	intepreter := getProp("interpreter", job.Attrs)
	log.Debugf("CMD: %s %v", intepreter, args)
	retcode, err := util.ExecCmd(job.Name, intepreter, args...)
	if err != nil {
		job.Status = dag.Failed
		return err
	}
	if retcode != 0 {
		job.Status = dag.Failed
		return fmt.Errorf("script failed: %d", retcode)
	}
	job.Status = dag.Finished
	return nil
}
コード例 #3
0
ファイル: hadoop.go プロジェクト: yonglehou/hpipe
func (this *HadoopExec) Run(job *dag.Job) error {
	if !job.ValidateAttr([]string{"mapper", "input", "output"}) {
		msg := "invalid job"
		log.Error(msg)
		return fmt.Errorf(msg)
	}

	// !!!VERY IMPORTANT!!!
	// Many other operations relay on this TrimRight.
	job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")

	this.hdfs.Rm(job.Attrs["output"])
	this.createOutput(job)

	args := this.genCmdArgs(job)
	log.Debugf("CMD: hadoop %v", args)
	retcode, err := util.ExecCmd(job.Name, "hadoop", args...)
	if err != nil {
		job.Status = dag.Failed
		return err
	}
	if retcode != 0 {
		job.Status = dag.Failed
		return fmt.Errorf("script failed: %d", retcode)
	}
	job.Status = dag.Finished
	return nil
}