func (this *HDFS) IsExist(path string) (bool, error) {
_, err := this.client.Stat(path)
if err != nil {
if os.IsNotExist(err) {
log.Debugf("path not exist: %s", path)
return false, nil
}
return false, err
}
log.Debugf("path exist: %s", path)
return true, nil
}
func (this *ScriptExec) Run(job *dag.Job) error {
if !job.ValidateAttr([]string{"output", "script", "interpreter"}) {
msg := "invalid job: missing output, script or interpreter"
log.Error(msg)
return fmt.Errorf(msg)
}
// !!!VERY IMPORTANT!!!
// Many other operations relay on this TrimRight.
job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")
args := this.genCmdArgs(job)
intepreter := getProp("interpreter", job.Attrs)
log.Debugf("CMD: %s %v", intepreter, args)
retcode, err := util.ExecCmd(job.Name, intepreter, args...)
if err != nil {
job.Status = dag.Failed
return err
}
if retcode != 0 {
job.Status = dag.Failed
return fmt.Errorf("script failed: %d", retcode)
}
job.Status = dag.Finished
return nil
}
func TestAll(t *testing.T) {
l := log.New(os.Stdout, "test_logger ", log.LOG_LEVEL_ALL)
l.Debug("debug")
l.Debugf("%s", "debug")
l.Trace("trace")
l.Tracef("%s", "trace")
l.Info("info")
l.Infof("%s", "info")
l.Warn("warn")
l.Warnf("%s", "warn")
l.Error("error")
l.Errorf("%s", "error")
l.Fatal("fatal")
l.Fatalf("%s", "fatal")
log.Debug("debug")
log.Debugf("%s", "debug")
log.Trace("trace")
log.Tracef("%s", "trace")
log.Info("info")
log.Infof("%s", "info")
log.Warn("warn")
log.Warnf("%s", "warn")
log.Error("error")
log.Errorf("%s", "error")
log.Fatal("fatal")
log.Fatalf("%s", "fatal")
}
func (this *OdpsExec) Run(job *dag.Job) error {
if !job.ValidateAttr([]string{"script", "output"}) {
msg := "invalid job"
log.Error(msg)
return fmt.Errorf(msg)
}
// !!!VERY IMPORTANT!!!
// Many other operations relay on this TrimRight.
job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")
args := this.genCmdArgs(job)
log.Debugf("CMD: odpscmd %s", strings.Join(args, " "))
retcode, err := util.ExecCmd(job.Name, "odpscmd", args...)
if err != nil {
job.Status = dag.Failed
return err
}
if retcode != 0 {
job.Status = dag.Failed
return fmt.Errorf("script failed: %d", retcode)
}
job.Status = dag.Finished
return nil
}
func (this *SqliteSaver) GetFlag(job *dag.Job) (dag.JobStatus, error) {
this.lock.Lock()
defer this.lock.Unlock()
sql := fmt.Sprintf(
"select * from status where output='%s'",
job.Attrs["output"],
)
log.Debugf("sql: %s", sql)
rows, err := this.db.Query(sql)
if err != nil {
log.Fatal(err)
return dag.UnknownStatus, err
}
defer rows.Close()
for rows.Next() {
var s StatusTable
rows.Scan(&s.Output, &s.Status)
if s := dag.ParseJobStatus(s.Status); s == dag.UnknownStatus {
return dag.NotStarted, nil
} else {
return s, nil
}
}
return dag.NotStarted, nil
}
func (this *HadoopExec) Run(job *dag.Job) error {
if !job.ValidateAttr([]string{"mapper", "input", "output"}) {
msg := "invalid job"
log.Error(msg)
return fmt.Errorf(msg)
}
// !!!VERY IMPORTANT!!!
// Many other operations relay on this TrimRight.
job.Attrs["output"] = strings.TrimRight(job.Attrs["output"], "/")
this.hdfs.Rm(job.Attrs["output"])
this.createOutput(job)
args := this.genCmdArgs(job)
log.Debugf("CMD: hadoop %v", args)
retcode, err := util.ExecCmd(job.Name, "hadoop", args...)
if err != nil {
job.Status = dag.Failed
return err
}
if retcode != 0 {
job.Status = dag.Failed
return fmt.Errorf("script failed: %d", retcode)
}
job.Status = dag.Finished
return nil
}
func ApplyVarToString(str string, vars map[string]string) (string, error) {
for _, token := range varPattern.FindAllStringSubmatch(str, -1) {
name := token[1]
if val, ok := vars[name]; ok {
nval := strings.Replace(str, "${"+name+"}", val, 1)
log.Debugf("resolve: %s => %s", str, nval)
str = nval
} else {
return "", fmt.Errorf("variable %s is not defined", name)
}
}
return str, nil
}
func (this *SqliteSaver) ClearFlag(job *dag.Job) error {
this.lock.Lock()
defer this.lock.Unlock()
sql := fmt.Sprintf(
"delete from status where output='%s'",
job.Attrs["output"],
)
log.Debugf("sql: %s", sql)
if _, err := this.db.Exec(sql); err != nil {
log.Fatal(err)
return err
}
return nil
}
func (this *SqliteSaver) SetFlag(job *dag.Job) error {
if err := this.ClearFlag(job); err != nil {
return err
}
this.lock.Lock()
defer this.lock.Unlock()
sql := fmt.Sprintf(
"insert into status(output, status) values('%s', '%s')",
job.Attrs["output"], job.Status,
)
log.Debugf("sql: %s", sql)
if _, err := this.db.Exec(sql); err != nil {
log.Fatal(err)
return err
}
return nil
}
func (this *Sched) genRunQueue(d *dag.DAG) []*dag.Job {
queue := []*dag.Job{}
for name, in := range d.InDegrees {
job, ok := d.Jobs[name]
if !ok {
panic(fmt.Errorf("panic: no corresponding job"))
}
if s, err := this.tracker.GetStatus(job); err != nil {
panic(err)
} else {
job.Status = s
}
log.Debugf("check job status: %s -> %s", job.Name, job.Status)
switch job.Status {
case dag.Finished:
if config.ReRun {
if this.tracker.HasReRan(job) {
continue
}
log.Infof("job is already finished, rerun: %s", job.Name)
this.tracker.SetReRan(job)
} else {
log.Infof("job is already finished, skip: %s", job.Name)
continue
}
case dag.Started:
if config.Force {
log.Warnf("job is already started, run still: %s", job.Name)
} else {
log.Warnf("job is already started, skip: %s", job.Name)
continue
}
}
fails := this.tracker.Fails[job.Name]
if in == 0 && fails < config.MaxRetry {
queue = append(queue, job)
} else if fails >= config.MaxRetry {
log.Errorf("job %s failed %d times, reaches max retry times: %d",
job.Name, this.tracker.Fails[job.Name], config.MaxRetry)
}
}
return queue
}
func NewSqliteSaver(path string) (*SqliteSaver, error) {
exist := isExist(path)
db, err := sql.Open("sqlite3", path)
if err != nil {
log.Fatal(err)
return nil, err
}
if !exist {
sql := "drop table if exists status;create table status(output string, status string);"
log.Debugf("sql: %s", sql)
if _, err := db.Exec(sql); err != nil {
log.Fatal(err)
return nil, err
}
}
return &SqliteSaver{
db: db,
}, nil
}
func (this *HDFS) Touch(path string) error {
log.Debugf("touch file: %s", path)
return this.client.CreateEmptyFile(path)
}
func (this *HDFS) Rm(path string) error {
log.Debugf("remove file: %s", path)
return this.client.Remove(path)
}
func (this *HDFS) MkdirP(path string) error {
log.Debugf("mkdir -p %s", path)
return this.client.MkdirAll(path, 0755)
}