func (this *OdpsExec) Setup() error {
if len(config.OdpsEndpoint) != 0 {
this.endpoint = config.OdpsEndpoint
} else {
msg := fmt.Errorf("odps endpoint not set")
log.Fatal(msg)
return msg
}
if len(config.OdpsProject) != 0 {
this.project = config.OdpsProject
} else {
msg := fmt.Errorf("odps project not set")
log.Fatal(msg)
return msg
}
if len(config.OdpsAccessID) != 0 {
this.accessId = config.OdpsAccessID
} else {
msg := fmt.Errorf("odps accessid not set")
log.Fatal(msg)
return msg
}
if len(config.OdpsAccessKey) != 0 {
this.accessKey = config.OdpsAccessKey
} else {
msg := fmt.Errorf("odps accesskey not set")
log.Fatal(msg)
return msg
}
return nil
}
func ExecCmd(jobname, name string, arg ...string) (int, error) {
cmd := exec.Command(name, arg...)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
return 0, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
log.Fatal(err)
return 0, err
}
if err := cmd.Start(); err != nil {
//log.Fatal(err)
return 0, err
}
errscanner := bufio.NewScanner(stderr)
for errscanner.Scan() {
if len(errscanner.Text()) != 0 {
log.Infof("<%s> %s", jobname, errscanner.Text())
}
}
if err := errscanner.Err(); err != nil {
log.Fatalf("reading standard input: %v", err)
}
outscanner := bufio.NewScanner(stdout)
for outscanner.Scan() {
if len(outscanner.Text()) != 0 {
log.Infof("<%s> %s", jobname, outscanner.Text())
}
}
if err := outscanner.Err(); err != nil {
log.Fatalf("reading standard input: %v", err)
}
if err := cmd.Wait(); err != nil {
if exiterr, ok := err.(*exec.ExitError); ok {
// The program has exited with an exit code != 0
// This works on both Unix and Windows. Although package
// syscall is generally platform dependent, WaitStatus is
// defined for both Unix and Windows and in both cases has
// an ExitStatus() method with the same signature.
status, ok := exiterr.Sys().(syscall.WaitStatus)
if !ok {
return 0, fmt.Errorf("get exit status fail")
}
return status.ExitStatus(), nil
} else {
log.Fatalf("cmd.Wait: %v", err)
return 0, err
}
}
return 0, err
}
func TestAll(t *testing.T) {
l := log.New(os.Stdout, "test_logger ", log.LOG_LEVEL_ALL)
l.Debug("debug")
l.Debugf("%s", "debug")
l.Trace("trace")
l.Tracef("%s", "trace")
l.Info("info")
l.Infof("%s", "info")
l.Warn("warn")
l.Warnf("%s", "warn")
l.Error("error")
l.Errorf("%s", "error")
l.Fatal("fatal")
l.Fatalf("%s", "fatal")
log.Debug("debug")
log.Debugf("%s", "debug")
log.Trace("trace")
log.Tracef("%s", "trace")
log.Info("info")
log.Infof("%s", "info")
log.Warn("warn")
log.Warnf("%s", "warn")
log.Error("error")
log.Errorf("%s", "error")
log.Fatal("fatal")
log.Fatalf("%s", "fatal")
}
func (this *SqliteSaver) GetFlag(job *dag.Job) (dag.JobStatus, error) {
this.lock.Lock()
defer this.lock.Unlock()
sql := fmt.Sprintf(
"select * from status where output='%s'",
job.Attrs["output"],
)
log.Debugf("sql: %s", sql)
rows, err := this.db.Query(sql)
if err != nil {
log.Fatal(err)
return dag.UnknownStatus, err
}
defer rows.Close()
for rows.Next() {
var s StatusTable
rows.Scan(&s.Output, &s.Status)
if s := dag.ParseJobStatus(s.Status); s == dag.UnknownStatus {
return dag.NotStarted, nil
} else {
return s, nil
}
}
return dag.NotStarted, nil
}
func (this *DAG) ResolveJob(job *Job) error {
vars := ""
if v, ok := job.Attrs["vars"]; ok {
vars = v
}
if resolved, err := symbol.Resolve(strings.Trim(vars, "\"'"), this.Builtins.builtins); err != nil {
return err
} else {
for _, stmt := range resolved {
job.Vars[stmt.Value.(string)] = stmt.Children[0].Value.(string)
}
}
for k, v := range job.Attrs {
if k == "vars" {
continue
}
nval, err := ApplyVarToString(v, job.Vars)
if err != nil {
log.Fatal(err)
return err
}
job.Attrs[k] = nval
}
return nil
}
func (this *Sched) Run(d *dag.DAG) error {
if err := this.checkDAG(d); err != nil {
log.Fatal(err)
return err
}
d.Builtins.SetBizdate(config.Bizdate)
queue := this.genRunQueue(d)
for len(queue) != 0 {
if err := this.runQueue(queue, d); err != nil {
return err
}
for _, job := range queue {
this.updateFails(job)
this.updateDependences(job, d)
}
queue = this.genRunQueue(d)
}
this.Summary()
if len(this.tracker.Fails) == 0 {
log.Info("All jobs done")
return nil
} else {
log.Errorf("some job failed")
return fmt.Errorf("some job failed")
}
}
func newSched() (*sched.Sched, error) {
saver, err := newStatusFlag()
if err != nil {
log.Fatal(err)
os.Exit(1)
}
return sched.NewSched(status.NewStatusTracker(saver))
}
func NewHDFS(namenode string) (Storage, error) {
if client, err := hdfs.New(config.NameNode); err != nil {
msg := fmt.Sprintf("connect to hdfs namenode failed: %s", config.NameNode)
log.Fatal(msg)
return nil, fmt.Errorf(msg)
} else {
return &HDFS{client: client}, nil
}
}
func FileExist(path string) bool {
_, err := os.Stat(path)
if err == nil {
return true
}
if os.IsNotExist(err) {
return false
}
log.Fatal("check file exist failed: %v", err)
return false
}
func main() {
config.InitFlags()
config.Parse()
loglevel := 0
if config.Verbose {
loglevel = log.LOG_LEVEL_ALL
} else {
loglevel = log.LOG_LEVEL_TRACE | log.LOG_LEVEL_INFO |
log.LOG_LEVEL_WARN | log.LOG_LEVEL_ERROR |
log.LOG_LEVEL_FATAL
}
if config.LessLog {
log.StdLogger = log.NewDefaultCleanLogger(
os.Stdout, "hpipe", loglevel)
} else {
log.StdLogger = log.NewDefault(
os.Stdout, "hpipe", loglevel)
}
d, err := dag.LoadFromFile(config.EntryFile)
if err != nil {
log.Fatal(err)
os.Exit(1)
}
util.LogLines(config.LogoString(), nil)
util.LogLines(d.String(), nil)
s, err := newSched()
if err != nil {
log.Fatal(err)
os.Exit(1)
}
if err := s.Run(d); err != nil {
os.Exit(1)
}
}
func newStatusFlag() (status.Saver, error) {
switch config.StatusSaver {
case "hdfs":
return status.NewHDFSSaver(config.NameNode)
case "sqlite":
return status.NewSqliteSaver(config.SqliteFile)
default:
msg := fmt.Sprintf("invalid status keeper type: %s",
config.StatusSaver)
log.Fatal(msg)
return nil, fmt.Errorf(msg)
}
}
func NewSqliteSaver(path string) (*SqliteSaver, error) {
exist := isExist(path)
db, err := sql.Open("sqlite3", path)
if err != nil {
log.Fatal(err)
return nil, err
}
if !exist {
sql := "drop table if exists status;create table status(output string, status string);"
log.Debugf("sql: %s", sql)
if _, err := db.Exec(sql); err != nil {
log.Fatal(err)
return nil, err
}
}
return &SqliteSaver{
db: db,
}, nil
}
func (this *SqliteSaver) ClearFlag(job *dag.Job) error {
this.lock.Lock()
defer this.lock.Unlock()
sql := fmt.Sprintf(
"delete from status where output='%s'",
job.Attrs["output"],
)
log.Debugf("sql: %s", sql)
if _, err := this.db.Exec(sql); err != nil {
log.Fatal(err)
return err
}
return nil
}
func (this *SqliteSaver) SetFlag(job *dag.Job) error {
if err := this.ClearFlag(job); err != nil {
return err
}
this.lock.Lock()
defer this.lock.Unlock()
sql := fmt.Sprintf(
"insert into status(output, status) values('%s', '%s')",
job.Attrs["output"], job.Status,
)
log.Debugf("sql: %s", sql)
if _, err := this.db.Exec(sql); err != nil {
log.Fatal(err)
return err
}
return nil
}
func (this *HadoopExec) Setup() error {
if len(config.HadoopStreamingJar) == 0 {
msg := fmt.Sprintf("hadoop streaming jar not set")
log.Errorf(msg)
return fmt.Errorf(msg)
} else {
this.jar = config.HadoopStreamingJar
}
if fs, err := storage.NewHDFS(config.NameNode); err != nil {
msg := fmt.Sprintf("connect to hdfs namenode failed: %s", config.NameNode)
log.Fatal(msg)
return fmt.Errorf(msg)
} else {
this.hdfs = fs.(*storage.HDFS)
}
return nil
}