Example #1
0
func TestAll(t *testing.T) {
	l := log.New(os.Stdout, "test_logger ", log.LOG_LEVEL_ALL)
	l.Debug("debug")
	l.Debugf("%s", "debug")
	l.Trace("trace")
	l.Tracef("%s", "trace")
	l.Info("info")
	l.Infof("%s", "info")
	l.Warn("warn")
	l.Warnf("%s", "warn")
	l.Error("error")
	l.Errorf("%s", "error")
	l.Fatal("fatal")
	l.Fatalf("%s", "fatal")
	log.Debug("debug")
	log.Debugf("%s", "debug")
	log.Trace("trace")
	log.Tracef("%s", "trace")
	log.Info("info")
	log.Infof("%s", "info")
	log.Warn("warn")
	log.Warnf("%s", "warn")
	log.Error("error")
	log.Errorf("%s", "error")
	log.Fatal("fatal")
	log.Fatalf("%s", "fatal")
}
Example #2
0
func (this *Sched) Run(d *dag.DAG) error {
	if err := this.checkDAG(d); err != nil {
		log.Fatal(err)
		return err
	}

	d.Builtins.SetBizdate(config.Bizdate)

	queue := this.genRunQueue(d)
	for len(queue) != 0 {

		if err := this.runQueue(queue, d); err != nil {
			return err
		}

		for _, job := range queue {
			this.updateFails(job)
			this.updateDependences(job, d)
		}

		queue = this.genRunQueue(d)
	}

	this.Summary()

	if len(this.tracker.Fails) == 0 {
		log.Info("All jobs done")
		return nil
	} else {
		log.Errorf("some job failed")
		return fmt.Errorf("some job failed")
	}
}
Example #3
0
func (this *Sched) updateFails(job *dag.Job) {
	switch job.Status {
	case dag.Failed:
		log.Errorf("job %s failed", job.Name)
		if n, ok := this.tracker.Fails[job.Name]; !ok {
			this.tracker.Fails[job.Name] = 1
		} else {
			this.tracker.Fails[job.Name] = n + 1
		}
	case dag.Finished:
		if _, ok := this.tracker.Fails[job.Name]; ok {
			delete(this.tracker.Fails, job.Name)
		}
	}
}
Example #4
0
func (this *Sched) genRunQueue(d *dag.DAG) []*dag.Job {
	queue := []*dag.Job{}
	for name, in := range d.InDegrees {
		job, ok := d.Jobs[name]
		if !ok {
			panic(fmt.Errorf("panic: no corresponding job"))
		}

		if s, err := this.tracker.GetStatus(job); err != nil {
			panic(err)
		} else {
			job.Status = s
		}
		log.Debugf("check job status: %s -> %s", job.Name, job.Status)

		switch job.Status {
		case dag.Finished:
			if config.ReRun {
				if this.tracker.HasReRan(job) {
					continue
				}
				log.Infof("job is already finished, rerun: %s", job.Name)
				this.tracker.SetReRan(job)
			} else {
				log.Infof("job is already finished, skip: %s", job.Name)
				continue
			}
		case dag.Started:
			if config.Force {
				log.Warnf("job is already started, run still: %s", job.Name)
			} else {
				log.Warnf("job is already started, skip: %s", job.Name)
				continue
			}
		}

		fails := this.tracker.Fails[job.Name]
		if in == 0 && fails < config.MaxRetry {
			queue = append(queue, job)
		} else if fails >= config.MaxRetry {
			log.Errorf("job %s failed %d times, reaches max retry times: %d",
				job.Name, this.tracker.Fails[job.Name], config.MaxRetry)
		}
	}
	return queue
}
Example #5
0
func (this *HadoopExec) Setup() error {
	if len(config.HadoopStreamingJar) == 0 {
		msg := fmt.Sprintf("hadoop streaming jar not set")
		log.Errorf(msg)
		return fmt.Errorf(msg)
	} else {
		this.jar = config.HadoopStreamingJar
	}

	if fs, err := storage.NewHDFS(config.NameNode); err != nil {
		msg := fmt.Sprintf("connect to hdfs namenode failed: %s", config.NameNode)
		log.Fatal(msg)
		return fmt.Errorf(msg)
	} else {
		this.hdfs = fs.(*storage.HDFS)
	}

	return nil
}