Esempio n. 1
0
func (c *client) handleRemoveJob(msgID, payload []byte) (err error) {
	var job driver.Job
	var e error
	var conn = c.conn
	var sched = c.sched
	defer sched.jobLocker.Unlock()
	sched.jobLocker.Lock()
	job, e = driver.NewJob(payload)
	if e != nil {
		err = conn.Send([]byte(e.Error()))
		return
	}
	job, e = sched.driver.GetOne(job.Func, job.Name)
	if e == nil && job.ID > 0 {
		if _, ok := sched.procQueue[job.ID]; ok {
			delete(sched.procQueue, job.ID)
		}
		sched.driver.Delete(job.ID)
		sched.decrStatJob(job)
		if job.IsProc() {
			sched.decrStatProc(job)
			sched.removeRevertPQ(job)
		}
		sched.notifyJobTimer()
	}

	if e != nil {
		err = conn.Send([]byte(e.Error()))
	} else {
		err = c.handleCommand(msgID, protocol.SUCCESS)
	}
	return
}
Esempio n. 2
0
func (sched *Sched) SubmitJob(grabItem GrabItem, job driver.Job) bool {
	defer sched.JobLocker.Unlock()
	sched.JobLocker.Lock()
	if job.Name == "" {
		sched.driver.Delete(job.Id)
		return true
	}
	if _, ok := sched.procQueue[job.Id]; ok {
		return true
	}

	if !grabItem.w.alive {
		return false
	}
	if err := grabItem.w.HandleDo(grabItem.msgId, job); err != nil {
		grabItem.w.alive = false
		return false
	}
	now := time.Now()
	current := int64(now.Unix())
	job.Status = driver.JOB_STATUS_PROC
	job.RunAt = current
	sched.driver.Save(&job)
	sched.IncrStatProc(job)
	sched.pushRevertPQ(job)
	sched.NotifyRevertTimer()
	sched.procQueue[job.Id] = job
	sched.grabQueue.Remove(grabItem)
	return true
}
Esempio n. 3
0
func (sched *Sched) pushJobPQ(job driver.Job) bool {
	defer sched.PQLocker.Unlock()
	sched.PQLocker.Lock()
	if job.IsReady() {
		item := &queue.Item{
			Value:    job.ID,
			Priority: job.SchedAt,
		}
		if sched.cacheItem != nil && item.Priority < sched.cacheItem.Priority {
			if job.ID == sched.cacheItem.Value {
				return true
			}
			job, _ = sched.driver.Get(sched.cacheItem.Value)
			sched.cacheItem = item
			if job.ID <= 0 || !job.IsReady() {
				return false
			}
		}
		pq, ok := sched.jobPQ[job.Func]
		if !ok {
			pq1 := make(queue.PriorityQueue, 0)
			pq = &pq1
			sched.jobPQ[job.Func] = pq
			heap.Init(pq)
		}
		heap.Push(pq, item)
		return true
	}
	return false
}
Esempio n. 4
0
func SubmitJob(entryPoint string, job driver.Job) {
	parts := strings.SplitN(entryPoint, "://", 2)
	c, err := net.Dial(parts[0], parts[1])
	if err != nil {
		log.Fatal(err)
	}
	conn := protocol.NewClientConn(c)
	defer conn.Close()
	err = conn.Send(protocol.TYPE_CLIENT.Bytes())
	if err != nil {
		log.Fatal(err)
	}
	var msgId = []byte("100")
	buf := bytes.NewBuffer(nil)
	buf.Write(msgId)
	buf.Write(protocol.NULL_CHAR)
	buf.WriteByte(byte(protocol.SUBMIT_JOB))
	buf.Write(protocol.NULL_CHAR)
	buf.Write(job.Bytes())
	err = conn.Send(buf.Bytes())
	if err != nil {
		log.Fatal(err)
	}
	payload, err := conn.Receive()
	if err != nil {
		log.Fatal(err)
	}
	_, cmd, _ := protocol.ParseCommand(payload)
	fmt.Printf("%s\n", cmd.String())
}
Esempio n. 5
0
func (sched *Sched) submitJob(item grabItem, job driver.Job) bool {
	defer sched.jobLocker.Unlock()
	sched.jobLocker.Lock()
	if job.Name == "" {
		sched.driver.Delete(job.ID)
		return true
	}
	if _, ok := sched.procQueue[job.ID]; ok {
		return true
	}

	if !item.w.alive {
		return false
	}
	if err := item.w.handleJobAssign(item.msgID, job); err != nil {
		item.w.alive = false
		return false
	}
	now := time.Now()
	current := int64(now.Unix())
	job.SetProc()
	job.RunAt = current
	sched.driver.Save(&job)
	sched.incrStatProc(job)
	sched.pushRevertPQ(job)
	sched.notifyRevertTimer()
	sched.procQueue[job.ID] = job
	sched.grabQueue.remove(item)
	return true
}
Esempio n. 6
0
func (c *httpClient) handleRemoveJob(req *http.Request) {
	var job driver.Job
	var e error
	var sched = c.sched
	defer sched.jobLocker.Unlock()
	sched.jobLocker.Lock()
	url := req.URL.String()
	funcName := url[1:]
	if funcName == "" {
		funcName = req.FormValue("func")
	}
	name := req.FormValue("name")
	job, e = sched.driver.GetOne(funcName, name)
	if e == nil && job.ID > 0 {
		if _, ok := sched.procQueue[job.ID]; ok {
			delete(sched.procQueue, job.ID)
		}
		sched.driver.Delete(job.ID)
		sched.decrStatJob(job)
		if job.IsProc() {
			sched.decrStatProc(job)
			sched.removeRevertPQ(job)
		}
		sched.notifyJobTimer()
	}

	if e != nil {
		c.sendErrResponse(e)
	} else {
		c.sendResponse("200 OK", []byte("{\"msg\": \""+protocol.SUCCESS.String()+"\"}"))
	}
}
Esempio n. 7
0
func (sched *Sched) removeRevertPQ(job driver.Job) {
	defer sched.PQLocker.Unlock()
	sched.PQLocker.Lock()
	if job.IsProc() && job.Timeout > 0 {
		for _, item := range sched.revertPQ {
			if item.Value == job.ID {
				heap.Remove(&sched.revertPQ, item.Index)
				break
			}
		}
	}
}
Esempio n. 8
0
func (worker *Worker) HandleDo(msgId int64, job driver.Job) (err error) {
	defer worker.locker.Unlock()
	worker.locker.Lock()
	worker.jobQueue[job.Id] = job
	buf := bytes.NewBuffer(nil)
	buf.WriteString(strconv.FormatInt(msgId, 10))
	buf.Write(protocol.NULL_CHAR)
	buf.WriteString(strconv.FormatInt(job.Id, 10))
	buf.Write(protocol.NULL_CHAR)
	buf.Write(job.Bytes())
	err = worker.conn.Send(buf.Bytes())
	return
}
Esempio n. 9
0
func (w *worker) handleJobAssign(msgId []byte, job driver.Job) (err error) {
	defer w.locker.Unlock()
	w.locker.Lock()
	w.jobQueue[job.Id] = job
	buf := bytes.NewBuffer(nil)
	buf.Write(msgId)
	buf.Write(protocol.NULL_CHAR)
	buf.Write(protocol.JOB_ASSIGN.Bytes())
	buf.Write(protocol.NULL_CHAR)
	buf.WriteString(strconv.FormatInt(job.Id, 10))
	buf.Write(protocol.NULL_CHAR)
	buf.Write(job.Bytes())
	err = w.conn.Send(buf.Bytes())
	return
}
Esempio n. 10
0
func (sched *Sched) pushRevertPQ(job driver.Job) {
	defer sched.PQLocker.Unlock()
	sched.PQLocker.Lock()
	if job.IsProc() && job.Timeout > 0 {
		runAt := job.RunAt
		if runAt == 0 {
			runAt = job.SchedAt
		}
		item := &queue.Item{
			Value:    job.ID,
			Priority: runAt + job.Timeout,
		}
		heap.Push(&sched.revertPQ, item)
	}
}
Esempio n. 11
0
func (c *httpClient) handleSubmitJob(req *http.Request) {
	var job driver.Job
	var e error
	var sched = c.sched
	defer sched.jobLocker.Unlock()
	sched.jobLocker.Lock()
	url := req.URL.String()
	funcName := url[1:]
	if funcName == "" {
		funcName = req.FormValue("func")
	}
	job.Name = req.FormValue("name")
	job.Func = funcName
	job.Args = req.FormValue("args")
	job.Timeout, _ = strconv.ParseInt(req.FormValue("timeout"), 10, 64)
	job.SchedAt, _ = strconv.ParseInt(req.FormValue("sched_at"), 10, 64)

	if job.Name == "" || job.Func == "" {
		c.sendErrResponse(errors.New("job name or func is required"))
		return
	}

	isNew := true
	changed := false
	job.SetReady()
	oldJob, e := sched.driver.GetOne(job.Func, job.Name)
	if e == nil && oldJob.ID > 0 {
		job.ID = oldJob.ID
		if job.IsProc() {
			sched.decrStatProc(oldJob)
			sched.removeRevertPQ(job)
			changed = true
		}
		isNew = false
	}
	e = sched.driver.Save(&job)
	if e != nil {
		c.sendErrResponse(e)
		return
	}

	if isNew {
		sched.incrStatJob(job)
	}
	if isNew || changed {
		sched.pushJobPQ(job)
	}
	sched.notifyJobTimer()
	c.sendResponse("200 OK", []byte("{\"msg\": \""+protocol.SUCCESS.String()+"\"}"))
	return
}
Esempio n. 12
0
func (r RedisDriver) Save(job *driver.Job) (err error) {
	defer r.RWLocker.Unlock()
	r.RWLocker.Lock()
	var key string
	var prefix = REDIS_PREFIX + job.Func + ":"
	var conn = r.pool.Get()
	defer conn.Close()
	if job.Id > 0 {
		old, e := r.get(job.Id)
		key = REDIS_PREFIX + strconv.FormatInt(job.Id, 10)
		if e != nil || old.Id < 1 {
			err = errors.New(fmt.Sprintf("Update Job %d fail, the old job is not exists.", job.Id))
			return
		}
		r.cache.Remove(key)
		if old.Name != job.Name {
			if _, e := conn.Do("ZERM", prefix+"name", old.Name); e != nil {
				log.Printf("Error: ZREM %s %s failed\n", prefix+"name", old.Name)
			}
		}
	} else {
		job.Id, err = redis.Int64(conn.Do("INCRBY", REDIS_PREFIX+"sequence", 1))
		if err != nil {
			return
		}
	}
	idx, _ := redis.Int64(conn.Do("ZSCORE", prefix+"name", job.Name))
	if idx > 0 && idx != job.Id {
		err = errors.New("Duplicate Job name: " + job.Name)
		return
	}
	key = REDIS_PREFIX + strconv.FormatInt(job.Id, 10)
	_, err = conn.Do("SET", key, job.Bytes())
	if err == nil {
		if _, e := conn.Do("ZADD", prefix+"name", job.Id, job.Name); e != nil {
			log.Printf("Error: ZADD %s %d %s fail\n", prefix+"name", job.Id, job.Name)
		}
		if _, e := conn.Do("ZADD", REDIS_PREFIX+"ID", job.Id, strconv.FormatInt(job.Id, 10)); e != nil {
			log.Printf("Error: ZADD %s %d %d fail\n", REDIS_PREFIX+"ID", job.Id, job.Id)
		}
	}
	return
}
Esempio n. 13
0
func (c *client) handleSubmitJob(msgID []byte, payload []byte) (err error) {
	var job driver.Job
	var e error
	var conn = c.conn
	var sched = c.sched
	defer sched.jobLocker.Unlock()
	sched.jobLocker.Lock()
	job, e = driver.NewJob(payload)
	if e != nil {
		err = conn.Send([]byte(e.Error()))
		return
	}
	isNew := true
	changed := false
	job.SetReady()
	oldJob, e := sched.driver.GetOne(job.Func, job.Name)
	if e == nil && oldJob.ID > 0 {
		job.ID = oldJob.ID
		if oldJob.IsProc() {
			sched.decrStatProc(oldJob)
			sched.removeRevertPQ(job)
			changed = true
		}
		isNew = false
	}
	e = sched.driver.Save(&job)
	if e != nil {
		err = conn.Send([]byte(e.Error()))
		return
	}

	if isNew {
		sched.incrStatJob(job)
	}
	if isNew || changed {
		sched.pushJobPQ(job)
	}
	sched.notifyJobTimer()
	err = c.handleCommand(msgID, protocol.SUCCESS)
	return
}
Esempio n. 14
0
func (client *Client) HandleSubmitJob(msgId int64, payload []byte) (err error) {
	var job driver.Job
	var e error
	var conn = client.conn
	var sched = client.sched
	defer sched.JobLocker.Unlock()
	sched.JobLocker.Lock()
	job, e = driver.NewJob(payload)
	if e != nil {
		err = conn.Send([]byte(e.Error()))
		return
	}
	is_new := true
	changed := false
	job.Status = driver.JOB_STATUS_READY
	oldJob, e := sched.driver.GetOne(job.Func, job.Name)
	if e == nil && oldJob.Id > 0 {
		job.Id = oldJob.Id
		if oldJob.Status == driver.JOB_STATUS_PROC {
			sched.DecrStatProc(oldJob)
			sched.removeRevertPQ(job)
			changed = true
		}
		is_new = false
	}
	e = sched.driver.Save(&job)
	if e != nil {
		err = conn.Send([]byte(e.Error()))
		return
	}

	if is_new {
		sched.IncrStatJob(job)
	}
	if is_new || changed {
		sched.pushJobPQ(job)
	}
	sched.NotifyJobTimer()
	err = client.HandleCommand(msgId, protocol.SUCCESS)
	return
}
Esempio n. 15
0
func (l LevelDBDriver) Save(job *driver.Job) (err error) {
	defer l.RWLocker.Unlock()
	l.RWLocker.Lock()
	batch := new(leveldb.Batch)
	var isNew = true
	if job.Id > 0 {
		isNew = false
	} else {
		last_id, e := l.db.Get([]byte(PRE_SEQUENCE+"JOB"), nil)
		if e != nil || last_id == nil {
			job.Id = 1
		} else {
			id, _ := strconv.ParseInt(string(last_id), 10, 64)
			job.Id = id + 1
		}
	}
	var strId = strconv.FormatInt(job.Id, 10)
	if isNew {
		batch.Put([]byte(PRE_SEQUENCE+"JOB"), []byte(strId))
		batch.Put([]byte(PRE_JOB_FUNC+job.Func+":"+job.Name), []byte(strId))
	} else {
		old, e := l.get(job.Id)
		if e != nil || old.Id == 0 {
			err = errors.New(fmt.Sprintf("Update Job %d fail, the old job is not exists.", job.Id))
			return
		}
		l.cache.Remove(PRE_JOB + strId)
		if old.Name != job.Name {
			batch.Delete([]byte(PRE_JOB_FUNC + job.Func + ":" + old.Name))
			batch.Put([]byte(PRE_JOB_FUNC+job.Func+":"+job.Name), []byte(strId))
		}
	}
	batch.Put([]byte(PRE_JOB+strId), job.Bytes())
	err = l.db.Write(batch, nil)
	return
}
Esempio n. 16
0
// Save job. when job is exists update it, other create one.
func (l Driver) Save(job *driver.Job, force ...bool) (err error) {
	defer l.RWLocker.Unlock()
	l.RWLocker.Lock()
	batch := new(leveldb.Batch)
	var isNew = true
	if job.ID > 0 {
		isNew = false
	} else {
		lastID, e := l.db.Get([]byte(PRESEQUENCE+"JOB"), nil)
		if e != nil || lastID == nil {
			job.ID = 1
		} else {
			id, _ := strconv.ParseInt(string(lastID), 10, 64)
			job.ID = id + 1
		}
	}
	var strID = strconv.FormatInt(job.ID, 10)
	if isNew {
		batch.Put([]byte(PRESEQUENCE+"JOB"), []byte(strID))
		batch.Put([]byte(PREFUNC+job.Func+":"+job.Name), []byte(strID))
	} else if len(force) == 0 || !force[0] {
		old, e := l.get(job.ID)
		if e != nil || old.ID == 0 {
			err = fmt.Errorf("Update Job %d fail, the old job is not exists.", job.ID)
			return
		}
		l.cache.Remove(PREJOB + strID)
		if old.Name != job.Name {
			batch.Delete([]byte(PREFUNC + job.Func + ":" + old.Name))
			batch.Put([]byte(PREFUNC+job.Func+":"+job.Name), []byte(strID))
		}
	}
	batch.Put([]byte(PREJOB+strID), job.Bytes())
	err = l.db.Write(batch, nil)
	return
}
Esempio n. 17
0
func (sched *Sched) decrStatProc(job driver.Job) {
	stat := sched.getFuncStat(job.Func)
	if job.IsProc() {
		stat.Processing.Decr()
	}
}
Esempio n. 18
0
func main() {
	app := cli.NewApp()
	app.Name = "periodic"
	app.Usage = "Periodic task system"
	app.Version = periodic.Version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "H",
			Value:  "unix:///tmp/periodic.sock",
			Usage:  "the server address eg: tcp://127.0.0.1:5000",
			EnvVar: "PERIODIC_PORT",
		},
		cli.StringFlag{
			Name:  "redis",
			Value: "tcp://127.0.0.1:6379",
			Usage: "The redis server address, required for driver redis",
		},
		cli.StringFlag{
			Name:  "driver",
			Value: "memstore",
			Usage: "The driver [memstore, leveldb, redis]",
		},
		cli.StringFlag{
			Name:  "dbpath",
			Value: "leveldb",
			Usage: "The db path, required for driver leveldb",
		},
		cli.BoolFlag{
			Name:  "d",
			Usage: "Enable daemon mode",
		},
		cli.IntFlag{
			Name:  "timeout",
			Value: 0,
			Usage: "The socket timeout",
		},
		cli.IntFlag{
			Name:   "cpus",
			Value:  runtime.NumCPU(),
			Usage:  "The runtime.GOMAXPROCS",
			EnvVar: "GOMAXPROCS",
		},
		cli.StringFlag{
			Name:  "cpuprofile",
			Value: "",
			Usage: "write cpu profile to file",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:  "status",
			Usage: "Show status",
			Action: func(c *cli.Context) {
				subcmd.ShowStatus(c.GlobalString("H"))
			},
		},
		{
			Name:  "submit",
			Usage: "Submit job",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "f",
					Value: "",
					Usage: "function name",
				},
				cli.StringFlag{
					Name:  "n",
					Value: "",
					Usage: "job name",
				},
				cli.StringFlag{
					Name:  "args",
					Value: "",
					Usage: "job workload",
				},
				cli.IntFlag{
					Name:  "t",
					Value: 0,
					Usage: "job running timeout",
				},
				cli.IntFlag{
					Name:  "sched_later",
					Value: 0,
					Usage: "job sched_later",
				},
			},
			Action: func(c *cli.Context) {
				var job = driver.Job{
					Name:    c.String("n"),
					Func:    c.String("f"),
					Args:    c.String("args"),
					Timeout: int64(c.Int("t")),
				}
				if len(job.Name) == 0 || len(job.Func) == 0 {
					cli.ShowCommandHelp(c, "submit")
					log.Fatal("Job name and func is require")
				}
				delay := c.Int("sched_later")
				var now = time.Now()
				job.SchedAt = int64(now.Unix()) + int64(delay)
				subcmd.SubmitJob(c.GlobalString("H"), job)
			},
		},
		{
			Name:  "remove",
			Usage: "Remove job",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "f",
					Value: "",
					Usage: "function name",
				},
				cli.StringFlag{
					Name:  "n",
					Value: "",
					Usage: "job name",
				},
			},
			Action: func(c *cli.Context) {
				var job = driver.Job{
					Name: c.String("n"),
					Func: c.String("f"),
				}
				if len(job.Name) == 0 || len(job.Func) == 0 {
					cli.ShowCommandHelp(c, "remove")
					log.Fatal("Job name and func is require")
				}
				subcmd.RemoveJob(c.GlobalString("H"), job)
			},
		},
		{
			Name:  "drop",
			Usage: "Drop func",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "f",
					Value: "",
					Usage: "function name",
				},
			},
			Action: func(c *cli.Context) {
				Func := c.String("f")
				if len(Func) == 0 {
					cli.ShowCommandHelp(c, "drop")
					log.Fatal("function name is required")
				}
				subcmd.DropFunc(c.GlobalString("H"), Func)
			},
		},
		{
			Name:  "run",
			Usage: "Run func",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "f",
					Value: "",
					Usage: "function name required",
				},
				cli.StringFlag{
					Name:  "exec",
					Value: "",
					Usage: "command required",
				},
			},
			Action: func(c *cli.Context) {
				Func := c.String("f")
				exec := c.String("exec")
				if len(Func) == 0 {
					cli.ShowCommandHelp(c, "run")
					log.Fatal("function name is required")
				}
				if len(exec) == 0 {
					cli.ShowCommandHelp(c, "run")
					log.Fatal("command is required")
				}
				subcmd.Run(c.GlobalString("H"), Func, exec)
			},
		},
		{
			Name:  "dump",
			Usage: "Dump database to file.",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "o",
					Value: "dump.db",
					Usage: "output file name required",
				},
			},
			Action: func(c *cli.Context) {
				subcmd.Dump(c.GlobalString("H"), c.String("o"))
			},
		},
		{
			Name:  "load",
			Usage: "Load file to database.",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i",
					Value: "dump.db",
					Usage: "input file name required",
				},
			},
			Action: func(c *cli.Context) {
				subcmd.Load(c.GlobalString("H"), c.String("i"))
			},
		},
	}
	app.Action = func(c *cli.Context) {
		if c.Bool("d") {
			if c.String("cpuprofile") != "" {
				f, err := os.Create(c.String("cpuprofile"))
				if err != nil {
					log.Fatal(err)
				}
				pprof.StartCPUProfile(f)
				defer pprof.StopCPUProfile()
			}
			var store driver.StoreDriver
			switch c.String("driver") {
			case "memstore":
				store = driver.NewMemStroeDriver()
				break
			case "redis":
				store = redis.NewDriver(c.String("redis"))
				break
			case "leveldb":
				store = leveldb.NewDriver(c.String("dbpath"))
				break
			default:
				store = driver.NewMemStroeDriver()
				break
			}

			runtime.GOMAXPROCS(c.Int("cpus"))
			timeout := time.Duration(c.Int("timeout"))
			periodicd := periodic.NewSched(c.String("H"), store, timeout)
			go periodicd.Serve()
			s := make(chan os.Signal, 1)
			signal.Notify(s, os.Interrupt, os.Kill)
			<-s
			periodicd.Close()
		} else {
			cli.ShowAppHelp(c)
		}
	}

	app.Run(os.Args)
}