コード例 #1
0
ファイル: taskqueue.go プロジェクト: nevsnode/gordon
func taskWorker(ct config.Task, errorBackoff *backoff.Backoff) {
	for task := range workerChan[ct.Type] {
		output.Debug("Executing task type", ct.Type, "with arguments", task.Args)
		stats.IncrTaskCount(ct.Type)

		err := task.Execute(ct.Script)
		if err != nil {
			task.ErrorMessage = fmt.Sprintf("%s", err)
			failedChan <- failedTask{
				configTask: ct,
				queueTask:  task,
			}

			msg := fmt.Sprintf("Failed executing task: %s \"%s\"\n%s", ct.Script, strings.Join(task.Args, "\" \""), err)
			output.NotifyError(msg)
		}

		if errorBackoff != nil {
			if err == nil {
				errorBackoff.Reset()
			} else {
				errorBackoff.Duration()
			}
		}

		output.Debug("Finished task type", ct.Type, "with arguments", task.Args)
	}

	waitGroup.Done()
}
コード例 #2
0
ファイル: taskqueue.go プロジェクト: nevsnode/gordon
func failedTaskWorker() {
	for ft := range failedChan {
		ct := ft.configTask
		qt := ft.queueTask

		if ct.FailedTasksTTL == 0 {
			return
		}

		rc, err := redisPool.Get()
		if err != nil {
			output.NotifyError("redisPool.Get():", err)
			return
		}
		defer redisPool.Put(rc)

		queueKey := conf.RedisQueueKey + ":" + ct.Type + ":failed"

		jsonString, err := qt.GetJSONString()
		if err != nil {
			output.NotifyError("failedTaskWorker(), qt.GetJSONString():", err)
			return
		}

		// add to list
		reply := rc.Cmd("RPUSH", queueKey, jsonString)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), RPUSH:", reply.Err)
			return
		}

		// set expire
		reply = rc.Cmd("EXPIRE", queueKey, ct.FailedTasksTTL)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), EXPIRE:", reply.Err)
			return
		}
	}

	waitGroupFailed.Done()
}
コード例 #3
0
ファイル: taskqueue.go プロジェクト: nevsnode/gordon
// Start initialises several variables and creates necessary go-routines
func Start(c config.Config) {
	conf = c

	poolSize := 1
	for _, ct := range conf.Tasks {
		if ct.FailedTasksTTL > 0 {
			poolSize++
			break
		}
	}

	var err error
	redisPool, err = pool.New(conf.RedisNetwork, conf.RedisAddress, poolSize)
	if err != nil {
		output.NotifyError("redis pool.New():", err)
	}

	stats.InitTasks(conf.Tasks)

	workerChan = make(map[string]chan QueueTask)
	failedChan = make(chan failedTask)
	shutdownChan = make(chan bool, 1)

	for _, ct := range conf.Tasks {
		var eb *backoff.Backoff
		if ct.BackoffEnabled {
			eb = &backoff.Backoff{
				Min:    time.Duration(ct.BackoffMin) * time.Millisecond,
				Max:    time.Duration(ct.BackoffMax) * time.Millisecond,
				Factor: ct.BackoffFactor,
				Jitter: true,
			}
		}

		workerChan[ct.Type] = make(chan QueueTask, backlog)

		for i := 0; i < ct.Workers; i++ {
			waitGroup.Add(1)
			go taskWorker(ct, eb)
		}
	}

	waitGroupFailed.Add(1)
	go failedTaskWorker()

	waitGroup.Add(1)
	go queueWorker()
}
コード例 #4
0
ファイル: taskqueue.go プロジェクト: nevsnode/gordon
func queueWorker() {
	interval := backoff.Backoff{
		Min:    time.Duration(conf.IntervalMin) * time.Millisecond,
		Max:    time.Duration(conf.IntervalMax) * time.Millisecond,
		Factor: conf.IntervalFactor,
	}

	runIntervalLoop := make(chan bool)
	doneIntervalLoop := make(chan bool)

	go func() {
		for {
			<-doneIntervalLoop
			time.Sleep(interval.Duration())

			if isShuttingDown() {
				break
			}

			runIntervalLoop <- true
		}
	}()

	go func() {
		for {
			select {
			case <-shutdownChan:
				runIntervalLoop <- false
			}
		}
	}()

	doneIntervalLoop <- true

	for <-runIntervalLoop {
		for taskType, configTask := range conf.Tasks {
			if isShuttingDown() {
				break
			}

			output.Debug("Checking for new tasks (" + taskType + ")")

			// check if there are available workers
			if !acceptsTasks(taskType) {
				continue
			}

			queueKey := conf.RedisQueueKey + ":" + taskType

			llen, err := redisPool.Cmd("LLEN", queueKey).Int()
			if err != nil {
				// Errors here are likely redis-connection errors, so we'll
				// need to notify about it
				output.NotifyError("redisPool.Cmd() Error:", err)
				break
			}

			// there are no new tasks in redis
			if llen == 0 {
				continue
			}

			// iterate over all entries in redis, until no more are available,
			// or all workers are busy, for a maximum of 2 * workers
			for i := 0; i < (configTask.Workers * 2); i++ {
				if !acceptsTasks(taskType) {
					break
				}

				value, err := redisPool.Cmd("LPOP", queueKey).Str()
				if err != nil {
					// no more tasks found
					break
				}

				output.Debug("Fetched task for type", taskType, "with payload", value)

				task, err := NewQueueTask(value)
				if err != nil {
					output.NotifyError("NewQueueTask():", err)
					continue
				}

				workerChan[taskType] <- task

				// we've actually are handling new tasks so reset the interval
				interval.Reset()
			}
		}

		doneIntervalLoop <- true
	}

	Stop()
	waitGroup.Done()
}
コード例 #5
0
ファイル: main.go プロジェクト: nevsnode/gordon
func main() {
	flag.Parse()

	if cli.version == true {
		fmt.Printf("Gordon version %s\n", GordonVersion)
		os.Exit(0)
	}

	// When no configuration file was passed as a flag, use the default location.
	if cli.config == "" {
		cli.config = utils.Basepath(config.DefaultConfig)
	}

	conf, err := config.New(cli.config)

	// When test-flag is set, respond accordingly
	if cli.test {
		if err != nil {
			fmt.Println("Configuration is invalid:", err)
		} else {
			fmt.Println("Configuration is valid")
		}
		os.Exit(0)
	}

	if err != nil {
		log.Fatal("Configuration is invalid:", err)
	}

	stats.GordonVersion = GordonVersion
	output.SetDebug(cli.verbose)
	output.SetErrorScript(conf.ErrorScript)
	output.SetTempDir(utils.Basepath(conf.TempDir))

	// Set logfile for output, when configured
	if conf.Logfile != "" {
		err = output.SetLogfile(utils.Basepath(conf.Logfile))
		if err != nil {
			log.Fatal("output.SetLogfile(): ", err)
		}
	}

	taskqueue.Start(conf)

	// If the StatsInterface was set, start the HTTP-server for it.
	if conf.Stats.Interface != "" {
		if conf.Stats.Pattern == "" {
			conf.Stats.Pattern = "/"
		}

		go func() {
			if err := stats.Serve(conf.Stats); err != nil {
				output.NotifyError("stats.Serve():", err)
			}
		}()
	}

	// Start another go-routine to initiate the graceful shutdown of all taskqueue-workers,
	// when the application shall be terminated.
	cc := make(chan os.Signal)
	signal.Notify(cc, os.Interrupt, os.Kill, syscall.SIGTERM)
	go func() {
		<-cc
		output.Debug("Stopping taskqueue")
		taskqueue.Stop()
	}()

	output.Debug("Up and waiting for tasks")
	taskqueue.Wait()
}