Example #1
0
func (eb backoff) beginAttempt(retryKey string) (int, error) {
	conn, err := goworker.GetConn()
	if err != nil {
		return -1, err
	}
	defer goworker.PutConn(conn)

	// Create the retry key if not exists
	_, err = conn.Do("SETNX", retryKey, -1)
	if err != nil {
		return -1, err
	}

	// Increment the attempt we're on
	retryAttempt, err := redis.Int(conn.Do("INCR", retryKey))
	if err != nil {
		return -1, err
	}

	// Expire the retry key so we don't leave it hanging
	// (an hour after it was supposed to be removed)
	conn.Do("EXPIRE", retryKey, eb.retryDelay(retryAttempt)+3600)

	return retryAttempt, nil
}
Example #2
0
func (eb backoff) WorkerFunc() func(string, ...interface{}) error {
	return func(queue string, args ...interface{}) error {
		retryKey := eb.retryKey(args)

		// Setup the attempt
		retryAttempt, err := eb.beginAttempt(retryKey)
		if err != nil {
			return err
		}

		// Run the job
		workerErr := eb.worker(queue, args...)

		// Get redis connection
		conn, err := goworker.GetConn()
		if err != nil {
			return err
		}
		defer goworker.PutConn(conn)

		// Success, just clear the retry key
		if workerErr == nil {
			conn.Do("DEL", retryKey)
			return nil
		}

		if retryAttempt >= eb.RetryLimit {
			// If we've retried too many times, give up
			conn.Do("DEL", retryKey)
		} else {
			// Otherwise schedule the retry attempt
			seconds := eb.retryDelay(retryAttempt)
			if seconds <= 0 {
				// If there's no delay, just enqueue it
				_, err = resque.Enqueue(conn.Conn, queue, eb.jobName, args...)
			} else {
				// Otherwise schedule it
				delay := time.Duration(seconds) * time.Second
				err = resque.EnqueueIn(conn.Conn, delay, queue, eb.jobName, args...)
			}

			if err != nil {
				return err
			}
		}

		// Wrap the error
		return fmt.Errorf("retry: attempt %d of %d failed: %s", retryAttempt, eb.RetryLimit, workerErr.Error())
	}
}