Beispiel #1
0
// Launch starts a new worker process. The worker subscribes
// to the default queue and processes incoming registered tasks
func (worker *Worker) Launch() error {
	cnf := worker.server.GetConfig()
	broker := worker.server.GetBroker()

	log.Printf("Launching a worker with the following settings:")
	log.Printf("- Broker: %s", cnf.Broker)
	log.Printf("- ResultBackend: %s", cnf.ResultBackend)
	log.Printf("- Exchange: %s", cnf.Exchange)
	log.Printf("- ExchangeType: %s", cnf.ExchangeType)
	log.Printf("- DefaultQueue: %s", cnf.DefaultQueue)
	log.Printf("- BindingKey: %s", cnf.BindingKey)

	errChan := make(chan error)

	go func() {
		retryFunc := utils.RetryClosure()
		for {
			retryFunc()
			retry, err := broker.StartConsuming(worker.ConsumerTag, worker)

			if !retry {
				errChan <- err // stop the goroutine
				break
			}

			log.Print(err)
		}
	}()

	return <-errChan
}
Beispiel #2
0
// StartConsuming enters a loop and waits for incoming messages
func (amqpBroker *AMQPBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
	if amqpBroker.retryFunc == nil {
		amqpBroker.retryFunc = utils.RetryClosure()
	}

	conn, channel, queue, _, err := amqpBroker.open()
	if err != nil {
		amqpBroker.retryFunc()
		return true, err // retry true
	}

	amqpBroker.retryFunc = utils.RetryClosure()

	defer amqpBroker.close(channel, conn)

	amqpBroker.stopChan = make(chan int)

	if err := channel.Qos(
		3,     // prefetch count
		0,     // prefetch size
		false, // global
	); err != nil {
		return false, fmt.Errorf("Channel Qos: %s", err)
	}

	deliveries, err := channel.Consume(
		queue.Name,  // queue
		consumerTag, // consumer tag
		false,       // auto-ack
		false,       // exclusive
		false,       // no-local
		false,       // no-wait
		nil,         // arguments
	)
	if err != nil {
		return false, fmt.Errorf("Queue Consume: %s", err)
	}

	log.Print("[*] Waiting for messages. To exit press CTRL+C")

	if err := amqpBroker.consume(deliveries, taskProcessor); err != nil {
		return true, err // retry true
	}

	return false, nil
}
Beispiel #3
0
// Consumes messages
func (redisBroker *RedisBroker) consume(errorsChan chan error, taskProcessor TaskProcessor) {
	defer redisBroker.wg.Done()

	for {
		conn, err := redisBroker.open()
		if err != nil {
			redisBroker.retryFunc()
			errorsChan <- err
			return
		}

		redisBroker.retryFunc = utils.RetryClosure()

		defer conn.Close()

		log.Print("[*] Waiting for messages. To exit press CTRL+C")

		select {
		case <-redisBroker.quitChan:
			return
		default:
			// Return value of BLPOP is an array. For example:
			// redis> RPUSH list1 a b c
			// (integer) 3
			// redis> BLPOP list1 list2 0
			// 1) "list1"
			// 2) "a"
			multiBulk, err := redis.MultiBulk(conn.Do("BLPOP", redisBroker.config.DefaultQueue, "0"))
			if err != nil {
				errorsChan <- err
				return
			}

			item, err := redis.Bytes(multiBulk[1], nil)
			if err != nil {
				errorsChan <- err
				return
			}

			log.Printf("Received new message: %s", item)

			signature := signatures.TaskSignature{}
			if err := json.Unmarshal(item, &signature); err != nil {
				errorsChan <- err
				return
			}

			if err := taskProcessor.Process(&signature); err != nil {
				errorsChan <- err
				return
			}
		}
	}
}
Beispiel #4
0
// StartConsuming enters a loop and waits for incoming messages
func (redisBroker *RedisBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
	redisBroker.stopChan = make(chan int)
	redisBroker.quitChan = make(chan int)
	redisBroker.wg.Add(1)
	errorsChan := make(chan error)
	if redisBroker.retryFunc == nil {
		redisBroker.retryFunc = utils.RetryClosure()
	}

	go redisBroker.consume(errorsChan, taskProcessor)

	for {
		select {
		case err := <-errorsChan:
			return true, err // retry true
		case <-redisBroker.stopChan:
			return false, nil // retry false
		}
	}
}
Beispiel #5
0
// StartConsuming enters a loop and waits for incoming messages
func (redisBroker *RedisBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
	if redisBroker.retryFunc == nil {
		redisBroker.retryFunc = utils.RetryClosure()
	}

	redisBroker.pool = redisBroker.newPool()
	defer redisBroker.pool.Close()

	_, err := redisBroker.pool.Get().Do("PING")
	if err != nil {
		redisBroker.retryFunc()
		return redisBroker.retry, err // retry true
	}

	redisBroker.retryFunc = utils.RetryClosure()
	redisBroker.stopChan = make(chan int)
	redisBroker.stopReceivingChan = make(chan int)
	redisBroker.errorsChan = make(chan error)
	deliveries := make(chan []byte)

	redisBroker.wg.Add(1)

	go func() {
		defer redisBroker.wg.Done()

		log.Print("[*] Waiting for messages. To exit press CTRL+C")

		conn := redisBroker.pool.Get()

		for {
			select {
			// A way to stop this goroutine from redisBroker.StopConsuming
			case <-redisBroker.stopReceivingChan:
				return
			default:
				itemBytes, err := conn.Do("BLPOP", redisBroker.config.DefaultQueue, "1")
				if err != nil {
					redisBroker.errorsChan <- err
					return
				}
				// Unline BLPOP, LPOP is non blocking so nil means we can keep iterating
				if itemBytes == nil {
					continue
				}

				items, err := redis.ByteSlices(itemBytes, nil)
				if err != nil {
					redisBroker.errorsChan <- err
					return
				}

				if len(items) != 2 {
					log.Println("Got unexpected amount of byte arrays, ignoring")
					continue
				}
				// items[0] - queue name (key), items[1] - value
				item := items[1]
				signature := signatures.TaskSignature{}
				if err := json.Unmarshal(item, &signature); err != nil {
					redisBroker.errorsChan <- err
					return
				}

				// If the task is not registered, we requeue it,
				// there might be different workers for processing specific tasks
				if !redisBroker.IsTaskRegistered(signature.Name) {
					_, err := conn.Do("RPUSH", redisBroker.config.DefaultQueue, item)

					if err != nil {
						redisBroker.errorsChan <- err
						return
					}

					continue
				}

				deliveries <- item
			}
		}
	}()

	if err := redisBroker.consume(deliveries, taskProcessor); err != nil {
		return redisBroker.retry, err // retry true
	}

	return redisBroker.retry, nil
}
Beispiel #6
0
// StartConsuming enters a loop and waits for incoming messages
func (redisBroker *RedisBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
	if redisBroker.retryFunc == nil {
		redisBroker.retryFunc = utils.RetryClosure()
	}

	redisBroker.pool = redisBroker.newPool()
	defer redisBroker.pool.Close()

	_, err := redisBroker.pool.Get().Do("PING")
	if err != nil {
		redisBroker.retryFunc()
		return true, err // retry true
	}

	redisBroker.retryFunc = utils.RetryClosure()
	redisBroker.stopChan = make(chan int)
	redisBroker.stopReceivingChan = make(chan int)
	redisBroker.errorsChan = make(chan error)
	deliveries := make(chan []byte)

	redisBroker.wg.Add(1)

	go func() {
		defer redisBroker.wg.Done()

		log.Print("[*] Waiting for messages. To exit press CTRL+C")

		conn := redisBroker.pool.Get()

		for {
			select {
			// A way to stop this goroutine from redisBroker.StopConsuming
			case <-redisBroker.stopReceivingChan:
				return
			default:
				itemBytes, err := conn.Do("LPOP", redisBroker.config.DefaultQueue)
				if err != nil {
					redisBroker.errorsChan <- err
					return
				}
				// Unline BLPOP, LPOP is non blocking so nil means we can keep iterating
				if itemBytes == nil {
					continue
				}

				item, err := redis.Bytes(itemBytes, nil)
				if err != nil {
					redisBroker.errorsChan <- err
					return
				}

				deliveries <- item
			}
		}
	}()

	if err := redisBroker.consume(deliveries, taskProcessor); err != nil {
		return true, err // retry true
	}

	return false, nil
}