// Launch starts a new worker process. The worker subscribes
// to the default queue and processes incoming registered tasks
func (worker *Worker) Launch() error {
cnf := worker.server.GetConfig()
broker := worker.server.GetBroker()
log.Printf("Launching a worker with the following settings:")
log.Printf("- Broker: %s", cnf.Broker)
log.Printf("- ResultBackend: %s", cnf.ResultBackend)
log.Printf("- Exchange: %s", cnf.Exchange)
log.Printf("- ExchangeType: %s", cnf.ExchangeType)
log.Printf("- DefaultQueue: %s", cnf.DefaultQueue)
log.Printf("- BindingKey: %s", cnf.BindingKey)
errChan := make(chan error)
go func() {
retryFunc := utils.RetryClosure()
for {
retryFunc()
retry, err := broker.StartConsuming(worker.ConsumerTag, worker)
if !retry {
errChan <- err // stop the goroutine
break
}
log.Print(err)
}
}()
return <-errChan
}
// StartConsuming enters a loop and waits for incoming messages
func (amqpBroker *AMQPBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
if amqpBroker.retryFunc == nil {
amqpBroker.retryFunc = utils.RetryClosure()
}
conn, channel, queue, _, err := amqpBroker.open()
if err != nil {
amqpBroker.retryFunc()
return true, err // retry true
}
amqpBroker.retryFunc = utils.RetryClosure()
defer amqpBroker.close(channel, conn)
amqpBroker.stopChan = make(chan int)
if err := channel.Qos(
3, // prefetch count
0, // prefetch size
false, // global
); err != nil {
return false, fmt.Errorf("Channel Qos: %s", err)
}
deliveries, err := channel.Consume(
queue.Name, // queue
consumerTag, // consumer tag
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // arguments
)
if err != nil {
return false, fmt.Errorf("Queue Consume: %s", err)
}
log.Print("[*] Waiting for messages. To exit press CTRL+C")
if err := amqpBroker.consume(deliveries, taskProcessor); err != nil {
return true, err // retry true
}
return false, nil
}
// Consumes messages
func (redisBroker *RedisBroker) consume(errorsChan chan error, taskProcessor TaskProcessor) {
defer redisBroker.wg.Done()
for {
conn, err := redisBroker.open()
if err != nil {
redisBroker.retryFunc()
errorsChan <- err
return
}
redisBroker.retryFunc = utils.RetryClosure()
defer conn.Close()
log.Print("[*] Waiting for messages. To exit press CTRL+C")
select {
case <-redisBroker.quitChan:
return
default:
// Return value of BLPOP is an array. For example:
// redis> RPUSH list1 a b c
// (integer) 3
// redis> BLPOP list1 list2 0
// 1) "list1"
// 2) "a"
multiBulk, err := redis.MultiBulk(conn.Do("BLPOP", redisBroker.config.DefaultQueue, "0"))
if err != nil {
errorsChan <- err
return
}
item, err := redis.Bytes(multiBulk[1], nil)
if err != nil {
errorsChan <- err
return
}
log.Printf("Received new message: %s", item)
signature := signatures.TaskSignature{}
if err := json.Unmarshal(item, &signature); err != nil {
errorsChan <- err
return
}
if err := taskProcessor.Process(&signature); err != nil {
errorsChan <- err
return
}
}
}
}
// StartConsuming enters a loop and waits for incoming messages
func (redisBroker *RedisBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
redisBroker.stopChan = make(chan int)
redisBroker.quitChan = make(chan int)
redisBroker.wg.Add(1)
errorsChan := make(chan error)
if redisBroker.retryFunc == nil {
redisBroker.retryFunc = utils.RetryClosure()
}
go redisBroker.consume(errorsChan, taskProcessor)
for {
select {
case err := <-errorsChan:
return true, err // retry true
case <-redisBroker.stopChan:
return false, nil // retry false
}
}
}
// StartConsuming enters a loop and waits for incoming messages
func (redisBroker *RedisBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
if redisBroker.retryFunc == nil {
redisBroker.retryFunc = utils.RetryClosure()
}
redisBroker.pool = redisBroker.newPool()
defer redisBroker.pool.Close()
_, err := redisBroker.pool.Get().Do("PING")
if err != nil {
redisBroker.retryFunc()
return redisBroker.retry, err // retry true
}
redisBroker.retryFunc = utils.RetryClosure()
redisBroker.stopChan = make(chan int)
redisBroker.stopReceivingChan = make(chan int)
redisBroker.errorsChan = make(chan error)
deliveries := make(chan []byte)
redisBroker.wg.Add(1)
go func() {
defer redisBroker.wg.Done()
log.Print("[*] Waiting for messages. To exit press CTRL+C")
conn := redisBroker.pool.Get()
for {
select {
// A way to stop this goroutine from redisBroker.StopConsuming
case <-redisBroker.stopReceivingChan:
return
default:
itemBytes, err := conn.Do("BLPOP", redisBroker.config.DefaultQueue, "1")
if err != nil {
redisBroker.errorsChan <- err
return
}
// Unline BLPOP, LPOP is non blocking so nil means we can keep iterating
if itemBytes == nil {
continue
}
items, err := redis.ByteSlices(itemBytes, nil)
if err != nil {
redisBroker.errorsChan <- err
return
}
if len(items) != 2 {
log.Println("Got unexpected amount of byte arrays, ignoring")
continue
}
// items[0] - queue name (key), items[1] - value
item := items[1]
signature := signatures.TaskSignature{}
if err := json.Unmarshal(item, &signature); err != nil {
redisBroker.errorsChan <- err
return
}
// If the task is not registered, we requeue it,
// there might be different workers for processing specific tasks
if !redisBroker.IsTaskRegistered(signature.Name) {
_, err := conn.Do("RPUSH", redisBroker.config.DefaultQueue, item)
if err != nil {
redisBroker.errorsChan <- err
return
}
continue
}
deliveries <- item
}
}
}()
if err := redisBroker.consume(deliveries, taskProcessor); err != nil {
return redisBroker.retry, err // retry true
}
return redisBroker.retry, nil
}
// StartConsuming enters a loop and waits for incoming messages
func (redisBroker *RedisBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
if redisBroker.retryFunc == nil {
redisBroker.retryFunc = utils.RetryClosure()
}
redisBroker.pool = redisBroker.newPool()
defer redisBroker.pool.Close()
_, err := redisBroker.pool.Get().Do("PING")
if err != nil {
redisBroker.retryFunc()
return true, err // retry true
}
redisBroker.retryFunc = utils.RetryClosure()
redisBroker.stopChan = make(chan int)
redisBroker.stopReceivingChan = make(chan int)
redisBroker.errorsChan = make(chan error)
deliveries := make(chan []byte)
redisBroker.wg.Add(1)
go func() {
defer redisBroker.wg.Done()
log.Print("[*] Waiting for messages. To exit press CTRL+C")
conn := redisBroker.pool.Get()
for {
select {
// A way to stop this goroutine from redisBroker.StopConsuming
case <-redisBroker.stopReceivingChan:
return
default:
itemBytes, err := conn.Do("LPOP", redisBroker.config.DefaultQueue)
if err != nil {
redisBroker.errorsChan <- err
return
}
// Unline BLPOP, LPOP is non blocking so nil means we can keep iterating
if itemBytes == nil {
continue
}
item, err := redis.Bytes(itemBytes, nil)
if err != nil {
redisBroker.errorsChan <- err
return
}
deliveries <- item
}
}
}()
if err := redisBroker.consume(deliveries, taskProcessor); err != nil {
return true, err // retry true
}
return false, nil
}