/*
* WriteToRabbit connects to the rabbitMQ queue defined in the config
* (if it does not exit, it will error). Then it pushes to messages on that
* queue whenever it gets a new one on the jobs channel.
*/
func (conn *Connection) WriteToRabbit(queueName string, jobs <-chan interface{}) {
ch, err := conn.rabbitConnection.Channel()
rabbitbeans.FailOnError(err, "Failed to open a channel")
defer ch.Close() // Clean up by closing channel when function exits
q, err := ch.QueueInspect( // Make sure queue exists - don't create one otherwise and err.
queueName, // queue name
)
rabbitbeans.FailOnError(err, fmt.Sprintf("Failed to find queue named: %s", queueName))
log.Printf("Connected to queue: %s", q.Name)
if err := ch.Confirm(
false, // noWait = false - means, please do wait for confirms
); err != nil {
rabbitbeans.FailOnError(err, "Could not set channel confirm mode on")
}
// Buffer of 1 for our single outstanding publishing
confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1))
log.Printf(" [*] Sending rabbits. To exit press CTRL+C")
for j := range jobs {
job, ok := j.(rabbitbeans.Job)
if !ok {
rabbitbeans.LogOnError(errors.New("Unreadable job on the channel"), "Skipping item")
continue
}
if !conn.config.Quiet {
log.Printf("Sending rabbit to queue: %s", string(job.Body))
}
err = ch.Publish(
"", // exchange
queueName, // routing key
false, // mandatory
false, // immediate
amqp.Publishing{
ContentType: job.ContentType,
Body: job.Body,
})
if err != nil {
job.Nack(job.Id)
} else {
// only ack the source delivery when the destination acks the publishing
if confirmed := <-confirms; confirmed.Ack {
job.Ack(job.Id)
} else {
job.Nack(job.Id)
}
rabbitbeans.FailOnError(err, fmt.Sprintf("Failed to find queue named: %s", queueName))
}
}
}
/*
* WriteToBeanstalkd puts jobs onto beanstalkd
*/
func (conn *Connection) WriteToBeanstalkd(jobs <-chan interface{}) {
log.Printf(" [*] Publishing beans. To exit press CTRL+C")
for j := range jobs {
job, ok := j.(rabbitbeans.Job)
if !ok {
rabbitbeans.FailOnError(errors.New("Unknown message on channel"), "Can't put message on rabbit")
}
if !conn.config.Quiet {
log.Printf("Received a bean to create: %s", job.Body)
}
id, err := conn.beansConnection.Put(
job.Body, //body
job.Priority, //pri uint32
job.Delay, //delay
job.TTR, // TTR time to run -- is an integer number of seconds to allow a worker to run this job
)
rabbitbeans.LogOnError(err, fmt.Sprintf("Failed to put job on beanstalkd %s", job.Body))
if err != nil {
job.Nack(id)
} else {
job.Ack(id)
}
if !conn.config.Quiet {
fmt.Println("Created job", id)
}
}
}
/*
* Creates a new Pipe that has access to Redis.
*/
func NewRedisPipe(redisPool pool.Pool) rabbitbeans.Pipe {
client, err := redisPool.Get()
if err != nil {
// handle error
rabbitbeans.FailOnError(err, "Cannot get redis connection from pool")
}
return RedisPipe{client}
}
func (me *TestBeanHandler) WriteToBeanstalkd(c <-chan interface{}) {
for n := 0; n < me.NumToProduce; n++ {
m := <-c
msg, ok := m.(rabbitbeans.Job)
if !ok {
rabbitbeans.FailOnError(errors.New("Unknown message on channel"), "Can't put job on beanstalkd")
}
msg.Ack(0)
}
}
/*
* ReadFromRabbit connects to the rabbitMQ queue defined in the config
* (if it does not exit, it will error). Then it listens to messages on that
* queue and redirects then to the jobs channnel
*/
func (conn *Connection) ReadFromRabbit(queueName string, jobs chan<- interface{}) {
ch, err := conn.rabbitConnection.Channel()
rabbitbeans.FailOnError(err, "Failed to open a channel")
defer ch.Close() // Clean up by closing channel when function exits
q, err := ch.QueueInspect( // Make sure queue exists - don't create one otherwise and err.
queueName, // queue name
)
rabbitbeans.FailOnError(err, fmt.Sprintf("Failed to find queue named: %s", queueName))
log.Printf("Connected to queue: %s", queueName)
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
rabbitbeans.FailOnError(err, "Failed to register a consumer")
log.Printf(" [*] Waiting for rabbits. To exit press CTRL+C")
for d := range msgs {
if !conn.config.Quiet {
log.Printf("Received a rabbit from queue: %s", d.Body)
}
job := &rabbitbeans.Job{
0,
d.Body,
// d.Ack,
// d.Nack,
RabbitAcknowledger{d},
d.ContentType,
uint32(d.Priority),
0,
time.Minute * 50, // TTR 50 minutes
"",
}
jobs <- *job
}
}
func (me *TestRabbitHandler) WriteToRabbit(c <-chan interface{}) {
for n := 0; n < me.NumToProduce; n++ {
j := <-c
job, ok := j.(rabbitbeans.Job)
if !ok {
rabbitbeans.FailOnError(errors.New("Unknown message on channel"), "Can't put message on rabbit")
}
job.Ack(job.Id)
}
}
/*
* NewRedisPool creates a new pool of redis connections
*/
func NewRedisPool(config RedisConfig) *pool.Pool {
initConfig(&config)
log.Printf("Creating Redis pool with config: %v", config)
p, err := pool.New(protocol, fmt.Sprintf("%s:%s", config.Host, config.Port), config.PoolSize)
if err != nil {
// fatal error
rabbitbeans.FailOnError(err, "Cannot create redis pool")
}
return p
}
/*
* Dial connects to an amqp URL where it expects a rabbitMQ instance to be running.
* Returns a multiplexable connection that can then be used to produce/consume on different queues
*/
func Dial(config Config) RabbitHandler {
if config.AmqpUrl == "" {
config.AmqpUrl = LocalhostAmqpUrl
}
conn, err := amqp.DialConfig(config.AmqpUrl, config.AmqpConfig)
rabbitbeans.FailOnError(err, "Failed to connect to RabbitMQ")
return &Connection{
config,
conn,
}
}
/*
* ReadFromBeanstalkd reads jobs off of beanstalkd.
*/
func (conn *Connection) ReadFromBeanstalkd(jobs chan<- interface{}) {
log.Printf(" [*] Consuming beans. To exit press CTRL+C")
ticker := time.NewTicker(time.Duration(conn.config.TickInterval) * time.Second)
go func() {
for {
select {
case <-ticker.C:
log.Printf("Polling beanstalkd for beans")
var i = 0
for {
id, body, err := conn.beansConnection.Reserve(5 * time.Second)
if cerr, ok := err.(beanstalk.ConnError); !ok {
rabbitbeans.FailOnError(err, "expected connError")
} else if cerr.Err != beanstalk.ErrTimeout {
rabbitbeans.LogOnError(err, fmt.Sprintf("expected timeout on reserve %d", id))
// Means the job deadline is real soon!! Reserve job anyways
} else {
break
}
if !conn.config.Quiet {
log.Printf("Reserved job %v %s", id, body)
}
jobs <- rabbitbeans.Job{
id,
body,
conn,
"application/json",
0,
0,
0,
"",
}
i++
if i == conn.config.BatchSize {
break
}
}
log.Printf("Processed %d jobs this tick", i)
}
}
}()
}
/*
* Dial connects to a beanstalkd instance.
* Returns a multiplexable connection that can then be used to put/reserve jobs.
*/
func Dial(config Config) BeanHandler {
if config.Host == "" {
config.Host = defaultHost
}
if config.Port == "" {
config.Port = defaultPort
}
if config.TickInterval == 0 {
config.TickInterval = defaultTickInterval
}
if config.BatchSize == 0 {
config.BatchSize = defaultBatchSize
}
var connString = fmt.Sprintf("%s:%s", config.Host, config.Port)
conn, err := beanstalk.Dial(protocol, connString)
rabbitbeans.FailOnError(err, fmt.Sprintf("Failed to connect to Beanstalkd at: %s", connString))
return &Connection{
config,
conn,
}
}