// Connects to the message queue, opens a channel, declares a queue
func (amqpBroker *AMQPBroker) open() (*amqp.Connection, *amqp.Channel, amqp.Queue, <-chan amqp.Confirmation, error) {
var err error
var channel *amqp.Channel
var queue amqp.Queue
if conn == nil {
once.Do(amqpBroker.connect)
}
if conn == nil {
return conn, channel, queue, nil, fmt.Errorf("Can't connect to the server")
}
channel, err = conn.Channel()
if err != nil {
fmt.Printf("Channel: %s\n", err)
}
if err := channel.ExchangeDeclare(
amqpBroker.config.Exchange, // name of the exchange
amqpBroker.config.ExchangeType, // type
true, // durable
false, // delete when complete
false, // internal
false, // noWait
nil, // arguments
); err != nil {
fmt.Printf("Exchange: %s\n", err)
}
queue, err = channel.QueueDeclare(
amqpBroker.config.DefaultQueue, // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
fmt.Printf("Queue Declare: %s\n", err)
}
if err := channel.QueueBind(
queue.Name, // name of the queue
amqpBroker.config.BindingKey, // binding key
amqpBroker.config.Exchange, // source exchange
false, // noWait
nil, // arguments
); err != nil {
fmt.Printf("Queue Bind: %s\n", err)
}
confirmsChan := make(chan amqp.Confirmation, 1)
// Enable publish confirmations
if err := channel.Confirm(false); err != nil {
close(confirmsChan)
fmt.Printf("Channel could not be put into confirm mode: %s\n", err)
}
return conn, channel, queue, channel.NotifyPublish(confirmsChan), nil
}
// redial continually connects to the URL, exiting the program when no longer possible
func redial(ctx context.Context, url, exchange string) chan chan session {
sessions := make(chan chan session)
go func() {
sess := make(chan session)
defer close(sessions)
for {
select {
case sessions <- sess:
case <-ctx.Done():
log.Info("shutting down session factory")
return
}
connected := false
var conn *amqp.Connection
var ch *amqp.Channel
var err error
for !connected {
log.Debug("dialing amqp url: %s", url)
conn, err = amqp.Dial(url)
if err != nil {
log.Error(3, "cannot (re)dial: %v: %q", err, url)
time.Sleep(time.Second)
continue
}
log.Debug("connected to %s", url)
log.Debug("creating new channel on AMQP connection.")
ch, err = conn.Channel()
if err != nil {
log.Error(3, "cannot create channel: %v", err)
conn.Close()
time.Sleep(time.Second)
continue
}
log.Debug("Ensuring that %s topic exchange exists on AMQP server.", exchange)
if err := ch.ExchangeDeclare(exchange, "topic", true, false, false, false, nil); err != nil {
log.Error(3, "cannot declare topic exchange: %v", err)
conn.Close()
time.Sleep(time.Second)
}
log.Debug("Successfully connected to RabbitMQ.")
connected = true
}
select {
case sess <- session{conn, ch}:
case <-ctx.Done():
log.Info("shutting down new session")
return
}
}
}()
return sessions
}
func Exchange(exchange string, ch *amqp.Channel) error {
return ch.ExchangeDeclare(
exchange, // name
"fanout", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
}
// DeclareExc decleares an exchange with false auto-delete, and false internal flags.
func (r Rabbit) DeclareExc(ch *amqp.Channel) error {
err := ch.ExchangeDeclare(
r.Exchange, // name
r.ExchangeType, // type
r.Durable, // durable
r.Delete, // auto-deleted
r.Internal, // internal
r.NoWait, // no-wait
r.Arguments, // arguments
)
return err
}
// Connects to the message queue, opens a channel, declares a queue
func open(cnf *config.Config) (*amqp.Connection, *amqp.Channel, amqp.Queue, error) {
var conn *amqp.Connection
var channel *amqp.Channel
var queue amqp.Queue
var err error
conn, err = amqp.Dial(cnf.Broker)
if err != nil {
return conn, channel, queue, fmt.Errorf("Dial: %s", err)
}
channel, err = conn.Channel()
if err != nil {
return conn, channel, queue, fmt.Errorf("Channel: %s", err)
}
if err := channel.ExchangeDeclare(
cnf.Exchange, // name of the exchange
cnf.ExchangeType, // type
true, // durable
false, // delete when complete
false, // internal
false, // noWait
nil, // arguments
); err != nil {
return conn, channel, queue, fmt.Errorf("Exchange: %s", err)
}
queue, err = channel.QueueDeclare(
cnf.DefaultQueue, // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
return conn, channel, queue, fmt.Errorf("Queue Declare: %s", err)
}
if err := channel.QueueBind(
queue.Name, // name of the queue
cnf.BindingKey, // binding key
cnf.Exchange, // source exchange
false, // noWait
nil, // arguments
); err != nil {
return conn, channel, queue, fmt.Errorf("Queue Bind: %s", err)
}
return conn, channel, queue, nil
}
func declareExchange(channel *amqp.Channel) {
log.Printf("PUBLISHER: got Channel, declaring %q Exchange (%q)", "fanout", "changes")
if err := channel.ExchangeDeclare(
"changes", // name
"fanout", // type
true, // durable
true, // auto-deleted
false, // internal
false, // noWait
nil, // arguments
); err != nil {
log.Fatalf("Exchange Declare: %s", err)
}
log.Printf("PUBLISHER: declared Exchange")
}
func createQueue(channel *amqp.Channel, queueName string) {
err := channel.ExchangeDeclare(
queueName, // name of the exchange
"direct", // type
true, // durable
false, // delete when complete
false, // internal
false, // noWait
nil, // arguments
)
if err != nil {
log.Fatal("Couldn't declare exchange: %s", err)
}
queue, err := channel.QueueDeclare(
queueName, // name of the queue
true, // durable
false, // delete when usused
false, // exclusive
false, // noWait
nil, // arguments
)
if err != nil {
log.Fatal("Couldn't declare queue: %s", err)
}
err = channel.QueueBind(
queue.Name, // name of the queue
queueName, // bindingKey
queueName, // sourceExchange
false, // noWait
nil, // arguments
)
if err != nil {
log.Fatal("Couldn't bind queue: %s", err)
}
}
// connects to an AMQP topic and emits each message into streamtools.
func (b *FromAMQP) Run() {
var err error
var conn *amqp.Connection
var amqp_chan *amqp.Channel
toOut := make(blocks.MsgChan)
toError := make(chan error)
host := "localhost"
port := "5672"
username := "******"
password := "******"
routingkey := "#"
exchange := "amq.topic"
exchange_type := "topic"
for {
select {
case msg := <-toOut:
b.out <- msg
case err := <-toError:
b.Error(err)
case ruleI := <-b.inrule:
rule := ruleI.(map[string]interface{})
routingkey, err = util.ParseString(rule, "RoutingKey")
if err != nil {
b.Error(err)
continue
}
exchange, err = util.ParseString(rule, "Exchange")
if err != nil {
b.Error(err)
continue
}
exchange_type, err = util.ParseString(rule, "ExchangeType")
if err != nil {
b.Error(err)
continue
}
host, err = util.ParseString(rule, "Host")
if err != nil {
b.Error(err)
continue
}
port, err = util.ParseString(rule, "Port")
if err != nil {
b.Error(err)
continue
}
username, err = util.ParseString(rule, "Username")
if err != nil {
b.Error(err)
continue
}
password, err = util.ParseString(rule, "Password")
if err != nil {
b.Error(err)
continue
}
conn, err = amqp.Dial("amqp://" + username + ":" + password + "@" + host + ":" + port + "/")
if err != nil {
b.Error(err)
continue
}
amqp_chan, err = conn.Channel()
if err != nil {
b.Error(err)
continue
}
err = amqp_chan.ExchangeDeclare(
exchange, // name
exchange_type, // type
true, // durable
false, // auto-deleted
false, // internal
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
queue, err := amqp_chan.QueueDeclare(
"", // name
false, // durable
true, // delete when unused
false, // exclusive
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
err = amqp_chan.QueueBind(
queue.Name, // queue name
routingkey, // routing key
exchange, // exchange
false,
nil,
)
if err != nil {
b.Error(err)
continue
}
deliveries, err := amqp_chan.Consume(
queue.Name, // name
"", // consumerTag
true, // noAck
false, // exclusive
false, // noLocal
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
h := readWriteAMQPHandler{toOut, toError}
go h.handle(deliveries)
case <-b.quit:
if amqp_chan != nil {
amqp_chan.Close()
}
if conn != nil {
conn.Close()
}
return
case c := <-b.queryrule:
c <- map[string]interface{}{
"Host": host,
"Port": port,
"Username": username,
"Password": password,
"Exchange": exchange,
"ExchangeType": exchange_type,
"RoutingKey": routingkey,
}
}
}
}
// Connects to the message queue, opens a channel, declares a queue
func (amqpBackend *AMQPBackend) open(taskUUID string) (*amqp.Connection, *amqp.Channel, amqp.Queue, <-chan amqp.Confirmation, error) {
var (
conn *amqp.Connection
channel *amqp.Channel
queue amqp.Queue
err error
)
// Connect
// From amqp docs: DialTLS will use the provided tls.Config when it encounters an amqps:// scheme
// and will dial a plain connection when it encounters an amqp:// scheme.
conn, err = amqp.DialTLS(amqpBackend.config.Broker, amqpBackend.config.TLSConfig)
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Dial: %s", err)
}
// Open a channel
channel, err = conn.Channel()
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Channel: %s", err)
}
// Declare an exchange
err = channel.ExchangeDeclare(
amqpBackend.config.Exchange, // name of the exchange
amqpBackend.config.ExchangeType, // type
true, // durable
false, // delete when complete
false, // internal
false, // noWait
nil, // arguments
)
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Exchange Declare: %s", err)
}
// Declare a queue
arguments := amqp.Table{
"x-message-ttl": int32(amqpBackend.getExpiresIn()),
}
queue, err = channel.QueueDeclare(
taskUUID, // name
false, // durable
true, // delete when unused
false, // exclusive
false, // no-wait
arguments,
)
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Queue Declare: %s", err)
}
// Bind the queue
if err := channel.QueueBind(
queue.Name, // name of the queue
taskUUID, // binding key
amqpBackend.config.Exchange, // source exchange
false, // noWait
nil, // arguments
); err != nil {
return conn, channel, queue, nil, fmt.Errorf("Queue Bind: %s", err)
}
// Enable publish confirmations
if err := channel.Confirm(false); err != nil {
return conn, channel, queue, nil, fmt.Errorf("Channel could not be put into confirm mode: %s", err)
}
return conn, channel, queue, channel.NotifyPublish(make(chan amqp.Confirmation, 1)), nil
}
// Connects to the message queue, opens a channel, declares a queue
func (amqpBroker *AMQPBroker) open() (*amqp.Connection, *amqp.Channel, amqp.Queue, <-chan amqp.Confirmation, error) {
var (
conn *amqp.Connection
channel *amqp.Channel
queue amqp.Queue
err error
)
// Connect
conn, err = amqp.Dial(amqpBroker.config.Broker)
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Dial: %s", err)
}
// Open a channel
channel, err = conn.Channel()
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Channel: %s", err)
}
// Declare an exchange
if err := channel.ExchangeDeclare(
amqpBroker.config.Exchange, // name of the exchange
amqpBroker.config.ExchangeType, // type
true, // durable
false, // delete when complete
false, // internal
false, // noWait
nil, // arguments
); err != nil {
return conn, channel, queue, nil, fmt.Errorf("Exchange Declare: %s", err)
}
// Declare a queue
queue, err = channel.QueueDeclare(
amqpBroker.config.DefaultQueue, // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Queue Declare: %s", err)
}
// Bind the queue
if err := channel.QueueBind(
queue.Name, // name of the queue
amqpBroker.config.BindingKey, // binding key
amqpBroker.config.Exchange, // source exchange
false, // noWait
nil, // arguments
); err != nil {
return conn, channel, queue, nil, fmt.Errorf("Queue Bind: %s", err)
}
// Enable publish confirmations
if err := channel.Confirm(false); err != nil {
return conn, channel, queue, nil, fmt.Errorf("Channel could not be put into confirm mode: %s", err)
}
return conn, channel, queue, channel.NotifyPublish(make(chan amqp.Confirmation, 1)), nil
}
func createRabbitMqExchange(channel *amqp.Channel, lineConf LineConfig) {
err := channel.ExchangeDeclare(lineConf.Queue, "topic", true, false, false, false, nil)
panicOnError(err, "Could not create exchange")
}
// connects to an AMQP topic and emits each message into streamtools.
func (b *ToAMQP) Run() {
var err error
var conn *amqp.Connection
var amqp_chan *amqp.Channel
host := "localhost"
port := "5672"
username := "******"
password := "******"
routingkey := "streamtools"
exchange := "amq.topic"
exchange_type := "topic"
for {
select {
case ruleI := <-b.inrule:
routingkey, err = util.ParseString(ruleI, "RoutingKey")
if err != nil {
b.Error(err)
continue
}
exchange, err = util.ParseString(ruleI, "Exchange")
if err != nil {
b.Error(err)
continue
}
exchange_type, err = util.ParseString(ruleI, "ExchangeType")
if err != nil {
b.Error(err)
continue
}
host, err = util.ParseString(ruleI, "Host")
if err != nil {
b.Error(err)
continue
}
port, err = util.ParseString(ruleI, "Port")
if err != nil {
b.Error(err)
continue
}
username, err = util.ParseString(ruleI, "Username")
if err != nil {
b.Error(err)
continue
}
password, err = util.ParseString(ruleI, "Password")
if err != nil {
b.Error(err)
continue
}
conn, err = amqp.Dial("amqp://" + username + ":" + password + "@" + host + ":" + port + "/")
if err != nil {
b.Error(err)
continue
}
amqp_chan, err = conn.Channel()
if err != nil {
b.Error(err)
continue
}
err = amqp_chan.ExchangeDeclare(
exchange, // name
exchange_type, // type
true, // durable
false, // auto-deleted
false, // internal
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
case msg := <-b.in:
if conn == nil || amqp_chan == nil {
continue
}
msgBytes, err := json.Marshal(msg)
// Make the output JSON if msg wasn't JSON already
if err != nil {
json_msg := map[string]interface{}{
"data": msg,
}
msgBytes, err = json.Marshal(json_msg)
if err != nil {
b.Error(err)
continue
}
}
if len(msgBytes) == 0 {
b.Error("Zero byte length message")
continue
}
err = amqp_chan.Publish(
exchange,
routingkey,
false,
false,
amqp.Publishing{
Headers: amqp.Table{},
ContentType: "text/plain",
ContentEncoding: "",
Body: msgBytes,
DeliveryMode: amqp.Transient,
Priority: 0,
},
)
if err != nil {
b.Error(err)
continue
}
case <-b.quit:
if amqp_chan != nil {
amqp_chan.Close()
}
if conn != nil {
conn.Close()
}
return
case c := <-b.queryrule:
c <- map[string]interface{}{
"Host": host,
"Port": port,
"Username": username,
"Password": password,
"Exchange": exchange,
"ExchangeType": exchange_type,
"RoutingKey": routingkey,
}
}
}
}
// Connects to the message queue, opens a channel, declares a queue
func open(taskUUID string, cnf *config.Config) (*amqp.Connection, *amqp.Channel, amqp.Queue, error) {
var conn *amqp.Connection
var channel *amqp.Channel
var queue amqp.Queue
var err error
conn, err = amqp.Dial(cnf.Broker)
if err != nil {
return conn, channel, queue, fmt.Errorf("Dial: %s", err)
}
channel, err = conn.Channel()
if err != nil {
return conn, channel, queue, fmt.Errorf("Channel: %s", err)
}
err = channel.ExchangeDeclare(
cnf.Exchange, // name of the exchange
cnf.ExchangeType, // type
true, // durable
false, // delete when complete
false, // internal
false, // noWait
nil, // arguments
)
if err != nil {
return conn, channel, queue, fmt.Errorf("Exchange: %s", err)
}
resultsExpireIn := cnf.ResultsExpireIn * 1000
if resultsExpireIn == 0 {
// // expire results after 1 hour by default
resultsExpireIn = 3600 * 1000
}
arguments := amqp.Table{
"x-message-ttl": int32(resultsExpireIn),
}
queue, err = channel.QueueDeclare(
taskUUID, // name
false, // durable
true, // delete when unused
false, // exclusive
false, // no-wait
arguments,
)
if err != nil {
return conn, channel, queue, fmt.Errorf("Queue Declare: %s", err)
}
if err := channel.QueueBind(
queue.Name, // name of the queue
taskUUID, // binding key
cnf.Exchange, // source exchange
false, // noWait
nil, // arguments
); err != nil {
return conn, channel, queue, fmt.Errorf("Queue Bind: %s", err)
}
return conn, channel, queue, nil
}
// Connects to the message queue, opens a channel, declares a queue
func (amqpBackend *AMQPBackend) open(taskUUID string) (*amqp.Connection, *amqp.Channel, amqp.Queue, <-chan amqp.Confirmation, error) {
var conn *amqp.Connection
var channel *amqp.Channel
var queue amqp.Queue
var err error
conn, err = amqp.Dial(amqpBackend.config.ResultBackend)
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Dial: %s", err)
}
channel, err = conn.Channel()
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Channel: %s", err)
}
err = channel.ExchangeDeclare(
amqpBackend.config.Exchange, // name of the exchange
amqpBackend.config.ExchangeType, // type
true, // durable
false, // delete when complete
false, // internal
false, // noWait
nil, // arguments
)
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Exchange: %s", err)
}
arguments := amqp.Table{
"x-message-ttl": int32(amqpBackend.getExpiresIn()),
}
queue, err = channel.QueueDeclare(
taskUUID, // name
false, // durable
true, // delete when unused
false, // exclusive
false, // no-wait
arguments,
)
if err != nil {
return conn, channel, queue, nil, fmt.Errorf("Queue Declare: %s", err)
}
if err := channel.QueueBind(
queue.Name, // name of the queue
taskUUID, // binding key
amqpBackend.config.Exchange, // source exchange
false, // noWait
nil, // arguments
); err != nil {
return conn, channel, queue, nil, fmt.Errorf("Queue Bind: %s", err)
}
confirmsChan := make(chan amqp.Confirmation, 1)
// Enable publish confirmations
if err := channel.Confirm(false); err != nil {
close(confirmsChan)
return conn, channel, queue, nil, fmt.Errorf("Channel could not be put into confirm mode: %s", err)
}
return conn, channel, queue, channel.NotifyPublish(confirmsChan), nil
}
func NewConsumer(consumerName string, queueName string, exchange Exchange, config *Config) (consumer *Consumer, err error) {
if consumerName == "" {
err = errors.New("Must give the consumer a name")
return
}
if queueName == "" {
err = errors.New("Must give the queue name.")
return
}
if len(queueName) > 255 {
err = errors.New("Must give a queue name that contains 1-255 characters.")
return
}
if err = exchange.Validate(); err != nil {
return
}
if config == nil {
config = NewConfig()
}
conn, err := amqp.Dial(config.GetConnectionString())
if err != nil {
return
}
//setup the channel
var channel *amqp.Channel
if channel, err = conn.Channel(); err != nil {
return
}
//setup the exchange
if err = channel.ExchangeDeclare(
exchange.Name, //exchange name
exchange.Type, //exchange type
true, //durable
false, //remove when complete
false, //internal
false, //noWait
nil, //arguments
); err != nil {
return
}
//setup the queue
var queue amqp.Queue
if queue, err = channel.QueueDeclare(
queueName, // name of the queue
true, // durable
false, // delete when usused
false, // exclusive
false, // noWait
nil, // arguments
); err != nil {
return
}
if err = channel.QueueBind(
queue.Name, //queue name
exchange.RoutingKey, //routing key ("binding key")
exchange.Name, //exchange (source)
false, //noWait
nil, //arguments
); err != nil {
return
}
//setup the deliverables
var messages <-chan amqp.Delivery
if messages, err = channel.Consume(
queue.Name, //queue name
consumerName, //consumer name
false, //auto acknowledge
false, //exclusive
false, //not local
false, //no wait
nil, //arguments
); err != nil {
return
}
consumer = new(Consumer)
consumer.Name = consumerName
consumer.config = config
consumer.conn = conn
consumer.channel = channel
consumer.Exchange = exchange
consumer.incomingMessages = messages
return
}
func (d *Topic) Consume(ch *amqp.Channel, cfg config.EndpointConfig) (<-chan amqp.Delivery, error) {
queueName, ok := cfg.QueueConfig["queuename"].(string)
if !ok {
return nil, fmt.Errorf("unable to parse queuename from config")
}
topicName, ok := cfg.QueueConfig["topicname"].(string)
if !ok {
return nil, fmt.Errorf("unable to parse topicname from config")
}
exchangeName, ok := cfg.QueueConfig["exchangename"].(string)
if !ok {
return nil, fmt.Errorf("unable to parse exchangename from config")
}
prefetch, ok := cfg.QueueConfig["prefetch"].(int)
if !ok || prefetch < 1 {
prefetch = 1
}
err := ch.ExchangeDeclare(
exchangeName, // name
"topic", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
return nil, err
}
q, err := ch.QueueDeclare(
queueName, // name
true, // durable
false, // delete when usused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
return nil, err
}
err = ch.Qos(
prefetch, // prefetch count
0, // prefetch size
false, // global
)
if err != nil {
return nil, err
}
err = ch.QueueBind(
q.Name, // queue name
topicName, // routing key
exchangeName, // exchange
false,
nil,
)
if err != nil {
return nil, err
}
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
return nil, err
}
return msgs, nil
}
