func (q *AMQP) Connect() error {
q.Lock()
defer q.Unlock()
q.headers = amqp.Table{
"precision": q.Precision,
"database": q.Database,
"retention_policy": q.RetentionPolicy,
}
var connection *amqp.Connection
// make new tls config
tls, err := internal.GetTLSConfig(
q.SSLCert, q.SSLKey, q.SSLCA, q.InsecureSkipVerify)
if err != nil {
return err
}
if tls != nil {
connection, err = amqp.DialTLS(q.URL, tls)
} else {
connection, err = amqp.Dial(q.URL)
}
if err != nil {
return err
}
channel, err := connection.Channel()
if err != nil {
return fmt.Errorf("Failed to open a channel: %s", err)
}
err = channel.ExchangeDeclare(
q.Exchange, // name
"topic", // type
true, // durable
false, // delete when unused
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
return fmt.Errorf("Failed to declare an exchange: %s", err)
}
q.channel = channel
go func() {
log.Printf("Closing: %s", <-connection.NotifyClose(make(chan *amqp.Error)))
log.Printf("Trying to reconnect")
for err := q.Connect(); err != nil; err = q.Connect() {
log.Println(err)
time.Sleep(10 * time.Second)
}
}()
return nil
}
// openChannel opens up a RabbitMQ channel.
func openChannel(conn *amqp.Connection) *amqp.Channel {
ch, err := conn.Channel()
if err != nil {
log.Fatalf("Failed to open a channel: %s", err)
}
return ch
}
func (server *Server) ServeConn(conn *amqp.Connection, queue string) {
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
q, err := ch.QueueDeclare(
queue, // name
false, // durable
false, // delete when usused
false, // exclusive
false, // noWait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // autoAck
false, // exclusive
false, // noLocal
false, // noWait
nil, // arguments
)
failOnError(err, "Failed to register a consumer")
codec := &serverCodec{
conn: conn,
ch: ch,
msgs: msgs,
pending: make(map[uint64]prop),
}
server.s.ServeCodec(codec)
}
/*
If ch is nil, a new Connection and Channel will be created, and this publisher
will 'own' the connection. A call to close() will close both channel and connection.
If ch is provided, then this publisher will reuse the channel and
calls to close() will do nothing.
*/
func newPublisher(serverURI string, cfg *ChannelConfig, ch *amqp.Channel) *Publisher {
var conn *amqp.Connection
var err error
if ch == nil {
conn, err = amqp.Dial(serverURI)
if err != nil {
panic(fmt.Errorf("Failed to connect to RabbitMQ: %v", err))
}
ch, err = conn.Channel()
if err != nil {
panic(fmt.Errorf("Failed to open a channel: %v", err))
}
}
_, err = ch.QueueDeclare(*cfg.Name, *cfg.Durable, *cfg.AutoDelete, *cfg.Exclusive, false, *cfg.Args)
if err != nil {
panic(fmt.Errorf("Failed to declare queue %s: %v", cfg.Name, err))
}
ch.QueuePurge(*cfg.Name, true)
return &Publisher{exch: "", routingKey: *cfg.Name, conn: conn, ch: ch, typeTag: *cfg.TypeTag}
}
func Consume(connection *amqp.Connection, queue string, outChannel chan *protobuf.Notification) {
// create a channel on this connection
channel, err := connection.Channel()
if err != nil {
panic(err)
}
defer channel.Close()
// start consuming data
consumerTag := queue + "-consumer"
deliveries, err := channel.Consume(
queue, // name
consumerTag, // consumerTag
false, // noAck
false, // exclusive
false, // noLocal
false, // noWait
nil, // arguments
)
if err != nil {
panic(err)
}
defer channel.Cancel(consumerTag, false)
for delivery := range deliveries {
notif := &protobuf.Notification{}
proto.Unmarshal(delivery.Body, notif)
outChannel <- notif
if err := delivery.Ack(false); err != nil {
panic(err)
}
}
}
func CreateHelloTopology(connection *amqp.Connection) (*amqp.Channel, error) {
channel, err := connection.Channel()
if err != nil {
log.Println("Failed to get channel!: ", err)
return nil, err
}
err = channel.ExchangeDeclare("hello-exchange", "direct", true, false, false, false, nil)
if err != nil {
log.Println("Failed to declare exchange!: ", err)
return nil, err
}
_, err = channel.QueueDeclare("hello-queue", false, false, false, false, nil)
if err != nil {
log.Println("Failed to declare queue!: ", err)
return nil, err
}
err = channel.QueueBind("hello-queue", "hola", "hello-exchange", false, nil)
if err != nil {
log.Println("Failed to bind to queue!: ", err)
return nil, err
}
return channel, nil
}
//NewServerCodec returns a new rpc.ClientCodec using AMQP on conn. serverRouting is the routing
//key with with RPC calls are received, encodingCodec is an EncodingCoding implementation. This package provdes JSONCodec and GobCodec for the JSON and Gob encodings respectively.
func NewServerCodec(conn *amqp.Connection, serverRouting string, encodingCodec EncodingCodec) (rpc.ServerCodec, error) {
channel, err := conn.Channel()
if err != nil {
return nil, err
}
queue, err := channel.QueueDeclare(serverRouting, false, false, false, false, nil)
if err != nil {
return nil, err
}
messages, err := channel.Consume(queue.Name, "", true, false, false, false, nil)
if err != nil {
return nil, err
}
server := &serverCodec{
codec: &codec{
conn: conn,
channel: channel,
routing: queue.Name,
codec: encodingCodec,
message: messages,
},
lock: new(sync.RWMutex),
calls: make(map[uint64]route),
seq: 0,
}
return server, err
}
func newConsumeChannel(conn *amqp.Connection, queueName string) (<-chan amqp.Delivery, error) {
ch, err := conn.Channel()
if err != nil {
return nil, err
}
return ch.Consume(queueName, "", false, false, false, false, nil)
}
//NewClientCodec returns a new rpc.ClientCodec using AMQP on conn. serverRouting is the routing
//key with with RPC calls are sent, it should be the same routing key used with NewServerCodec.
//encodingCodec is an EncodingCoding implementation. This package provdes JSONCodec and GobCodec
//for the JSON and Gob encodings respectively.
func NewClientCodec(conn *amqp.Connection, serverRouting string, encodingCodec EncodingCodec) (rpc.ClientCodec, error) {
channel, err := conn.Channel()
if err != nil {
return nil, err
}
queue, err := channel.QueueDeclare("", false, true, false, false, nil)
if err != nil {
return nil, err
}
serverQueue, err := channel.QueueDeclare(serverRouting, false, false, false, false, nil)
if err != nil {
return nil, err
}
if serverQueue.Consumers == 0 {
return nil, ErrNoConsumers
}
message, err := channel.Consume(queue.Name, "", true, false, false, false, nil)
client := &clientCodec{
codec: &codec{
conn: conn,
channel: channel,
routing: queue.Name,
codec: encodingCodec,
message: message,
},
serverRouting: serverRouting,
}
return client, err
}
func connect(conn *amqp.Connection, exchange string, wg *sync.WaitGroup) {
ch, err := conn.Channel()
utils.FailOnError(err, "Failed to open a channel")
defer conn.Close()
err = ch.ExchangeDeclare(
exchange, // name
"direct", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
utils.FailOnError(err, "Failed to declare an exchange")
keys := []string{"create", "update"}
for {
for _, key := range keys {
go connectKey(ch, key, exchange)
}
log.Printf("Break \n \n")
time.Sleep(40 * time.Second)
}
}
func Log(connection *amqp.Connection, inChannel chan *protobuf.Log) {
// create a channel on this connection
channel, err := connection.Channel()
if err != nil {
panic(err)
}
defer channel.Close()
for log := range inChannel {
body, err := proto.Marshal(log)
if err != nil {
panic(err)
}
err = channel.Publish(
exchange, // publish to an exchange
routingKey, // routing to 0 or more queues
false, // mandatory
false, // immediate
amqp.Publishing{
Headers: amqp.Table{},
ContentType: "",
ContentEncoding: "",
Body: body,
DeliveryMode: amqp.Transient, // 1=non-persistent, 2=persistent
Priority: 0, // 0-9
// a bunch of application/implementation-specific fields
},
)
if err != nil {
panic(err)
}
}
}
// redial continually connects to the URL, exiting the program when no longer possible
func redial(ctx context.Context, url, exchange string) chan chan session {
sessions := make(chan chan session)
go func() {
sess := make(chan session)
defer close(sessions)
for {
select {
case sessions <- sess:
case <-ctx.Done():
log.Info("shutting down session factory")
return
}
connected := false
var conn *amqp.Connection
var ch *amqp.Channel
var err error
for !connected {
log.Debug("dialing amqp url: %s", url)
conn, err = amqp.Dial(url)
if err != nil {
log.Error(3, "cannot (re)dial: %v: %q", err, url)
time.Sleep(time.Second)
continue
}
log.Debug("connected to %s", url)
log.Debug("creating new channel on AMQP connection.")
ch, err = conn.Channel()
if err != nil {
log.Error(3, "cannot create channel: %v", err)
conn.Close()
time.Sleep(time.Second)
continue
}
log.Debug("Ensuring that %s topic exchange exists on AMQP server.", exchange)
if err := ch.ExchangeDeclare(exchange, "topic", true, false, false, false, nil); err != nil {
log.Error(3, "cannot declare topic exchange: %v", err)
conn.Close()
time.Sleep(time.Second)
}
log.Debug("Successfully connected to RabbitMQ.")
connected = true
}
select {
case sess <- session{conn, ch}:
case <-ctx.Done():
log.Info("shutting down new session")
return
}
}
}()
return sessions
}
/**
* Retrieves a channel from the given connection, ConnectionDefinition
*/
func OpenChannel(connection *amqp.Connection, cd ConnectionDefinition) error {
channel, err := connection.Channel()
if err != nil {
return err
}
Channel[cd.Vhost] = channel
return nil
}
func receive(conn *amqp.Connection, exchange string, key string, wg *sync.WaitGroup) {
ch, err := conn.Channel()
utils.FailOnError(err, "Failed to open a channel")
defer ch.Close()
err = ch.ExchangeDeclare(
exchange, // name
"direct", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
utils.FailOnError(err, "Failed to declare an exchange")
q, err := ch.QueueDeclare(
"", // name
false, // durable
false, // delete when usused
true, // exclusive
false, // no-wait
nil, // arguments
)
utils.FailOnError(err, "Failed to declare a queue")
err = ch.QueueBind(
q.Name, // queue name
key, // routing key
exchange, // exchange
false,
nil)
utils.FailOnError(err, "Failed to bind a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
utils.FailOnError(err, "Failed to register a consumer")
forever := make(chan bool)
go func() {
for msg := range msgs {
log.Printf(" [i] %s", key)
go processMessage(msg.Body)
}
}()
log.Printf(" [*] Waiting for logs. To exit press CTRL+C")
<-forever
}
// CreateChannel is a wrapper to simplify creating a channel with its attendant
// exchange.
func CreateChannel(conn *amqp.Connection, exchange, exchangeType string) (*amqp.Channel, error) {
ch, err := conn.Channel()
if err != nil {
return nil, err
}
if err = ch.ExchangeDeclare(exchange, exchangeType, true, false, false, false, nil); err != nil {
return nil, err
}
return ch, nil
}
// Fire is called when an event should be sent to the message broker.k
func (q amqpConn) Fire(entry *logrus.Entry) error {
ch, err := q.Connection.Channel()
if err != nil {
// Any other error other than connection closed, return.
if err != amqp.ErrClosed {
return err
}
// Attempt to connect again.
var conn *amqp.Connection
conn, err = amqp.Dial(q.params.URL)
if err != nil {
return err
}
ch, err = conn.Channel()
if err != nil {
return err
}
}
defer ch.Close()
err = ch.ExchangeDeclare(
q.params.Exchange,
q.params.ExchangeType,
q.params.Durable,
q.params.AutoDeleted,
q.params.Internal,
q.params.NoWait,
nil,
)
if err != nil {
return err
}
body, err := entry.String()
if err != nil {
return err
}
err = ch.Publish(
q.params.Exchange,
q.params.RoutingKey,
q.params.Mandatory,
q.params.Immediate,
amqp.Publishing{
ContentType: "application/json",
Body: []byte(body),
})
if err != nil {
return err
}
return nil
}
func Publish(messages chan *amqp.Publishing, connectionUri, exchange,
routingKey string, mandatory, immediate bool, results chan *PublishResult) {
var err error
var conn *amqp.Connection
var channel *amqp.Channel
defer close(results)
if conn, err = amqp.Dial(connectionUri); err != nil {
results <- &PublishResult{"Failed to connect", err, true}
return
}
defer conn.Close()
if channel, err = conn.Channel(); err != nil {
results <- &PublishResult{"Failed to get channel", err, true}
return
}
pubAcks, pubNacks := channel.NotifyConfirm(make(chan uint64), make(chan uint64))
chanClose := channel.NotifyClose(make(chan *amqp.Error))
if err = channel.Confirm(false); err != nil {
results <- &PublishResult{
"Failed to put channel into confirm mode",
err,
true,
}
return
}
for message := range messages {
err = channel.Publish(exchange, routingKey, mandatory, immediate, *message)
if err != nil {
results <- &PublishResult{"Failed to publish message", err, false}
continue
}
select {
case err = <-chanClose:
results <- &PublishResult{"Channel closed!", err, true}
case <-pubAcks:
results <- &PublishResult{
fmt.Sprintf("Published to exchange '%s' routing key '%v': %+v", exchange, routingKey, message),
nil,
false,
}
case <-pubNacks:
results <- &PublishResult{"Received basic.nack for message", errors.New("'basic.nack'"), false}
}
}
}
// Connects to the message queue, opens a channel, declares a queue
func open(cnf *config.Config) (*amqp.Connection, *amqp.Channel, amqp.Queue, error) {
var conn *amqp.Connection
var channel *amqp.Channel
var queue amqp.Queue
var err error
conn, err = amqp.Dial(cnf.Broker)
if err != nil {
return conn, channel, queue, fmt.Errorf("Dial: %s", err)
}
channel, err = conn.Channel()
if err != nil {
return conn, channel, queue, fmt.Errorf("Channel: %s", err)
}
if err := channel.ExchangeDeclare(
cnf.Exchange, // name of the exchange
cnf.ExchangeType, // type
true, // durable
false, // delete when complete
false, // internal
false, // noWait
nil, // arguments
); err != nil {
return conn, channel, queue, fmt.Errorf("Exchange: %s", err)
}
queue, err = channel.QueueDeclare(
cnf.DefaultQueue, // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
return conn, channel, queue, fmt.Errorf("Queue Declare: %s", err)
}
if err := channel.QueueBind(
queue.Name, // name of the queue
cnf.BindingKey, // binding key
cnf.Exchange, // source exchange
false, // noWait
nil, // arguments
); err != nil {
return conn, channel, queue, fmt.Errorf("Queue Bind: %s", err)
}
return conn, channel, queue, nil
}
// Build a Consumer to use in Workers that will consume an AMQP queue
//
// @param {String} tag(consumer_tag)
// @param {*amqp.Connection} connection
// @return {*Consumer}, {error}
// @api public
func BuildConsumer(tag string, connection *amqp.Connection) (*Consumer, error) {
c := &Consumer{
Conn: connection,
Channel: nil,
Tag: tag,
Done: make(chan error),
}
var err error
c.Channel, err = connection.Channel()
if err != nil {
return nil, fmt.Errorf("Channel: %s", err)
}
return c, nil
}
func channelHelper(
tc *testClient,
conn *amqpclient.Connection,
) (
*amqpclient.Channel,
chan amqpclient.Return,
chan *amqpclient.Error,
) {
ch, err := conn.Channel()
if err != nil {
panic("Bad channel!")
}
retChan := make(chan amqpclient.Return)
closeChan := make(chan *amqpclient.Error)
ch.NotifyReturn(retChan)
ch.NotifyClose(closeChan)
return ch, retChan, closeChan
}
// Start endpoint, stopping first if it was already running. update index
func (m *EndpointMgr) RestartEndpoint(conn *amqp.Connection, cfg config.EndpointConfig) error {
if old, ok := m.eps[cfg.Name]; ok {
old.Stop()
delete(m.eps, cfg.Name)
}
ch, err := conn.Channel()
if err != nil {
return err
}
s, err := GetStrategy(cfg.ConsumerStrategy, cfg.DeliveryStrategy)
if err != nil {
return err
}
ep, err := New(ch, cfg, s)
if err != nil {
return err
}
m.eps[cfg.Name] = ep
return nil
}
func newAMQPLogWriter(ctx gocontext.Context, conn *amqp.Connection, jobID uint64) (*amqpLogWriter, error) {
channel, err := conn.Channel()
if err != nil {
return nil, err
}
err = channel.ExchangeDeclare("reporting", "topic", true, false, false, false, nil)
if err != nil {
return nil, err
}
_, err = channel.QueueDeclare("reporting.jobs.logs", true, false, false, false, nil)
if err != nil {
return nil, err
}
err = channel.QueueBind("reporting.jobs.logs", "reporting.jobs.logs", "reporting", false, nil)
if err != nil {
return nil, err
}
writer := &amqpLogWriter{
ctx: context.FromComponent(ctx, "log_writer"),
amqpConn: conn,
amqpChan: channel,
jobID: jobID,
closeChan: make(chan struct{}),
buffer: new(bytes.Buffer),
timer: time.NewTimer(time.Hour),
timeout: 0,
}
context.LoggerFromContext(ctx).WithFields(logrus.Fields{
"writer": writer,
"job_id": jobID,
}).Debug("created new log writer")
go writer.flushRegularly()
return writer, nil
}
// Function to open channel on the amqp connection
func OpenChannel(conn *amqp.Connection, msgType string) (*amqp.Channel, error) {
// Open a channel to communicate with the server
channel, err := conn.Channel()
if err != nil {
return nil, err
}
// Declare the exchange to use when publishing
if err := channel.ExchangeDeclare(
msgType,
"direct",
false,
false,
false,
false,
nil,
); err != nil {
return nil, err
}
// Declare the queue to use when publishing
channel.QueueDeclare(
msgType,
false,
true,
false,
false,
nil,
)
// Bind the queue to the exchange
channel.QueueBind(
msgType,
"",
msgType,
false,
nil,
)
return channel, nil
}
// NewAMQPJobQueue creates a AMQPJobQueue backed by the given AMQP connections and
// connects to the AMQP queue with the given name. The queue will be declared
// in AMQP when this function is called, so an error could be raised if the
// queue already exists, but with different attributes than we expect.
func NewAMQPJobQueue(conn *amqp.Connection, queue string) (*AMQPJobQueue, error) {
channel, err := conn.Channel()
if err != nil {
return nil, err
}
_, err = channel.QueueDeclare(queue, true, false, false, false, nil)
if err != nil {
return nil, err
}
err = channel.Close()
if err != nil {
return nil, err
}
return &AMQPJobQueue{
conn: conn,
queue: queue,
}, nil
}
func emptyQueue(amqpURI, queueName string) error {
var conn *amqp.Connection
var channel *amqp.Channel
var err error
var ok bool
var msg amqp.Delivery
log.Printf("dialing %q", amqpURI)
conn, err = amqp.Dial(amqpURI)
if err != nil {
return fmt.Errorf("Dial: %s", err)
}
log.Printf("got Connection, getting Channel")
channel, err = conn.Channel()
if err != nil {
return fmt.Errorf("Channel: %s", err)
}
for {
if msg, ok, err = channel.Get(
queueName, // name of the queue
false, // autoAck
); err != nil {
return fmt.Errorf("Queue Get: %s", err)
}
if ok != true {
log.Printf("Queue empty")
break
}
log.Printf(
"got %dB delivery: [%v] %q",
len(msg.Body),
msg.DeliveryTag,
msg.Body,
)
msg.Ack(false)
}
return nil
}
func consume(conn *amqp.Connection, workerNum int, wg sync.WaitGroup) {
//START OMIT2
channel, err1 := conn.Channel()
if err1 != nil {
panic(err1.Error())
}
defer channel.Close()
defer wg.Done()
//END OMIT2
//START OMIT3
name := "TransactionFirehose"
durable := true
autoDelete := false
exclusive := false
noWait := false
queue, err2 := channel.QueueDeclare(name, durable, autoDelete, exclusive, noWait, nil)
if err2 != nil {
panic(err2.Error())
}
//END OMIT3
//START4 OMIT
consumer := ""
autoAck := true
cons_exclusive := false
noLocal := false
cons_noWait := false
msgs, err3 := channel.Consume(queue.Name, consumer, autoAck, cons_exclusive, noLocal, cons_noWait, nil)
if err3 != nil {
panic(err3.Error())
}
for d := range msgs {
log.Printf("Worker %v got a message: %s", workerNum, d.Body)
}
//END4 OMIT
}
func consumer(id int, db *sql.DB, conn *amqp.Connection) {
fmt.Println("starting processer")
channel, _ := conn.Channel()
channel.QueueDeclare("repos", false, false, false, false, nil)
channel.QueueDeclare("repos-priority", false, false, false, false, nil)
channel.Qos(1, 0, true)
priRepos, _ := channel.Consume("repos-priority", "consumer-"+strconv.Itoa(rand.Int()),
false, false, false, false, nil)
regRepos, _ := channel.Consume("repos", "consumer-"+strconv.Itoa(rand.Int()),
false, false, false, false, nil)
for {
var message amqp.Delivery
select {
case message = <-priRepos:
processRepo(string(message.Body), db, id)
message.Ack(false)
case message = <-regRepos:
processRepo(string(message.Body), db, id)
message.Ack(false)
}
}
}
func (que *TSQueue) Channel(conn *amqp.Connection) *amqp.Channel {
var err error
var ch *amqp.Channel
ch, err = conn.Channel()
if que.Acknowledge {
ch.Confirm(false)
que.Ack, que.NAck = ch.NotifyConfirm(make(chan uint64, 1), make(chan uint64, 1))
}
Fail(err, "Failed to open a channel")
_, err = ch.QueueDeclare(
que.Name, // name
false, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
Fail(err, "Failed to declare a queue")
return ch
}
func CreateRPCTopology(connection *amqp.Connection) (channel *amqp.Channel, err error) {
if channel, err = connection.Channel(); err != nil {
log.Println("Failed to get channel!: ", err)
return nil, err
}
if err = channel.ExchangeDeclare("rpc", "direct", true, false, false, false, nil); err != nil {
log.Println("Failed to declare exchange!: ", err)
return nil, err
}
if _, err = channel.QueueDeclare("ping", false, false, false, false, nil); err != nil {
log.Println("Failed to declare queue!: ", err)
return nil, err
}
if err = channel.QueueBind("ping", "ping", "rpc", false, nil); err != nil {
log.Println("Failed to bind to queue!: ", err)
return nil, err
}
return channel, nil
}
// Publish a Protocol Buffer to an AMQP Exchange
//
// @param {String} key
// @param {[]byte} body
// @param {*amqp.Connection} connection
// @return {error}
// @api public
func Publish(key string, body []byte, connection *amqp.Connection) error {
channel, err := connection.Channel()
if err != nil {
return fmt.Errorf("Channel: %s", err)
}
defer channel.Close()
// Reliable publisher confirms require confirm.select support from the
// connection.
if reliable {
if err := channel.Confirm(false); err != nil {
return fmt.Errorf("Channel could not be put into confirm mode: %s", err)
}
ack, nack := channel.NotifyConfirm(make(chan uint64), make(chan uint64))
defer confirmOne(ack, nack)
}
if err = channel.Publish(
exchangeName, // publish to an exchange
key, // routing to 0 or more queues
true, // mandatory
false, // immediate
amqp.Publishing{
Headers: amqp.Table{},
ContentType: "",
ContentEncoding: "",
Body: body,
DeliveryMode: amqp.Persistent,
Priority: 0, // 0-9
},
); err != nil {
return fmt.Errorf("Exchange Publish: %s", err)
}
return nil
}
