func (q *AMQP) Connect() error {
q.Lock()
defer q.Unlock()
q.headers = amqp.Table{
"precision": q.Precision,
"database": q.Database,
"retention_policy": q.RetentionPolicy,
}
var connection *amqp.Connection
// make new tls config
tls, err := internal.GetTLSConfig(
q.SSLCert, q.SSLKey, q.SSLCA, q.InsecureSkipVerify)
if err != nil {
return err
}
if tls != nil {
connection, err = amqp.DialTLS(q.URL, tls)
} else {
connection, err = amqp.Dial(q.URL)
}
if err != nil {
return err
}
channel, err := connection.Channel()
if err != nil {
return fmt.Errorf("Failed to open a channel: %s", err)
}
err = channel.ExchangeDeclare(
q.Exchange, // name
"topic", // type
true, // durable
false, // delete when unused
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
return fmt.Errorf("Failed to declare an exchange: %s", err)
}
q.channel = channel
go func() {
log.Printf("Closing: %s", <-connection.NotifyClose(make(chan *amqp.Error)))
log.Printf("Trying to reconnect")
for err := q.Connect(); err != nil; err = q.Connect() {
log.Println(err)
time.Sleep(10 * time.Second)
}
}()
return nil
}
func (i *CLI) amqpErrorWatcher(amqpConn *amqp.Connection) {
errChan := make(chan *amqp.Error)
errChan = amqpConn.NotifyClose(errChan)
err, ok := <-errChan
if ok {
i.logger.WithField("err", err).Error("amqp connection errored, terminating")
i.cancel()
}
}
func (c *DefaultClient) connect(reconnect bool) (chan *AMQP.Error, error) {
var err error
var conn *AMQP.Connection
for retries := 0; reconnect || retries < ConnectRetries; retries++ {
conn, err = AMQP.Dial(c.url)
if err == nil {
break
}
c.ctx.Warnf("Could not connect to AMQP server (%s). Retry attempt %d, reconnect is %v...", err.Error(), retries+1, reconnect)
<-time.After(ConnectRetryDelay)
}
if err != nil {
return nil, fmt.Errorf("Could not connect to AMQP server (%s)", err)
}
closed := make(chan *AMQP.Error)
conn.NotifyClose(closed)
go func() {
err := <-closed
if err != nil {
c.ctx.Warnf("Connection closed (%s). Reconnecting...", err)
c.connect(true)
} else {
c.ctx.Info("Connection closed")
}
}()
c.conn = conn
c.ctx.Info("Connected to AMQP")
if reconnect {
c.mutex.Lock()
defer c.mutex.Unlock()
for user, channel := range c.channels {
channel.Close()
channel, err = c.conn.Channel()
if err != nil {
c.ctx.Warnf("Failed to reopen channel %s for %s (%s)", user.name, user.exchange, err)
continue
}
c.ctx.Infof("Reopened channel %s for %s", user.name, user.exchange)
user.channel = channel
c.channels[user] = channel
}
}
return closed, nil
}
func (ab *AmqpBeat) handleDisconnect(conn *amqp.Connection) {
connClosed := make(chan *amqp.Error)
conn.NotifyClose(connClosed)
go func() {
// This select is here to remove a race condition that
// can appear since stopping the beat triggers
// a disconnect. Calling ab.Stop() in that
// case causes a panic, since the stop chan is
// already closed
select {
case <-ab.stop:
// already stooped, fall through
case <-connClosed:
// server disconnect received
logp.Info("Detected AMQP connection closed. Stopping.")
ab.Stop()
}
}()
}
// Function to restart everything
func restart(reset chan *amqp.Error, config Configuration, conn *amqp.Connection, ch_gcm_log, ch_db_log, ch_apn_log, ch_gcm_log_success, ch_apn_log_success chan []byte, logger *log.Logger,
killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck chan int) {
// Kill all Worker
killAllWorkers(config, killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck)
conn.Close()
conn = initConn(config)
defer conn.Close()
olog(fmt.Sprintf("Spinning up workers"), config.DebugMode)
// For all GcmQueues start new goroutines
for i := 0; i < len(config.GcmQueues); i++ {
for j := 0; j < config.GcmQueues[i].Numworkers; j++ {
go gcm_processor(j, config, conn, config.GcmQueues[i].GcmTokenUpdateQueue, config.GcmQueues[i].GcmStatusInactiveQueue,
config.GcmQueues[i].Name, ch_gcm_log, ch_gcm_log_success, logger, killWorker, config.GcmQueues[i])
}
go gcm_error_processor_status_inactive(config, conn, config.GcmQueues[i].GcmStatusInactiveQueue, ch_db_log, logger, killStatusInactive, killStatusInactiveAck, config.GcmQueues[i])
go gcm_error_processor_token_update(config, conn, config.GcmQueues[i].GcmTokenUpdateQueue, ch_db_log, logger, killTokenUpd, killTokenUpdAck, config.GcmQueues[i])
}
//For all APN Queues start workers
for i := 0; i < len(config.ApnQueues); i++ {
// For all GCM Queues start workers
for j := 0; j < config.ApnQueues[i].NumWorkers; j++ {
go apn_processor(j, config, conn, config.ApnQueues[i].ApnStatusInactiveQueue,
config.ApnQueues[i].Name, ch_apn_log, ch_apn_log_success, logger, killWorker, config.ApnQueues[i])
}
olog(fmt.Sprintf("Startting workers for status_inactive for APN %s", config.GcmQueues[i].Identifier), config.DebugMode)
go apn_error_processor_status_inactive(config, conn, config.ApnQueues[i].ApnStatusInactiveQueue, ch_db_log, logger, killApnStatusInactive, killApnStatusInactiveAck, config.ApnQueues[i])
}
olog("Starting error processors", config.DebugMode)
reset = conn.NotifyClose(make(chan *amqp.Error))
for range reset {
go restart(reset, config, conn, ch_gcm_log, ch_db_log, ch_apn_log, ch_gcm_log_success, ch_apn_log_success, logger, killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck)
}
}
// NotifyClose registers a listener for close events either initiated by an error
// accompaning a connection.close method or by a normal shutdown.
// On normal shutdowns, the chan will be closed.
// To reconnect after a transport or protocol error, we should register a listener here and
// re-connect to server
// Reconnection is -not- working by now
func (r *RabbitMQ) handleErrors(conn *amqp.Connection) {
go func() {
for amqpErr := range conn.NotifyClose(make(chan *amqp.Error)) {
// if the computer sleeps then wakes longer than a heartbeat interval,
// the connection will be closed by the client.
// https://github.com/streadway/amqp/issues/82
r.log.Fatal(amqpErr.Error())
if strings.Contains(amqpErr.Error(), "NOT_FOUND") {
// do not continue
}
// CRITICAL Exception (320) Reason: "CONNECTION_FORCED - broker forced connection closure with reason 'shutdown'"
// CRITICAL Exception (501) Reason: "read tcp 127.0.0.1:5672: i/o timeout"
// CRITICAL Exception (503) Reason: "COMMAND_INVALID - unimplemented method"
if amqpErr.Code == 501 {
// reconnect
}
if amqpErr.Code == 320 {
// fmt.Println("tryin to reconnect")
// c.reconnect()
}
}
}()
go func() {
for b := range conn.NotifyBlocked(make(chan amqp.Blocking)) {
if b.Active {
r.log.Info("TCP blocked: %q", b.Reason)
} else {
r.log.Info("TCP unblocked")
}
}
}()
}
func (q *AMQP) Connect() error {
q.Lock()
defer q.Unlock()
q.headers = amqp.Table{
"precision": q.Precision,
"database": q.Database,
"retention_policy": q.RetentionPolicy,
}
var connection *amqp.Connection
var err error
if q.SslCert != "" && q.SslKey != "" {
// make new tls config
cfg := new(tls.Config)
if q.SslCa != "" {
// create ca pool
cfg.RootCAs = x509.NewCertPool()
// add self-signed cert
if ca, err := ioutil.ReadFile(q.SslCa); err == nil {
cfg.RootCAs.AppendCertsFromPEM(ca)
} else {
log.Println(err)
}
}
if cert, err := tls.LoadX509KeyPair(q.SslCert, q.SslKey); err == nil {
cfg.Certificates = append(cfg.Certificates, cert)
} else {
log.Println(err)
}
connection, err = amqp.DialTLS(q.URL, cfg)
} else {
connection, err = amqp.Dial(q.URL)
}
if err != nil {
return err
}
channel, err := connection.Channel()
if err != nil {
return fmt.Errorf("Failed to open a channel: %s", err)
}
err = channel.ExchangeDeclare(
q.Exchange, // name
"topic", // type
true, // durable
false, // delete when unused
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
return fmt.Errorf("Failed to declare an exchange: %s", err)
}
q.channel = channel
go func() {
log.Printf("Closing: %s", <-connection.NotifyClose(make(chan *amqp.Error)))
log.Printf("Trying to reconnect")
for err := q.Connect(); err != nil; err = q.Connect() {
log.Println(err)
time.Sleep(10 * time.Second)
}
}()
return nil
}
func (q *AMQP) Connect() error {
q.Lock()
defer q.Unlock()
q.headers = amqp.Table{
"precision": q.Precision,
"database": q.Database,
"retention_policy": q.RetentionPolicy,
}
var connection *amqp.Connection
// make new tls config
tls, err := internal.GetTLSConfig(
q.SSLCert, q.SSLKey, q.SSLCA, q.InsecureSkipVerify)
if err != nil {
return err
}
// parse auth method
var sasl []amqp.Authentication // nil by default
if strings.ToUpper(q.AuthMethod) == "EXTERNAL" {
sasl = []amqp.Authentication{&externalAuth{}}
}
amqpConf := amqp.Config{
TLSClientConfig: tls,
SASL: sasl, // if nil, it will be PLAIN
}
connection, err = amqp.DialConfig(q.URL, amqpConf)
if err != nil {
return err
}
channel, err := connection.Channel()
if err != nil {
return fmt.Errorf("Failed to open a channel: %s", err)
}
err = channel.ExchangeDeclare(
q.Exchange, // name
"topic", // type
true, // durable
false, // delete when unused
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
return fmt.Errorf("Failed to declare an exchange: %s", err)
}
q.channel = channel
go func() {
log.Printf("I! Closing: %s", <-connection.NotifyClose(make(chan *amqp.Error)))
log.Printf("I! Trying to reconnect")
for err := q.Connect(); err != nil; err = q.Connect() {
log.Println("E! ", err.Error())
time.Sleep(10 * time.Second)
}
}()
return nil
}
