func StartSubscribers(amqpChan *amqp.Channel, esClient *elastic.Client, config *config.Config) {
log.Println("Starting RabbitMQ subscribers")
msgChan, err := amqpChan.Consume(config.AmqpQueues[0], "", true, false, false, false, nil)
utils.CheckFatalError(err)
go incomingDocConsumer(msgChan, esClient, config)
log.Println("Started RabbitMQ subscribers")
}
func amqpEventReceiver(ch *amqp.Channel, q amqp.Queue, c chan Event) {
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
//fixme send error to something ?
}
for d := range msgs {
var ev Event
ev.Headers = d.Headers
ev.Headers["_transport-exchange"] = d.Exchange
ev.Headers["_transport-RoutingKey"] = d.RoutingKey
ev.Headers["_transport-ContentType"] = d.ContentType
ev.Body = d.Body
c <- ev
}
c <- Event{
Body: []byte("dc1?"),
}
os.Exit(1)
}
/*
Run ...
*/
func (c *Consumer) Run(ch *amqp.Channel) chan []common.MapStr {
events := make(chan []common.MapStr)
// TODO: make exchange part of config
deliveries, err := ch.Consume("test", *c.cfg.Name, false, false, false, false, nil)
utils.FailOnError(err, "Failed to register consumer")
go c.consume(deliveries, events)
return events
}
func Consume(q *amqp.Queue, ch *amqp.Channel) (<-chan amqp.Delivery, error) {
return ch.Consume(
q.Name, // queue
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
}
//ConsumeQueueWithChannel consumes a RabbitMQConnection queue with predefined settings from a provided amqp.Channel
func (c *RabbitMQConnection) ConsumeQueueWithChannel(queue string, ch *amqp.Channel) (<-chan amqp.Delivery, error) {
return ch.Consume(
queue,
"",
false, //Auto ACK. False because we want to make sure of data integrity
false,
false,
false,
nil,
)
}
func getDeliveries(channel *amqp.Channel, queueName string) (<-chan amqp.Delivery, error) {
deliveries, err := channel.Consume(queueName, "", false, true, true, false, nil)
if err != nil {
if err, ok := err.(*amqp.Error); ok {
if err.Code == amqp.AccessRefused && strings.HasSuffix(err.Reason, "in exclusive use") {
return nil, nil
}
}
return nil, err
}
return deliveries, nil
}
func subscribe(ch *amqp.Channel, o *Options, tag string) (<-chan amqp.Delivery, error) {
autoAck := false
exclusive := false
noLocal := false
noWait := false
var args amqp.Table
if o.Priority > 0 {
args = amqp.Table{}
args["x-priority"] = o.Priority
}
return ch.Consume(o.Queue, tag, autoAck, exclusive, noLocal, noWait, args)
}
func listenForDiscoveryRequests(name string, ch *amqp.Channel) {
msgs, _ := ch.Consume(
name,
"",
true,
false,
false,
false,
nil)
for range msgs {
publishQueueName(ch)
}
}
func sineMain(ch *amqp.Channel) {
go sniffy()
q, err := ch.QueueDeclare(
"", // name
false, // durable
false, // delete when usused
true, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.QueueBind(
q.Name, // queue name
"", // routing key
"bpm", // exchange
false,
nil)
failOnError(err, "Failed to bind a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
portaudio.Initialize()
defer portaudio.Terminate()
//s := newStereoSine(256, 320, sampleRate)
s := newStereoSine(220, 220, sampleRate)
defer s.Close()
chk(s.Start())
defer s.Stop()
for m := range msgs {
var bm BpmMsg
err := json.Unmarshal(m.Body, &bm)
if err != nil {
fmt.Println("blah", err)
}
bpm = bm.Bpm
}
}
func WaitForFile(channel *amqp.Channel, filePath string, queueName string) {
defer channel.Close()
<-LockFile(filePath)
defer UnlockFile(filePath)
msgs, err := channel.Consume(
queueName, // queue
"", // consumer
true, // auto-ack
true, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
panic(fmt.Errorf("Failed to register a consumer: %s", err.Error()))
}
log.Println("waiting for file to be transfered")
fullPath := filepath.Join(rootDir, filePath)
file, err := os.OpenFile(fullPath, os.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
log.Fatalf("couldn't open local file %s", fullPath, err)
}
defer file.Close()
for {
msg := <-msgs
order, err := strconv.Atoi(msg.Headers["Order"].(string))
if err != nil {
log.Println(fmt.Errorf("no order was present in the header for file transfer"))
break
}
chunk := &FileChunk{
order,
msg.Body,
}
if chunk.Order == -1 {
break
}
log.Printf("writing chunk %d for file %s\n", chunk.Order, file.Name())
file.Write(chunk.Chunk)
}
log.Println("finished receiving file", filePath)
}
func Consume(ch *amqp.Channel, b *bind) (<-chan amqp.Delivery, error) {
var err error
q, err := ch.QueueDeclare(b.name, true, false, false, false, nil)
if err != nil {
Log.Error("queue declare err: %v", err)
}
err = ch.QueueBind(b.name, b.key, b.exchange, false, nil)
if err != nil {
Log.Error("queue bind err: %v", err)
}
tasks, err := ch.Consume(q.Name, b.tag, false, false, false, false, nil)
return tasks, err
}
func consumeQueue(channel *amqp.Channel, chanName string) (<-chan amqp.Delivery, error) {
msgChan, err := channel.Consume(
chanName, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
return nil, fmt.Errorf("CONSUMING '%s' CHANNEL: %s", chanName, err.Error())
}
return msgChan, nil
}
func (s *Server) consumer() {
var (
ch *amqp.Channel
err error
q amqp.Queue
consumer <-chan amqp.Delivery
)
qArgs := amqp.Table{}
if *ttl > 0 {
qArgs["x-message-ttl"] = int64(*ttl)
}
if s.Ttl > 0 {
qArgs["x-message-ttl"] = int64(s.Ttl)
}
for s.alive {
if err != nil {
ch = nil
log.Println(err)
err = nil
time.Sleep(*backoff)
continue
}
if ch == nil {
ch, err = s.dial()
}
if ch != nil {
q, err = ch.QueueDeclare(*queue, false, true, true, false, qArgs)
if err != nil {
continue
}
err = ch.QueueBind(q.Name, s.Source, s.Source, false, nil)
if err != nil {
continue
}
consumer, err = ch.Consume(q.Name, "mqslice", true, true, false, false, nil)
if err != nil {
continue
}
for elt := range consumer {
s.ch <- elt.Body
}
ch.Close()
ch = nil
}
}
}
func (d *DefaultWorker) Consume(ch *amqp.Channel, cfg config.EndpointConfig) (<-chan amqp.Delivery, error) {
queueName, ok := cfg.QueueConfig["queuename"].(string)
if !ok {
return nil, fmt.Errorf("unable to parse queuename from config")
}
prefetch, ok := cfg.QueueConfig["prefetch"].(int)
if !ok || prefetch < 1 {
prefetch = 1
}
q, err := ch.QueueDeclare(
queueName, // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
return nil, err
}
err = ch.Qos(
prefetch, // prefetch count
0, // prefetch size
false, // global
)
if err != nil {
return nil, err
}
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
return nil, err
}
return msgs, nil
}
func msgListener(ch *amqp.Channel) {
q, err := ch.QueueDeclare(
"", // name
false, // durable
false, // delete when usused
true, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.QueueBind(
q.Name, // queue name
"", // routing key
"updateMsgs", // exchange
false,
nil)
failOnError(err, "Failed to bind a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
for d := range msgs {
var m UpdateMsg
err := json.Unmarshal(d.Body, &m)
if err != nil {
fmt.Println("blah", err)
}
if m.Name == "bpm" {
fmt.Println("Setting BPM to", m.Value)
bpm = m.Value
}
}
}
func (rb *AmqpBeat) consumeIntoStream(stream chan<- *AmqpEvent, ch *amqp.Channel, c *ChannelConfig, wg *sync.WaitGroup) {
defer wg.Done()
err := ch.Qos(*c.QosPrefetchCount, 0, false)
if err != nil {
logp.Err("Failed to set QoS on queue '%s': %v", *c.Name, err)
}
_, err = ch.QueueDeclare(*c.Name, *c.Durable, *c.AutoDelete, false, false, *c.Args)
if err != nil {
logp.Err("Failed to declare queue '%s': %v", *c.Name, err)
return
}
q, err := ch.Consume(*c.Name, "", false, false, false, false, *c.Args)
if err != nil {
logp.Err("Failed to consume queue %s: %v", *c.Name, err)
return
}
mName := fmt.Sprintf("consumer.%s", *c.Name)
var msgCount int64
for {
select {
case d := <-q:
evt, err := rb.newAmqpEvent(&d, c.TypeTag, c.TsField, c.TsFormat)
msgCount++
logp.Debug("flow", "Consumed %d into %s", msgCount, *c.Name)
if err != nil {
logp.Warn("failed to build event for delivery, will be Nacked. (delivery = %v) (error = %v)", d, err)
d.Nack(false, true)
}
stream <- evt
rb.metrics <- &metric{name: mName, value: msgCount}
case <-rb.stop:
logp.Info("Consumer '%s' is stopping...", *c.Name)
return
}
}
logp.Info("Consumer '%s' has stopped.")
}
func NewConsumer(consumerName string, queueName string, exchange Exchange, config *Config) (consumer *Consumer, err error) {
if consumerName == "" {
err = errors.New("Must give the consumer a name")
return
}
if queueName == "" {
err = errors.New("Must give the queue name.")
return
}
if len(queueName) > 255 {
err = errors.New("Must give a queue name that contains 1-255 characters.")
return
}
if err = exchange.Validate(); err != nil {
return
}
if config == nil {
config = NewConfig()
}
conn, err := amqp.Dial(config.GetConnectionString())
if err != nil {
return
}
//setup the channel
var channel *amqp.Channel
if channel, err = conn.Channel(); err != nil {
return
}
//setup the exchange
if err = channel.ExchangeDeclare(
exchange.Name, //exchange name
exchange.Type, //exchange type
true, //durable
false, //remove when complete
false, //internal
false, //noWait
nil, //arguments
); err != nil {
return
}
//setup the queue
var queue amqp.Queue
if queue, err = channel.QueueDeclare(
queueName, // name of the queue
true, // durable
false, // delete when usused
false, // exclusive
false, // noWait
nil, // arguments
); err != nil {
return
}
if err = channel.QueueBind(
queue.Name, //queue name
exchange.RoutingKey, //routing key ("binding key")
exchange.Name, //exchange (source)
false, //noWait
nil, //arguments
); err != nil {
return
}
//setup the deliverables
var messages <-chan amqp.Delivery
if messages, err = channel.Consume(
queue.Name, //queue name
consumerName, //consumer name
false, //auto acknowledge
false, //exclusive
false, //not local
false, //no wait
nil, //arguments
); err != nil {
return
}
consumer = new(Consumer)
consumer.Name = consumerName
consumer.config = config
consumer.conn = conn
consumer.channel = channel
consumer.Exchange = exchange
consumer.incomingMessages = messages
return
}
func Start() {
mq := g.Config().Mq
nodes := map[string]interface{}{}
// Retry RetriedLimit times if there is some problem during connecting
var ch *amqp.Channel
var conn *amqp.Connection
var err error
for i := 0; i < RetriedLimit; i++ {
if conn, ch, err = setup(mq.Queue); err != nil {
time.Sleep(time.Second * SleepTimePeriod)
}
}
if err != nil {
log.Println(ExitStringPrefix + "retried too many times.")
return
}
defer conn.Close()
defer ch.Close()
q, err := ch.QueueDeclare(
"mute", // name
false, // durable
false, // delete when usused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
log.Printf(LogStringFormat, ExitStringPrefix+"failed to declare a queue", err)
return
}
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
log.Printf(LogStringFormat, ExitStringPrefix+"failed to register a consumer", err)
return
}
forever := make(chan bool)
var pkt Message
go func() {
for d := range msgs {
json.Unmarshal(d.Body, &pkt)
log.Printf("Received a message: %v", pkt)
params := map[string]string{
"host": pkt.Hostname,
"mute": "",
}
if pkt.Mute {
params["mute"] = "1"
} else {
params["mute"] = "0"
}
payload := map[string]interface{}{
"method": "host.update",
"params": params,
}
log.Println("payload =", payload)
s, err := json.Marshal(payload)
if err != nil {
log.Println("json.Marshal Error:", err.Error())
}
url := mq.Consumer
log.Println("url =", url)
reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(s)))
if err != nil {
log.Println("reqPost Error:", err.Error())
}
reqPost.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(reqPost)
if err != nil {
log.Println("resp Error:", err.Error())
} else {
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &nodes)
if err != nil {
log.Println("Unmarshal Error:", err.Error())
}
log.Println("nodes =", nodes)
}
}
}()
log.Println("mq.Start ok. Waiting for messages.")
<-forever
}
// connects to an AMQP topic and emits each message into streamtools.
func (b *FromAMQP) Run() {
var err error
var conn *amqp.Connection
var amqp_chan *amqp.Channel
toOut := make(blocks.MsgChan)
toError := make(chan error)
host := "localhost"
port := "5672"
username := "******"
password := "******"
routingkey := "#"
exchange := "amq.topic"
exchange_type := "topic"
for {
select {
case msg := <-toOut:
b.out <- msg
case err := <-toError:
b.Error(err)
case ruleI := <-b.inrule:
rule := ruleI.(map[string]interface{})
routingkey, err = util.ParseString(rule, "RoutingKey")
if err != nil {
b.Error(err)
continue
}
exchange, err = util.ParseString(rule, "Exchange")
if err != nil {
b.Error(err)
continue
}
exchange_type, err = util.ParseString(rule, "ExchangeType")
if err != nil {
b.Error(err)
continue
}
host, err = util.ParseString(rule, "Host")
if err != nil {
b.Error(err)
continue
}
port, err = util.ParseString(rule, "Port")
if err != nil {
b.Error(err)
continue
}
username, err = util.ParseString(rule, "Username")
if err != nil {
b.Error(err)
continue
}
password, err = util.ParseString(rule, "Password")
if err != nil {
b.Error(err)
continue
}
conn, err = amqp.Dial("amqp://" + username + ":" + password + "@" + host + ":" + port + "/")
if err != nil {
b.Error(err)
continue
}
amqp_chan, err = conn.Channel()
if err != nil {
b.Error(err)
continue
}
err = amqp_chan.ExchangeDeclare(
exchange, // name
exchange_type, // type
true, // durable
false, // auto-deleted
false, // internal
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
queue, err := amqp_chan.QueueDeclare(
"", // name
false, // durable
true, // delete when unused
false, // exclusive
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
err = amqp_chan.QueueBind(
queue.Name, // queue name
routingkey, // routing key
exchange, // exchange
false,
nil,
)
if err != nil {
b.Error(err)
continue
}
deliveries, err := amqp_chan.Consume(
queue.Name, // name
"", // consumerTag
true, // noAck
false, // exclusive
false, // noLocal
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
h := readWriteAMQPHandler{toOut, toError}
go h.handle(deliveries)
case <-b.quit:
if amqp_chan != nil {
amqp_chan.Close()
}
if conn != nil {
conn.Close()
}
return
case c := <-b.queryrule:
c <- map[string]interface{}{
"Host": host,
"Port": port,
"Username": username,
"Password": password,
"Exchange": exchange,
"ExchangeType": exchange_type,
"RoutingKey": routingkey,
}
}
}
}
func clientMain(ch *amqp.Channel) {
q, err := ch.QueueDeclare(
"", // name
false, // durable
false, // delete when usused
true, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.QueueBind(
q.Name, // queue name
"", // routing key
"bpm", // exchange
false,
nil)
failOnError(err, "Failed to bind a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
for d := range msgs {
var bm BpmMsg
err := json.Unmarshal(d.Body, &bm)
if err != nil {
fmt.Println("blah", err)
}
fmt.Println(bm)
// if bm.TickCounter%4 == 1 {
if bm.MicroTick == 1 {
fmt.Println("KICK Got one Modulo 4!", bm.TickCounter)
//go playrrr(kick, Soxfilter{})
//if rand.Intn(10) > 2 {
go playrrr(kick, Soxfilter{Effect: "pitch", Val: strconv.Itoa(int(math.Pow(float64(bm.TickCounter%1000), 3.0)) % 1000)})
//} else {
// fmt.Println("RANDY MISSS!")
//}
}
//if bm.TickCounter%4 == 0 {
// fmt.Println("SNARE Got one Modulo 3!", bm.TickCounter)
// //go playrrr(snare, Soxfilter{})
// if rand.Intn(10) > 1 {
// //go playrrr(snare, Soxfilter{Effect: "pitch", Val: strconv.Itoa(int(math.Pow(float64(bm.TickCounter%1000), 3.0)) % 1000)})
// go playrrr(snare, Soxfilter{})
// }
//}
//if bm.TickCounter%2 == 0 {
// fmt.Println("HAT Got one Modulo 1!", bm.TickCounter)
// // go playrrr(hat + " pitch " + strconv.Itoa(bm.TickCounter%1000))
// if rand.Intn(10) > 4 {
// go playrrr(hat, Soxfilter{Effect: "pitch", Val: strconv.Itoa(int(math.Pow(float64(bm.TickCounter%1000), 3.0)) % 1000)})
// }
//}
}
}
func (d *Topic) Consume(ch *amqp.Channel, cfg config.EndpointConfig) (<-chan amqp.Delivery, error) {
queueName, ok := cfg.QueueConfig["queuename"].(string)
if !ok {
return nil, fmt.Errorf("unable to parse queuename from config")
}
topicName, ok := cfg.QueueConfig["topicname"].(string)
if !ok {
return nil, fmt.Errorf("unable to parse topicname from config")
}
exchangeName, ok := cfg.QueueConfig["exchangename"].(string)
if !ok {
return nil, fmt.Errorf("unable to parse exchangename from config")
}
prefetch, ok := cfg.QueueConfig["prefetch"].(int)
if !ok || prefetch < 1 {
prefetch = 1
}
err := ch.ExchangeDeclare(
exchangeName, // name
"topic", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
return nil, err
}
q, err := ch.QueueDeclare(
queueName, // name
true, // durable
false, // delete when usused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
return nil, err
}
err = ch.Qos(
prefetch, // prefetch count
0, // prefetch size
false, // global
)
if err != nil {
return nil, err
}
err = ch.QueueBind(
q.Name, // queue name
topicName, // routing key
exchangeName, // exchange
false,
nil,
)
if err != nil {
return nil, err
}
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
return nil, err
}
return msgs, nil
}