func main() {
log.Println("Starting elasticgin")
//Config fetch
config := config.GetConfFromJSONFile("config.json")
//ES init
esClient := initESClient(config.ElasticURL, config.Indices, config.SniffCluster)
defer esClient.Stop()
//Rabbitmq init
var amqpChannel *amqp.Channel
if config.UseAMQP {
amqpConnection, amqpChannel := initAMQPClient(config)
defer amqpConnection.Close()
defer amqpChannel.Close()
rabbitmq.StartSubscribers(amqpChannel, esClient, config)
} else {
amqpChannel = nil
}
api := api.SetupAPI(esClient, amqpChannel, config)
api.Run(config.ApiURL)
}
func monitoring(uri string, queueName string, mgmtUri string, prefix string) {
var (
queueConn *amqp.Connection
queueChan *amqp.Channel
err error
)
queueConn, queueChan, err = rabbitmqConnect(uri, queueName)
if err != nil {
return
}
for {
log.Printf("fetch rabbitmq stats")
var metrics []graphite.Metric
for _, metric := range fetchQueueMetrics(mgmtUri, prefix) {
metrics = append(metrics, metric)
}
for _, metric := range fetchExchangeMetrics(mgmtUri, prefix) {
metrics = append(metrics, metric)
}
for _, metric := range metrics {
body := []byte(metric.Name + "\t" + metric.Value + "\t" + strconv.FormatInt(metric.Timestamp, 10))
msg := amqp.Publishing{ContentType: "text/plain", Body: body}
err = queueChan.Publish("", queueName, false, false, msg)
if err != nil {
log.Printf("publish err: %s", err)
return
}
//log.Printf("metric\t%s\t\t%s", metric.Name, metric.Value)
}
time.Sleep(time.Second * 5)
}
queueChan.Close()
queueConn.Close()
}
// Closes the connection
func (amqpBackend *AMQPBackend) close(channel *amqp.Channel, conn *amqp.Connection) error {
if err := channel.Close(); err != nil {
return fmt.Errorf("Channel Close: %s", err)
}
if err := conn.Close(); err != nil {
return fmt.Errorf("Connection Close: %s", err)
}
return nil
}
func publish(ch *amqp.Channel, q amqp.Queue, c <-chan string) {
for {
connectionStatus := <-c
if connectionStatus == "lost" {
for i := 0; i < 40; i++ {
conn, err := amqp.Dial("amqp://*****:*****@localhost:5672/")
if err != nil {
fmt.Println("Failed to connect to RabbitMQ")
time.Sleep(1 * time.Second)
continue
} else {
defer conn.Close()
}
}
}
if connectionStatus == "ok" {
conn, err := amqp.Dial("amqp://*****:*****@localhost:5672/")
if err != nil {
fmt.Println("Failed to connect to RabbitMQ")
time.Sleep(1 * time.Second)
continue
} else {
defer conn.Close()
}
ch, _ := conn.Channel()
defer ch.Close()
body := bodyFrom(os.Args)
q, err := ch.QueueDeclare(
"task_queue", // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
err = ch.Publish(
"", // exchange
q.Name, // routing key
false, // mandatory
false,
amqp.Publishing{
DeliveryMode: amqp.Persistent,
ContentType: "text/plain",
Body: []byte(body),
})
if err == nil {
log.Printf(" [x] Sent %s", body)
}
}
time.Sleep(1 * time.Second)
}
}
func WaitForFile(channel *amqp.Channel, filePath string, queueName string) {
defer channel.Close()
<-LockFile(filePath)
defer UnlockFile(filePath)
msgs, err := channel.Consume(
queueName, // queue
"", // consumer
true, // auto-ack
true, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
panic(fmt.Errorf("Failed to register a consumer: %s", err.Error()))
}
log.Println("waiting for file to be transfered")
fullPath := filepath.Join(rootDir, filePath)
file, err := os.OpenFile(fullPath, os.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
log.Fatalf("couldn't open local file %s", fullPath, err)
}
defer file.Close()
for {
msg := <-msgs
order, err := strconv.Atoi(msg.Headers["Order"].(string))
if err != nil {
log.Println(fmt.Errorf("no order was present in the header for file transfer"))
break
}
chunk := &FileChunk{
order,
msg.Body,
}
if chunk.Order == -1 {
break
}
log.Printf("writing chunk %d for file %s\n", chunk.Order, file.Name())
file.Write(chunk.Chunk)
}
log.Println("finished receiving file", filePath)
}
// Closes the connection
func (amqpBroker *AMQPBroker) close(channel *amqp.Channel, conn *amqp.Connection) error {
if channel != nil {
if err := channel.Close(); err != nil {
return fmt.Errorf("Channel Close: %s", err)
}
}
if conn != nil {
if err := conn.Close(); err != nil {
return fmt.Errorf("Connection Close: %s", err)
}
}
return nil
}
// shutdownChannel is a general closer function for channels
func shutdownChannel(channel *amqp.Channel, tag string) error {
// This waits for a server acknowledgment which means the sockets will have
// flushed all outbound publishings prior to returning. It's important to
// block on Close to not lose any publishings.
if err := channel.Cancel(tag, true); err != nil {
if amqpError, isAmqpError := err.(*amqp.Error); isAmqpError && amqpError.Code != 504 {
return fmt.Errorf("AMQP connection close error: %s", err)
}
}
if err := channel.Close(); err != nil {
return err
}
return nil
}
func (s *Server) consumer() {
var (
ch *amqp.Channel
err error
q amqp.Queue
consumer <-chan amqp.Delivery
)
qArgs := amqp.Table{}
if *ttl > 0 {
qArgs["x-message-ttl"] = int64(*ttl)
}
if s.Ttl > 0 {
qArgs["x-message-ttl"] = int64(s.Ttl)
}
for s.alive {
if err != nil {
ch = nil
log.Println(err)
err = nil
time.Sleep(*backoff)
continue
}
if ch == nil {
ch, err = s.dial()
}
if ch != nil {
q, err = ch.QueueDeclare(*queue, false, true, true, false, qArgs)
if err != nil {
continue
}
err = ch.QueueBind(q.Name, s.Source, s.Source, false, nil)
if err != nil {
continue
}
consumer, err = ch.Consume(q.Name, "mqslice", true, true, false, false, nil)
if err != nil {
continue
}
for elt := range consumer {
s.ch <- elt.Body
}
ch.Close()
ch = nil
}
}
}
// connects to an AMQP topic and emits each message into streamtools.
func (b *FromAMQP) Run() {
var err error
var conn *amqp.Connection
var amqp_chan *amqp.Channel
toOut := make(blocks.MsgChan)
toError := make(chan error)
host := "localhost"
port := "5672"
username := "******"
password := "******"
routingkey := "#"
exchange := "amq.topic"
exchange_type := "topic"
for {
select {
case msg := <-toOut:
b.out <- msg
case err := <-toError:
b.Error(err)
case ruleI := <-b.inrule:
rule := ruleI.(map[string]interface{})
routingkey, err = util.ParseString(rule, "RoutingKey")
if err != nil {
b.Error(err)
continue
}
exchange, err = util.ParseString(rule, "Exchange")
if err != nil {
b.Error(err)
continue
}
exchange_type, err = util.ParseString(rule, "ExchangeType")
if err != nil {
b.Error(err)
continue
}
host, err = util.ParseString(rule, "Host")
if err != nil {
b.Error(err)
continue
}
port, err = util.ParseString(rule, "Port")
if err != nil {
b.Error(err)
continue
}
username, err = util.ParseString(rule, "Username")
if err != nil {
b.Error(err)
continue
}
password, err = util.ParseString(rule, "Password")
if err != nil {
b.Error(err)
continue
}
conn, err = amqp.Dial("amqp://" + username + ":" + password + "@" + host + ":" + port + "/")
if err != nil {
b.Error(err)
continue
}
amqp_chan, err = conn.Channel()
if err != nil {
b.Error(err)
continue
}
err = amqp_chan.ExchangeDeclare(
exchange, // name
exchange_type, // type
true, // durable
false, // auto-deleted
false, // internal
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
queue, err := amqp_chan.QueueDeclare(
"", // name
false, // durable
true, // delete when unused
false, // exclusive
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
err = amqp_chan.QueueBind(
queue.Name, // queue name
routingkey, // routing key
exchange, // exchange
false,
nil,
)
if err != nil {
b.Error(err)
continue
}
deliveries, err := amqp_chan.Consume(
queue.Name, // name
"", // consumerTag
true, // noAck
false, // exclusive
false, // noLocal
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
h := readWriteAMQPHandler{toOut, toError}
go h.handle(deliveries)
case <-b.quit:
if amqp_chan != nil {
amqp_chan.Close()
}
if conn != nil {
conn.Close()
}
return
case c := <-b.queryrule:
c <- map[string]interface{}{
"Host": host,
"Port": port,
"Username": username,
"Password": password,
"Exchange": exchange,
"ExchangeType": exchange_type,
"RoutingKey": routingkey,
}
}
}
}
// connects to an AMQP topic and emits each message into streamtools.
func (b *ToAMQP) Run() {
var err error
var conn *amqp.Connection
var amqp_chan *amqp.Channel
host := "localhost"
port := "5672"
username := "******"
password := "******"
routingkey := "streamtools"
exchange := "amq.topic"
exchange_type := "topic"
for {
select {
case ruleI := <-b.inrule:
routingkey, err = util.ParseString(ruleI, "RoutingKey")
if err != nil {
b.Error(err)
continue
}
exchange, err = util.ParseString(ruleI, "Exchange")
if err != nil {
b.Error(err)
continue
}
exchange_type, err = util.ParseString(ruleI, "ExchangeType")
if err != nil {
b.Error(err)
continue
}
host, err = util.ParseString(ruleI, "Host")
if err != nil {
b.Error(err)
continue
}
port, err = util.ParseString(ruleI, "Port")
if err != nil {
b.Error(err)
continue
}
username, err = util.ParseString(ruleI, "Username")
if err != nil {
b.Error(err)
continue
}
password, err = util.ParseString(ruleI, "Password")
if err != nil {
b.Error(err)
continue
}
conn, err = amqp.Dial("amqp://" + username + ":" + password + "@" + host + ":" + port + "/")
if err != nil {
b.Error(err)
continue
}
amqp_chan, err = conn.Channel()
if err != nil {
b.Error(err)
continue
}
err = amqp_chan.ExchangeDeclare(
exchange, // name
exchange_type, // type
true, // durable
false, // auto-deleted
false, // internal
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
case msg := <-b.in:
if conn == nil || amqp_chan == nil {
continue
}
msgBytes, err := json.Marshal(msg)
// Make the output JSON if msg wasn't JSON already
if err != nil {
json_msg := map[string]interface{}{
"data": msg,
}
msgBytes, err = json.Marshal(json_msg)
if err != nil {
b.Error(err)
continue
}
}
if len(msgBytes) == 0 {
b.Error("Zero byte length message")
continue
}
err = amqp_chan.Publish(
exchange,
routingkey,
false,
false,
amqp.Publishing{
Headers: amqp.Table{},
ContentType: "text/plain",
ContentEncoding: "",
Body: msgBytes,
DeliveryMode: amqp.Transient,
Priority: 0,
},
)
if err != nil {
b.Error(err)
continue
}
case <-b.quit:
if amqp_chan != nil {
amqp_chan.Close()
}
if conn != nil {
conn.Close()
}
return
case c := <-b.queryrule:
c <- map[string]interface{}{
"Host": host,
"Port": port,
"Username": username,
"Password": password,
"Exchange": exchange,
"ExchangeType": exchange_type,
"RoutingKey": routingkey,
}
}
}
}
func (c *Consumer) Consume() {
c.InfLogger.Println("Registering consumer... ")
msgs, err := c.Channel.Consume(c.Queue, "", false, false, false, false, nil)
if err != nil {
c.ErrLogger.Fatalf("Failed to register a consumer: %s", err)
}
c.InfLogger.Println("Succeeded registering consumer.")
var sendCh *amqp.Channel
if c.DeadLetter {
var err error
sendCh, err = c.Connection.Channel()
if err != nil {
c.ErrLogger.Println("Could not open channel to republish failed jobs %s", err)
}
defer sendCh.Close()
}
defer c.Connection.Close()
defer c.Channel.Close()
forever := make(chan bool)
go func() {
for d := range msgs {
c.InfLogger.Println("reading deliveries")
input := d.Body
if c.Compression {
var b bytes.Buffer
w, err := zlib.NewWriterLevel(&b, zlib.BestCompression)
if err != nil {
c.ErrLogger.Println("Could not create zlib handler")
d.Nack(true, true)
}
c.InfLogger.Println("Compressed message")
w.Write(input)
w.Close()
input = b.Bytes()
}
if c.DeadLetter {
var retryCount int
if d.Headers == nil {
d.Headers = make(map[string]interface{}, 0)
}
retry, ok := d.Headers["retry_count"]
if !ok {
retry = "0"
}
c.InfLogger.Println(fmt.Sprintf("retry %s", retry))
retryCount, err = strconv.Atoi(retry.(string))
if err != nil {
c.ErrLogger.Fatal("could not parse retry header")
}
c.InfLogger.Println(fmt.Sprintf("retryCount : %d max retries: %d", retryCount, c.Retry))
var cmd *exec.Cmd
if c.WriteToPath != "" {
var filepath string
filepath += c.WriteToPath
filepath += c.Queue
filepath += "-"
filepath += time.Now().Format("20060102150405")
filepath += "-"
filepath += strconv.FormatInt(rand.Int63(), 10) // Convert random int to string
filepath += ".json"
cmd = c.Factory.Create(filepath)
jsonContent := jsonPrettyPrint(input) // Pretty print
err := ioutil.WriteFile(filepath, []byte(jsonContent), 0644)
if err != nil {
c.ErrLogger.Println("Could not write input to file")
d.Nack(true, true)
}
} else {
cmd = c.Factory.Create(base64.StdEncoding.EncodeToString(input))
}
if c.Executer.Execute(cmd, d.Body[:]) {
d.Ack(true)
} else if retryCount >= c.Retry {
d.Nack(true, false)
} else {
//republish message with new retry header
retryCount++
d.Headers["retry_count"] = strconv.Itoa(retryCount)
republish := amqp.Publishing{
ContentType: d.ContentType,
ContentEncoding: d.ContentEncoding,
Timestamp: time.Now(),
Body: d.Body,
Headers: d.Headers,
}
err = sendCh.Publish("", c.Queue, false, false, republish)
if err != nil {
c.ErrLogger.Println("error republish %s", err)
}
d.Ack(true)
}
} else {
cmd := c.Factory.Create(base64.StdEncoding.EncodeToString(input))
if c.Executer.Execute(cmd, d.Body[:]) {
d.Ack(true)
} else {
d.Nack(true, false)
}
}
}
}()
c.InfLogger.Println("Waiting for messages...")
<-forever
}
func Start() {
mq := g.Config().Mq
nodes := map[string]interface{}{}
// Retry RetriedLimit times if there is some problem during connecting
var ch *amqp.Channel
var conn *amqp.Connection
var err error
for i := 0; i < RetriedLimit; i++ {
if conn, ch, err = setup(mq.Queue); err != nil {
time.Sleep(time.Second * SleepTimePeriod)
}
}
if err != nil {
log.Println(ExitStringPrefix + "retried too many times.")
return
}
defer conn.Close()
defer ch.Close()
q, err := ch.QueueDeclare(
"mute", // name
false, // durable
false, // delete when usused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
log.Printf(LogStringFormat, ExitStringPrefix+"failed to declare a queue", err)
return
}
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
log.Printf(LogStringFormat, ExitStringPrefix+"failed to register a consumer", err)
return
}
forever := make(chan bool)
var pkt Message
go func() {
for d := range msgs {
json.Unmarshal(d.Body, &pkt)
log.Printf("Received a message: %v", pkt)
params := map[string]string{
"host": pkt.Hostname,
"mute": "",
}
if pkt.Mute {
params["mute"] = "1"
} else {
params["mute"] = "0"
}
payload := map[string]interface{}{
"method": "host.update",
"params": params,
}
log.Println("payload =", payload)
s, err := json.Marshal(payload)
if err != nil {
log.Println("json.Marshal Error:", err.Error())
}
url := mq.Consumer
log.Println("url =", url)
reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(s)))
if err != nil {
log.Println("reqPost Error:", err.Error())
}
reqPost.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(reqPost)
if err != nil {
log.Println("resp Error:", err.Error())
} else {
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &nodes)
if err != nil {
log.Println("Unmarshal Error:", err.Error())
}
log.Println("nodes =", nodes)
}
}
}()
log.Println("mq.Start ok. Waiting for messages.")
<-forever
}