func StartSubscribers(amqpChan *amqp.Channel, esClient *elastic.Client, config *config.Config) {
log.Println("Starting RabbitMQ subscribers")
msgChan, err := amqpChan.Consume(config.AmqpQueues[0], "", true, false, false, false, nil)
utils.CheckFatalError(err)
go incomingDocConsumer(msgChan, esClient, config)
log.Println("Started RabbitMQ subscribers")
}
func incomingDocConsumer(incomingChan <-chan amqp.Delivery, esClient *elastic.Client, config *config.Config) {
for message := range incomingChan {
log.Printf("Received incoming Doc: %s", message.Body)
resp, err := es_requests.IndexDocJSONBytes(esClient, config.DefaultIndex, "document", string(message.Body))
log.Println(resp)
utils.CheckFatalError(err)
log.Printf("Indexed Incoming Doc")
}
}
//GetConfFromJSONFile reads application configuration from *filename* and maps it to a Config struct
func GetConfFromJSONFile(filename string) *Config {
confContent, err := ioutil.ReadFile(filename)
utils.CheckFatalError(err)
config := &Config{}
err = json.Unmarshal(confContent, config)
utils.CheckFatalError(err)
// If no DefaultIndex value is given, use the first Index value
if config.DefaultIndex == "" {
config.DefaultIndex = config.Indices[0]
}
log.Println("Configuration loaded:")
spew.Config.Indent = "\t"
spew.Dump(*config)
return config
}
func initESClient(url string, indices []string, doSniff bool) *elastic.Client {
log.Printf("Connecting to ES on: %v", url)
elasticClient, err := elastic.NewClient(elastic.SetURL(url), elastic.SetSniff(doSniff))
utils.CheckFatalError(err)
log.Println("Connected to ES")
for _, index := range indices {
log.Printf("Initializing Index: %s", index)
indexExists, err := elasticClient.IndexExists(index).Do()
utils.CheckFatalError(err)
if !indexExists {
resp, err := elasticClient.CreateIndex(index).Do()
utils.CheckFatalError(err)
if !resp.Acknowledged {
log.Fatalf("Cannot create index: %s on ES", index)
}
log.Printf("Created index: %s on ES", index)
} else {
log.Printf("Index: %s already exists on ES", index)
}
_, err = elasticClient.OpenIndex(index).Do()
utils.CheckFatalError(err)
mapping, err := elasticClient.GetMapping().Index(index).Do()
if err != nil {
log.Printf("Cannot get mapping for index: %s", index)
}
log.Printf("Mapping for index %s: %s", index, mapping)
}
return elasticClient
}
func initAMQPClient(config *config.Config) (*amqp.Connection, *amqp.Channel) {
log.Printf("Connecting to RabbitMQ on: %v", config.AmqpURL)
conn, err := amqp.Dial(config.AmqpURL)
utils.CheckFatalError(err)
ch, err := conn.Channel()
utils.CheckFatalError(err)
log.Println("Connected to RabbitMQ.")
for _, queue := range config.AmqpQueues {
log.Printf("Declaring Queue: %v", queue)
_, err = ch.QueueDeclare(
queue,
false, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
utils.CheckFatalError(err)
log.Println("Queue Declared")
}
return conn, ch
}