Example #1
0
func StartSubscribers(amqpChan *amqp.Channel, esClient *elastic.Client, config *config.Config) {
	log.Println("Starting RabbitMQ subscribers")
	msgChan, err := amqpChan.Consume(config.AmqpQueues[0], "", true, false, false, false, nil)
	utils.CheckFatalError(err)
	go incomingDocConsumer(msgChan, esClient, config)
	log.Println("Started RabbitMQ subscribers")
}
Example #2
0
func incomingDocConsumer(incomingChan <-chan amqp.Delivery, esClient *elastic.Client, config *config.Config) {
	for message := range incomingChan {
		log.Printf("Received incoming Doc: %s", message.Body)
		resp, err := es_requests.IndexDocJSONBytes(esClient, config.DefaultIndex, "document", string(message.Body))
		log.Println(resp)
		utils.CheckFatalError(err)
		log.Printf("Indexed Incoming Doc")
	}
}
Example #3
0
//GetConfFromJSONFile reads application configuration from *filename* and maps it to a Config struct
func GetConfFromJSONFile(filename string) *Config {

	confContent, err := ioutil.ReadFile(filename)
	utils.CheckFatalError(err)
	config := &Config{}
	err = json.Unmarshal(confContent, config)
	utils.CheckFatalError(err)

	// If no DefaultIndex value is given, use the first Index value
	if config.DefaultIndex == "" {
		config.DefaultIndex = config.Indices[0]
	}

	log.Println("Configuration loaded:")
	spew.Config.Indent = "\t"
	spew.Dump(*config)
	return config
}
Example #4
0
func initESClient(url string, indices []string, doSniff bool) *elastic.Client {

	log.Printf("Connecting to ES on: %v", url)
	elasticClient, err := elastic.NewClient(elastic.SetURL(url), elastic.SetSniff(doSniff))
	utils.CheckFatalError(err)

	log.Println("Connected to ES")

	for _, index := range indices {

		log.Printf("Initializing Index: %s", index)

		indexExists, err := elasticClient.IndexExists(index).Do()
		utils.CheckFatalError(err)
		if !indexExists {
			resp, err := elasticClient.CreateIndex(index).Do()
			utils.CheckFatalError(err)
			if !resp.Acknowledged {
				log.Fatalf("Cannot create index: %s on ES", index)
			}
			log.Printf("Created index: %s on ES", index)

		} else {
			log.Printf("Index: %s already exists on ES", index)
		}

		_, err = elasticClient.OpenIndex(index).Do()
		utils.CheckFatalError(err)

		mapping, err := elasticClient.GetMapping().Index(index).Do()
		if err != nil {
			log.Printf("Cannot get mapping for index: %s", index)
		}
		log.Printf("Mapping for index %s: %s", index, mapping)
	}

	return elasticClient
}
Example #5
0
func initAMQPClient(config *config.Config) (*amqp.Connection, *amqp.Channel) {

	log.Printf("Connecting to RabbitMQ on: %v", config.AmqpURL)
	conn, err := amqp.Dial(config.AmqpURL)
	utils.CheckFatalError(err)
	ch, err := conn.Channel()
	utils.CheckFatalError(err)
	log.Println("Connected to RabbitMQ.")
	for _, queue := range config.AmqpQueues {
		log.Printf("Declaring Queue: %v", queue)
		_, err = ch.QueueDeclare(
			queue,
			false, // durable
			false, // delete when unused
			false, // exclusive
			false, // no-wait
			nil,   // arguments
		)
		utils.CheckFatalError(err)
		log.Println("Queue Declared")
	}
	return conn, ch
}