// InitClient sets up the elastic client. If the client has already been
// initalized it is a noop
func (e ElasticHosts) InitClient() error {
if esClient == nil {
var err error
esClient, err = elastic.NewClient(elastic.SetURL(e...), elastic.SetMaxRetries(10), elastic.SetSniff(false))
if err != nil {
return err
}
}
return nil
}
func ExampleClient_NewClient_manyOptions() {
// Obtain a client for an Elasticsearch cluster of two nodes,
// running on 10.0.1.1 and 10.0.1.2. Do not run the sniffer.
// Set the healthcheck interval to 10s. When requests fail,
// retry 5 times. Print error messages to os.Stderr and informational
// messages to os.Stdout.
client, err := elastic.NewClient(
elastic.SetURL("http://10.0.1.1:9200", "http://10.0.1.2:9200"),
elastic.SetSniff(false),
elastic.SetHealthcheckInterval(10*time.Second),
elastic.SetMaxRetries(5),
elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)),
elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)))
if err != nil {
// Handle error
panic(err)
}
_ = client
}
func esClient() (*es.Client, error) {
urls := strings.Split(*esURL, ",")
options := []es.ClientOptionFunc{
es.SetURL(urls...),
es.SetSniff(false),
}
if *esUser != "" {
options = append(options, es.SetBasicAuth(*esUser, *esPass))
}
if *esDebug {
l := logger{}
options = append(options,
es.SetErrorLog(l),
es.SetInfoLog(l),
es.SetTraceLog(l),
)
}
client, err := es.NewClient(options...)
if err != nil {
return nil, err
}
exists, err := client.IndexExists(*esIndex).Do()
if err != nil {
return nil, err
}
if !exists {
resp, err := client.CreateIndex(*esIndex).Body(mapping).Do()
if err != nil {
return nil, err
}
if !resp.Acknowledged {
return nil, errors.New("index creation was not acknowledged")
}
logrus.Info("index created")
}
return client, nil
}
func (t *TestCase) setup() error {
var errorlogger *log.Logger
if t.errorlogfile != "" {
f, err := os.OpenFile(t.errorlogfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
if err != nil {
return err
}
errorlogger = log.New(f, "", log.Ltime|log.Lmicroseconds|log.Lshortfile)
}
var infologger *log.Logger
if t.infologfile != "" {
f, err := os.OpenFile(t.infologfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
if err != nil {
return err
}
infologger = log.New(f, "", log.LstdFlags)
}
// Trace request and response details like this
var tracelogger *log.Logger
if t.tracelogfile != "" {
f, err := os.OpenFile(t.tracelogfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
if err != nil {
return err
}
tracelogger = log.New(f, "", log.LstdFlags)
}
client, err := elastic.NewClient(
elastic.SetURL(t.nodes...),
elastic.SetErrorLog(errorlogger),
elastic.SetInfoLog(infologger),
elastic.SetTraceLog(tracelogger),
elastic.SetMaxRetries(t.maxRetries),
elastic.SetSniff(t.sniff),
elastic.SetSnifferInterval(t.snifferInterval),
elastic.SetHealthcheck(t.healthcheck),
elastic.SetHealthcheckInterval(t.healthcheckInterval))
if err != nil {
// Handle error
return err
}
t.client = client
// Use the IndexExists service to check if a specified index exists.
exists, err := t.client.IndexExists(t.index).Do()
if err != nil {
return err
}
if exists {
deleteIndex, err := t.client.DeleteIndex(t.index).Do()
if err != nil {
return err
}
if !deleteIndex.Acknowledged {
return errors.New("delete index not acknowledged")
}
}
// Create a new index.
createIndex, err := t.client.CreateIndex(t.index).Do()
if err != nil {
return err
}
if !createIndex.Acknowledged {
return errors.New("create index not acknowledged")
}
// Index a tweet (using JSON serialization)
tweet1 := Tweet{User: "******", Message: "Take Five", Retweets: 0}
_, err = t.client.Index().
Index(t.index).
Type("tweet").
Id("1").
BodyJson(tweet1).
Do()
if err != nil {
return err
}
// Index a second tweet (by string)
tweet2 := `{"user" : "olivere", "message" : "It's a Raggy Waltz"}`
_, err = t.client.Index().
Index(t.index).
Type("tweet").
Id("2").
BodyString(tweet2).
Do()
if err != nil {
return err
}
// Flush to make sure the documents got written.
_, err = t.client.Flush().Index(t.index).Do()
if err != nil {
return err
}
return nil
}
// Returns object to work with bleve
func NewElasticStorage() *Elastic {
url, err := config.Instance().String("elastic", "url")
if err != nil {
logger.Instance().
WithError(err).
Error("Elastic url is not provided")
os.Exit(1)
}
client, err := es.NewClient(
es.SetURL(url),
es.SetSniff(false),
es.SetHealthcheck(false),
es.SetMaxRetries(0),
)
if err != nil {
logger.Instance().
WithError(err).
Error("Unable to create client to elastic")
os.Exit(1)
}
indexName, err := config.Instance().String("elastic", "index")
if err != nil {
logger.Instance().
WithError(err).
Error("Index name is not provided")
os.Exit(1)
}
typeName, err := config.Instance().String("elastic", "type")
if err != nil {
logger.Instance().
WithError(err).
Error("Type name is not provided")
os.Exit(1)
}
batchSize, err := config.Instance().Int("elastic", "batch_size")
if err != nil {
batchSize = 10
}
var (
defaultIntervalSecond string = "1s"
defaultIntervalMonth string = "720h"
)
intervalCleanupStr, err := config.Instance().String("elastic", "interval_cleanup")
if err != nil {
intervalCleanupStr = defaultIntervalMonth
}
intervalCleanup, err := time.ParseDuration(intervalCleanupStr)
if err != nil {
intervalCleanup, _ = time.ParseDuration(defaultIntervalMonth)
}
intervalFlushStr, err := config.Instance().String("elastic", "interval_flush")
if err != nil {
intervalFlushStr = defaultIntervalSecond
}
intervalFlush, err := time.ParseDuration(intervalFlushStr)
if err != nil {
intervalFlush, _ = time.ParseDuration(defaultIntervalSecond)
}
return &Elastic{
batchSize: batchSize,
indexName: indexName,
typeName: typeName,
client: client,
messages: []*storage.Message{},
mutexHandleMessage: &sync.RWMutex{},
mutexFlushMessages: &sync.RWMutex{},
ttl: int64(intervalCleanup.Seconds() * 1000), // TTL is in milliseconds
intervalFlush: intervalFlush,
lastFlush: time.Now(),
}
}
func initElastic(www string) (*elastic.Client, error) {
username, password, host := parseBonsaiURL(www)
log.Infof("Initializing ES: %v.", host)
client, err := elastic.NewClient(elastic.SetURL(host), elastic.SetMaxRetries(10), elastic.SetBasicAuth(username, password), elastic.SetSniff(false))
if err != nil {
log.Fatalf("Error while connecting to ElasticSearch: %s", err)
return nil, err
}
log.Info("Initializing Indices.")
err = initIndices(client)
if err != nil {
log.Fatalf("Error while creating ElasticSearch Indices: %s", err)
return nil, err
}
return client, err
}