Beispiel #1
0
// InitClient sets up the elastic client. If the client has already been
// initalized it is a noop
func (e ElasticHosts) InitClient() error {
	if esClient == nil {
		var err error
		esClient, err = elastic.NewClient(elastic.SetURL(e...), elastic.SetMaxRetries(10), elastic.SetSniff(false))
		if err != nil {
			return err
		}
	}
	return nil
}
Beispiel #2
0
func ExampleClient_NewClient_manyOptions() {
	// Obtain a client for an Elasticsearch cluster of two nodes,
	// running on 10.0.1.1 and 10.0.1.2. Do not run the sniffer.
	// Set the healthcheck interval to 10s. When requests fail,
	// retry 5 times. Print error messages to os.Stderr and informational
	// messages to os.Stdout.
	client, err := elastic.NewClient(
		elastic.SetURL("http://10.0.1.1:9200", "http://10.0.1.2:9200"),
		elastic.SetSniff(false),
		elastic.SetHealthcheckInterval(10*time.Second),
		elastic.SetMaxRetries(5),
		elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)),
		elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)))
	if err != nil {
		// Handle error
		panic(err)
	}
	_ = client
}
Beispiel #3
0
func esClient() (*es.Client, error) {
	urls := strings.Split(*esURL, ",")
	options := []es.ClientOptionFunc{
		es.SetURL(urls...),
		es.SetSniff(false),
	}

	if *esUser != "" {
		options = append(options, es.SetBasicAuth(*esUser, *esPass))
	}

	if *esDebug {
		l := logger{}
		options = append(options,
			es.SetErrorLog(l),
			es.SetInfoLog(l),
			es.SetTraceLog(l),
		)
	}

	client, err := es.NewClient(options...)
	if err != nil {
		return nil, err
	}
	exists, err := client.IndexExists(*esIndex).Do()
	if err != nil {
		return nil, err
	}
	if !exists {
		resp, err := client.CreateIndex(*esIndex).Body(mapping).Do()
		if err != nil {
			return nil, err
		}
		if !resp.Acknowledged {
			return nil, errors.New("index creation was not acknowledged")
		}
		logrus.Info("index created")
	}

	return client, nil
}
func (t *TestCase) setup() error {
	var errorlogger *log.Logger
	if t.errorlogfile != "" {
		f, err := os.OpenFile(t.errorlogfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
		if err != nil {
			return err
		}
		errorlogger = log.New(f, "", log.Ltime|log.Lmicroseconds|log.Lshortfile)
	}

	var infologger *log.Logger
	if t.infologfile != "" {
		f, err := os.OpenFile(t.infologfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
		if err != nil {
			return err
		}
		infologger = log.New(f, "", log.LstdFlags)
	}

	// Trace request and response details like this
	var tracelogger *log.Logger
	if t.tracelogfile != "" {
		f, err := os.OpenFile(t.tracelogfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
		if err != nil {
			return err
		}
		tracelogger = log.New(f, "", log.LstdFlags)
	}

	client, err := elastic.NewClient(
		elastic.SetURL(t.nodes...),
		elastic.SetErrorLog(errorlogger),
		elastic.SetInfoLog(infologger),
		elastic.SetTraceLog(tracelogger),
		elastic.SetMaxRetries(t.maxRetries),
		elastic.SetSniff(t.sniff),
		elastic.SetSnifferInterval(t.snifferInterval),
		elastic.SetHealthcheck(t.healthcheck),
		elastic.SetHealthcheckInterval(t.healthcheckInterval))
	if err != nil {
		// Handle error
		return err
	}
	t.client = client

	// Use the IndexExists service to check if a specified index exists.
	exists, err := t.client.IndexExists(t.index).Do()
	if err != nil {
		return err
	}
	if exists {
		deleteIndex, err := t.client.DeleteIndex(t.index).Do()
		if err != nil {
			return err
		}
		if !deleteIndex.Acknowledged {
			return errors.New("delete index not acknowledged")
		}
	}

	// Create a new index.
	createIndex, err := t.client.CreateIndex(t.index).Do()
	if err != nil {
		return err
	}
	if !createIndex.Acknowledged {
		return errors.New("create index not acknowledged")
	}

	// Index a tweet (using JSON serialization)
	tweet1 := Tweet{User: "******", Message: "Take Five", Retweets: 0}
	_, err = t.client.Index().
		Index(t.index).
		Type("tweet").
		Id("1").
		BodyJson(tweet1).
		Do()
	if err != nil {
		return err
	}

	// Index a second tweet (by string)
	tweet2 := `{"user" : "olivere", "message" : "It's a Raggy Waltz"}`
	_, err = t.client.Index().
		Index(t.index).
		Type("tweet").
		Id("2").
		BodyString(tweet2).
		Do()
	if err != nil {
		return err
	}

	// Flush to make sure the documents got written.
	_, err = t.client.Flush().Index(t.index).Do()
	if err != nil {
		return err
	}

	return nil
}
Beispiel #5
0
// Returns object to work with bleve
func NewElasticStorage() *Elastic {
	url, err := config.Instance().String("elastic", "url")
	if err != nil {
		logger.Instance().
			WithError(err).
			Error("Elastic url is not provided")

		os.Exit(1)
	}

	client, err := es.NewClient(
		es.SetURL(url),
		es.SetSniff(false),
		es.SetHealthcheck(false),
		es.SetMaxRetries(0),
	)
	if err != nil {
		logger.Instance().
			WithError(err).
			Error("Unable to create client to elastic")

		os.Exit(1)
	}

	indexName, err := config.Instance().String("elastic", "index")
	if err != nil {
		logger.Instance().
			WithError(err).
			Error("Index name is not provided")

		os.Exit(1)
	}

	typeName, err := config.Instance().String("elastic", "type")
	if err != nil {
		logger.Instance().
			WithError(err).
			Error("Type name is not provided")

		os.Exit(1)
	}

	batchSize, err := config.Instance().Int("elastic", "batch_size")
	if err != nil {
		batchSize = 10
	}

	var (
		defaultIntervalSecond string = "1s"
		defaultIntervalMonth  string = "720h"
	)

	intervalCleanupStr, err := config.Instance().String("elastic", "interval_cleanup")
	if err != nil {
		intervalCleanupStr = defaultIntervalMonth
	}

	intervalCleanup, err := time.ParseDuration(intervalCleanupStr)
	if err != nil {
		intervalCleanup, _ = time.ParseDuration(defaultIntervalMonth)
	}

	intervalFlushStr, err := config.Instance().String("elastic", "interval_flush")
	if err != nil {
		intervalFlushStr = defaultIntervalSecond
	}

	intervalFlush, err := time.ParseDuration(intervalFlushStr)
	if err != nil {
		intervalFlush, _ = time.ParseDuration(defaultIntervalSecond)
	}

	return &Elastic{
		batchSize:          batchSize,
		indexName:          indexName,
		typeName:           typeName,
		client:             client,
		messages:           []*storage.Message{},
		mutexHandleMessage: &sync.RWMutex{},
		mutexFlushMessages: &sync.RWMutex{},
		ttl:                int64(intervalCleanup.Seconds() * 1000), // TTL is in milliseconds
		intervalFlush:      intervalFlush,
		lastFlush:          time.Now(),
	}
}
Beispiel #6
0
func initElastic(www string) (*elastic.Client, error) {
	username, password, host := parseBonsaiURL(www)

	log.Infof("Initializing ES: %v.", host)

	client, err := elastic.NewClient(elastic.SetURL(host), elastic.SetMaxRetries(10), elastic.SetBasicAuth(username, password), elastic.SetSniff(false))
	if err != nil {
		log.Fatalf("Error while connecting to ElasticSearch: %s", err)
		return nil, err
	}

	log.Info("Initializing Indices.")

	err = initIndices(client)
	if err != nil {
		log.Fatalf("Error while creating ElasticSearch Indices: %s", err)
		return nil, err
	}

	return client, err
}