Exemple #1
0
func main() {
	// Parse arguments
	kingpin.Version(Version)
	kingpin.CommandLine.Help = "Search for logs in a Logstash Elasticsearch index."
	kingpin.Parse()

	errorLog = log.New(os.Stderr, "ERROR ", log.Ltime|log.Lshortfile)
	if *verboseFlag || *debugFlag {
		infoLog = log.New(os.Stderr, "INFO ", log.Ltime|log.Lshortfile)
	}
	if *debugFlag {
		debugLog = log.New(os.Stderr, "TRACE ", log.Ltime|log.Lshortfile)
	}

	// Connect to the Elasticsearch cluster
	logInfo("Creating client...\n")
	client, err := elastic.NewClient(
		elastic.SetURL((*urlFlag).String()),
		elastic.SetSniff(false),
		elastic.SetHealthcheck(false),
		elastic.SetErrorLog(errorLog),
		elastic.SetInfoLog(infoLog),
		elastic.SetTraceLog(debugLog))
	exitIfErr(err)

	if *tailFlag {
		Tail(client)
	} else {
		Search(client)
	}
}
Exemple #2
0
func (es *ElasticSearch) Conn() error {
	client, err := elastic.NewClient(
		elastic.SetURL(es.uri),
		elastic.SetSniff(false),
		elastic.SetHealthcheckInterval(10*time.Second),
		elastic.SetMaxRetries(5))

	if err != nil {
		// Handle error
		return err
	}

	exists, err := client.IndexExists(es.index).Do()
	if err != nil {
		// Handle error
		return err
	}
	if !exists {
		// Create a new index.
		createIndex, err := client.CreateIndex(es.index).Do()
		if err != nil {
			// Handle error
			return err
		}
		if !createIndex.Acknowledged {
			// Not acknowledged
		}
	}

	es.client = client
	return nil
}
Exemple #3
0
func runIndices(cmd *Command, args []string) {
	var pattern = ""
	if len(args) > 0 {
		pattern = args[0]
	}

	// Get a client
	client, err := elastic.NewClient(elastic.SetURL(esUrl))
	if err != nil {
		log.Fatal("%v", err)
	}
	indices, err := client.IndexNames()
	if err != nil {
		log.Fatal("%v", err)
	}

	// Sort by default
	sort.Strings(indices)

	for _, index := range indices {
		if len(pattern) > 0 {
			matched, err := regexp.MatchString(pattern, index)
			if err != nil {
				log.Fatal("invalid pattern")
			}
			if matched {
				fmt.Println(index)
			}
		} else {
			fmt.Println(index)
		}
	}
}
func checkClient() {
	// Create a client and connect to http://192.1.199.81:9200
	// client, err := elastic.NewClient(elastic.SetURL("http://192.1.199.81:9200"))
	client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200"))

	if err != nil {
		// Handle error
		panic(err)
	}

	// Ping the Elasticsearch server to get e.g. the version number
	info, code, err := client.Ping().Do()
	if err != nil {
		// Handle error
		panic(err)
	}

	fmt.Printf("Elasticsearch returned with code %d and version %s\n", code, info.Version.Number)

	// Getting the ES version number
	esversion, err := client.ElasticsearchVersion("http://127.0.0.1:9200")
	if err != nil {
		// Handle error
		panic(err)
	}
	fmt.Printf("Elasticsearch version %s\n", esversion)

}
func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) {

	var esConfig ElasticSearchConfig
	opts, err := url.ParseQuery(uri.RawQuery)
	if err != nil {
		return nil, fmt.Errorf("failed to parser url's query string: %s", err)
	}

	// set the index for es,the default value is "heapster"
	esConfig.Index = ESIndex
	if len(opts["index"]) > 0 {
		esConfig.Index = opts["index"][0]
	}

	// If the ES cluster needs authentication, the username and secret
	// should be set in sink config.Else, set the Authenticate flag to false
	esConfig.NeedAuthen = false
	if len(opts["esUserName"]) > 0 && len(opts["esUserSecret"]) > 0 {
		esConfig.EsUserName = opts["esUserName"][0]
		esConfig.NeedAuthen = true
	}

	// set the URL endpoints of the ES's nodes. Notice that
	// when sniffing is enabled, these URLs are used to initially sniff the
	// cluster on startup.
	if len(opts["nodes"]) < 1 {
		return nil, fmt.Errorf("There is no node assigned for connecting ES cluster")
	}
	esConfig.EsNodes = append(esConfig.EsNodes, opts["nodes"]...)
	glog.V(2).Infof("configing elasticsearch sink with ES's nodes - %v", esConfig.EsNodes)

	var client *(elastic.Client)
	if esConfig.NeedAuthen == false {
		client, err = elastic.NewClient(elastic.SetURL(esConfig.EsNodes...))
	} else {
		client, err = elastic.NewClient(elastic.SetBasicAuth(esConfig.EsUserName, esConfig.EsUserSecret), elastic.SetURL(esConfig.EsNodes...))
	}

	if err != nil {
		return nil, fmt.Errorf("failed to create ElasticSearch client: %v", err)
	}
	esConfig.EsClient = client
	glog.V(2).Infof("elasticsearch sink configure successfully")
	return &esConfig, nil
}
func elasticInsert() {
	// Create a client and connect to http://127.0.0.1:9200
	client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200"))

	if err != nil {
		// Handle error
		panic(err)
	}

	// Use the IndexExists service to check if a specified index exists.
	exists, err := client.IndexExists("products").Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	if !exists {
		// Create a new index.
		createIndex, err := client.CreateIndex("products").Do()
		if err != nil {
			// Handle error
			panic(err)
		}
		if !createIndex.Acknowledged {
			// Not acknowledged
		}
	}

	var product = "name"

	// Index a product (using JSON serialization)
	put1, err := client.Index().
		Index("products").
		Type("product").
		Id("1").
		BodyJson(product).
		Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	fmt.Printf("Indexed product %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type)

	//Get product with specified ID
	get1, err := client.Get().
		Index("products").
		Type("product").
		Id("1").
		Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	if get1.Found {
		fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type)
	}
}
Exemple #7
0
// InitClient sets up the elastic client. If the client has already been
// initalized it is a noop
func (e LogstashElasticHosts) InitClient() error {
	if lsClient == nil {
		var err error
		lsClient, err = elastic.NewClient(elastic.SetURL(e...), elastic.SetMaxRetries(10))
		if err != nil {
			return err
		}
	}
	return nil
}
func elasticDel() {
	client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200"))

	// Delete the index again
	_, err = client.DeleteIndex("products").Do()
	if err != nil {
		// Handle error
		panic(err)
	}
}
Exemple #9
0
func ExampleClient_NewClient_cluster() {
	// Obtain a client for an Elasticsearch cluster of two nodes,
	// running on 10.0.1.1 and 10.0.1.2.
	client, err := elastic.NewClient(elastic.SetURL("http://10.0.1.1:9200", "http://10.0.1.2:9200"))
	if err != nil {
		// Handle error
		panic(err)
	}
	_ = client
}
func elasticQuery() {
	client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200"))

	// Flush to make sure the documents got written.
	_, err = client.Flush().Index("products").Do()
	if err != nil {
		panic(err)
	}

	// Search with a term query
	termQuery := elastic.NewTermQuery("name", "annual_fee")
	searchResult, err := client.Search().
		Index("products").  // search in index "products"
		Query(&termQuery).  // specify the query
		Sort("name", true). // sort by "name" field, ascending
		From(0).Size(10).   // take documents 0-9
		Pretty(true).       // pretty print request and response JSON
		Do()                // execute
	if err != nil {
		// Handle error
		panic(err)
	}

	// searchResult is of type SearchResult and returns hits, suggestions,
	// and all kinds of other information from Elasticsearch.
	fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)

	// // TotalHits is another convenience function that works even when something goes wrong.
	fmt.Printf("Found a total of %d products\n", searchResult.TotalHits())

	// Here's how you iterate through results with full control over each step.
	if searchResult.Hits != nil {
		fmt.Printf("Found a total of %d products\n", searchResult.Hits.TotalHits)

		// Iterate through results
		for _, hit := range searchResult.Hits.Hits {
			// hit.Index contains the name of the index

			// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
			var p models.Product
			err := json.Unmarshal(*hit.Source, &p)
			if err != nil {
				// Deserialization failed
			}

			// Work with product
			fmt.Printf("The new product is %s: %s\n", p.Name, p.Description)
		}
	} else {
		// No hits
		fmt.Print("Found no products\n")
	}

}
Exemple #11
0
func (shell *Shell) refreshClient() {
	url := fmt.Sprint("http://", shell.prompt.Host, ":", strconv.Itoa(shell.prompt.Port), "/")
	util.LogInfo(fmt.Sprint("Connecting to ", url, "..."))
	client, err := elastic.NewClient(
		elastic.SetURL(url),
	)
	if err == nil {
		shell.client = client
	} else {
		util.LogError(err.Error())
	}
}
Exemple #12
0
func ExampleClusterStateService() {
	client, err := elastic.NewClient()
	if err != nil {
		panic(err)
	}

	// Get cluster state
	res, err := client.ClusterState().Metric("version").Do()
	if err != nil {
		panic(err)
	}
	fmt.Printf("Cluster %q has version %d", res.ClusterName, res.Version)
}
Exemple #13
0
func ExampleClient_NewClient_default() {
	// Obtain a client to the Elasticsearch instance on http://localhost:9200.
	client, err := elastic.NewClient()
	if err != nil {
		// Handle error
		fmt.Printf("connection failed: %v\n", err)
	} else {
		fmt.Println("connected")
	}
	_ = client
	// Output:
	// connected
}
Exemple #14
0
func ExampleGetTemplateService() {
	client, err := elastic.NewClient()
	if err != nil {
		panic(err)
	}

	// Get template stored under "my-search-template"
	resp, err := client.GetTemplate().Id("my-search-template").Do()
	if err != nil {
		panic(err)
	}
	fmt.Printf("search template is: %q\n", resp.Template)
}
Exemple #15
0
func ExampleSearchResult() {
	client, err := elastic.NewClient()
	if err != nil {
		panic(err)
	}

	// Do a search
	searchResult, err := client.Search().Index("twitter").Query(elastic.NewMatchAllQuery()).Do()
	if err != nil {
		panic(err)
	}

	// searchResult is of type SearchResult and returns hits, suggestions,
	// and all kinds of other information from Elasticsearch.
	fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)

	// Each is a utility function that iterates over hits in a search result.
	// It makes sure you don't need to check for nil values in the response.
	// However, it ignores errors in serialization. If you want full control
	// over iterating the hits, see below.
	var ttyp Tweet
	for _, item := range searchResult.Each(reflect.TypeOf(ttyp)) {
		t := item.(Tweet)
		fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
	}
	fmt.Printf("Found a total of %d tweets\n", searchResult.TotalHits())

	// Here's how you iterate hits with full control.
	if searchResult.Hits != nil {
		fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)

		// Iterate through results
		for _, hit := range searchResult.Hits.Hits {
			// hit.Index contains the name of the index

			// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
			var t Tweet
			err := json.Unmarshal(*hit.Source, &t)
			if err != nil {
				// Deserialization failed
			}

			// Work with tweet
			fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
		}
	} else {
		// No hits
		fmt.Print("Found no tweets\n")
	}
}
Exemple #16
0
func (f *Factory) New(uri *url.URL) bridge.RegistryAdapter {
	urls := "http://127.0.0.1:9200"

	if uri.Host != "" {
		urls = "http://" + uri.Host
	}

	client, err := elasticapi.NewClient(elasticapi.SetURL(urls))
	if err != nil {
		log.Fatal("elastic: ", uri.Scheme)
	}

	return &ElasticAdapter{client: client}
}
Exemple #17
0
func ExampleSearchService() {
	// Get a client to the local Elasticsearch instance.
	client, err := elastic.NewClient()
	if err != nil {
		// Handle error
		panic(err)
	}

	// Search with a term query
	termQuery := elastic.NewTermQuery("user", "olivere")
	searchResult, err := client.Search().
		Index("twitter").   // search in index "twitter"
		Query(&termQuery).  // specify the query
		Sort("user", true). // sort by "user" field, ascending
		From(0).Size(10).   // take documents 0-9
		Pretty(true).       // pretty print request and response JSON
		Do()                // execute
	if err != nil {
		// Handle error
		panic(err)
	}

	// searchResult is of type SearchResult and returns hits, suggestions,
	// and all kinds of other information from Elasticsearch.
	fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)

	// Number of hits
	if searchResult.Hits != nil {
		fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)

		// Iterate through results
		for _, hit := range searchResult.Hits.Hits {
			// hit.Index contains the name of the index

			// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
			var t Tweet
			err := json.Unmarshal(*hit.Source, &t)
			if err != nil {
				// Deserialization failed
			}

			// Work with tweet
			fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
		}
	} else {
		// No hits
		fmt.Print("Found no tweets\n")
	}
}
Exemple #18
0
func ExampleDeleteTemplateService() {
	client, err := elastic.NewClient()
	if err != nil {
		panic(err)
	}

	// Delete template
	resp, err := client.DeleteTemplate().Id("my-search-template").Do()
	if err != nil {
		panic(err)
	}
	if resp != nil && resp.Found {
		fmt.Println("template deleted")
	}
}
Exemple #19
0
func NewElasticHistory(host string, index string) *ElasticHistory {
	var err error

	history := new(ElasticHistory)
	history.client, err = elastic.NewClient(elastic.SetURL(host))
	history.incomingMessages = make(chan *Message)
	history.index = strings.Replace(index, "#", "", -1)

	if err != nil {
		log.Panicf("Failed to connect to elasticsearch : %e", err)
	}

	go history.incomingLoop()

	return history
}
Exemple #20
0
func (a *Appbase) setupClient() error {
	var err error
	a.client, err = elastic.NewClient(
		elastic.SetURL(a.uri.String()),
		elastic.SetSniff(false),
	)

	if err != nil {
		return err
	}

	a.bulkService = a.client.Bulk().Index(a.appName).Type(a.typename)

	return nil

}
Exemple #21
0
func ExampleClusterHealthService() {
	client, err := elastic.NewClient()
	if err != nil {
		panic(err)
	}

	// Get cluster health
	res, err := client.ClusterHealth().Index("twitter").Do()
	if err != nil {
		panic(err)
	}
	if res == nil {
		panic(err)
	}
	fmt.Printf("Cluster status is %q\n", res.Status)
}
Exemple #22
0
func ExampleClusterHealthService_WaitForGreen() {
	client, err := elastic.NewClient()
	if err != nil {
		panic(err)
	}

	// Wait for status green
	res, err := client.ClusterHealth().WaitForStatus("green").Timeout("15s").Do()
	if err != nil {
		panic(err)
	}
	if res.TimedOut {
		fmt.Printf("time out waiting for cluster status %q\n", "green")
	} else {
		fmt.Printf("cluster status is %q\n", res.Status)
	}
}
Exemple #23
0
func ExampleDeleteIndexService() {
	// Get a client to the local Elasticsearch instance.
	client, err := elastic.NewClient()
	if err != nil {
		// Handle error
		panic(err)
	}
	// Delete an index.
	deleteIndex, err := client.DeleteIndex("twitter").Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	if !deleteIndex.Acknowledged {
		// Not acknowledged
	}
}
Exemple #24
0
func ExampleIndexExistsService() {
	// Get a client to the local Elasticsearch instance.
	client, err := elastic.NewClient()
	if err != nil {
		// Handle error
		panic(err)
	}
	// Use the IndexExists service to check if the index "twitter" exists.
	exists, err := client.IndexExists("twitter").Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	if exists {
		// ...
	}
}
Exemple #25
0
func processor(ic <-chan string) {
	es, err := elastic.NewClient(elastic.SetURL("http://testbox02.chinacloudapp.cn:9200"), elastic.SetSniff(false))
	if err != nil {
		log.Println(err)
	}
	for item := range ic {
		if item == "" {
			break
		}
		var m media
		if err = json.Unmarshal([]byte(item), &m); err == nil {
			es_push(es, "fsmedia", "media", m)
		}
	}
	if es != nil {
		es.Flush().Do()
	}
}
func TestCreateElasticSearchConfig(t *testing.T) {
	url, err := url.Parse("?nodes=https://foo.com:20468&nodes=https://bar.com:20468&esUserName=test&esUserSecret=password&maxRetries=10&startupHealthcheckTimeout=30&sniff=false&healthCheck=false")
	if err != nil {
		t.Fatalf("Error when parsing URL: %s", err.Error())
	}

	config, err := CreateElasticSearchConfig(url)
	if err != nil {
		t.Fatalf("Error when creating config: %s", err.Error())
	}

	expectedClient, err := elastic.NewClient(
		elastic.SetURL("https://foo.com:20468", "https://bar.com:20468"),
		elastic.SetBasicAuth("test", "password"),
		elastic.SetMaxRetries(10),
		elastic.SetHealthcheckTimeoutStartup(30*time.Second),
		elastic.SetSniff(false), elastic.SetHealthcheck(false))

	if err != nil {
		t.Fatalf("Error when creating client: %s", err.Error())
	}

	actualClientRefl := reflect.ValueOf(config.EsClient).Elem()
	expectedClientRefl := reflect.ValueOf(expectedClient).Elem()

	if actualClientRefl.FieldByName("basicAuthUsername").String() != expectedClientRefl.FieldByName("basicAuthUsername").String() {
		t.Fatalf("basicAuthUsername is not equal")
	}
	if actualClientRefl.FieldByName("basicAuthUsername").String() != expectedClientRefl.FieldByName("basicAuthUsername").String() {
		t.Fatalf("basicAuthUsername is not equal")
	}
	if actualClientRefl.FieldByName("maxRetries").Int() != expectedClientRefl.FieldByName("maxRetries").Int() {
		t.Fatalf("maxRetries is not equal")
	}
	if actualClientRefl.FieldByName("healthcheckTimeoutStartup").Int() != expectedClientRefl.FieldByName("healthcheckTimeoutStartup").Int() {
		t.Fatalf("healthcheckTimeoutStartup is not equal")
	}
	if actualClientRefl.FieldByName("snifferEnabled").Bool() != expectedClientRefl.FieldByName("snifferEnabled").Bool() {
		t.Fatalf("snifferEnabled is not equal")
	}
	if actualClientRefl.FieldByName("healthcheckEnabled").Bool() != expectedClientRefl.FieldByName("healthcheckEnabled").Bool() {
		t.Fatalf("healthcheckEnabled is not equal")
	}
}
Exemple #27
0
func ExampleAggregations() {
	// Get a client to the local Elasticsearch instance.
	client, err := elastic.NewClient()
	if err != nil {
		// Handle error
		panic(err)
	}

	// Create an aggregation for users and a sub-aggregation for a date histogram of tweets (per year).
	timeline := elastic.NewTermsAggregation().Field("user").Size(10).OrderByCountDesc()
	histogram := elastic.NewDateHistogramAggregation().Field("created").Interval("year")
	timeline = timeline.SubAggregation("history", histogram)

	// Search with a term query
	searchResult, err := client.Search().
		Index("twitter").                  // search in index "twitter"
		Query(elastic.NewMatchAllQuery()). // return all results, but ...
		SearchType("count").               // ... do not return hits, just the count
		Aggregation("timeline", timeline). // add our aggregation to the query
		Pretty(true).                      // pretty print request and response JSON
		Do()                               // execute
	if err != nil {
		// Handle error
		panic(err)
	}

	// Access "timeline" aggregate in search result.
	agg, found := searchResult.Aggregations.Terms("timeline")
	if !found {
		log.Fatalf("we sould have a terms aggregation called %q", "timeline")
	}
	for _, userBucket := range agg.Buckets {
		// Every bucket should have the user field as key.
		user := userBucket.Key

		// The sub-aggregation history should have the number of tweets per year.
		histogram, found := userBucket.DateHistogram("history")
		if found {
			for _, year := range histogram.Buckets {
				fmt.Printf("user %q has %d tweets in %q\n", user, year.DocCount, year.KeyAsString)
			}
		}
	}
}
Exemple #28
0
func ExampleClient_NewClient_manyOptions() {
	// Obtain a client for an Elasticsearch cluster of two nodes,
	// running on 10.0.1.1 and 10.0.1.2. Do not run the sniffer.
	// Set the healthcheck interval to 10s. When requests fail,
	// retry 5 times. Print error messages to os.Stderr and informational
	// messages to os.Stdout.
	client, err := elastic.NewClient(
		elastic.SetURL("http://10.0.1.1:9200", "http://10.0.1.2:9200"),
		elastic.SetSniff(false),
		elastic.SetHealthcheckInterval(10*time.Second),
		elastic.SetMaxRetries(5),
		elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)),
		elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)))
	if err != nil {
		// Handle error
		panic(err)
	}
	_ = client
}
Exemple #29
0
func main() {
	kingpin.Parse()

	// Create a client
	b, err := elastic.NewClient()
	Check(err)

	// Found the filesystem.
	nfs := pathfs.NewPathNodeFs(&ElasticFs{
		Index:      *cliIndex,
		Backend:    b,
		FileSystem: pathfs.NewDefaultFileSystem(),
	}, nil)
	s, _, err := nodefs.MountRoot(*cliDir, nfs.Root(), nil)
	if err != nil {
		log.Fatalf("Mount fail: %v\n", err)
	}

	s.Serve()
}
func ExampleWildcardQuery() {
	// Get a client to the local Elasticsearch instance.
	client, err := elastic.NewClient()
	if err != nil {
		// Handle error
		panic(err)
	}

	// Define wildcard query
	q := elastic.NewWildcardQuery("user", "oli*er?").Boost(1.2)
	searchResult, err := client.Search().
		Index("twitter"). // search in index "twitter"
		Query(q).         // use wildcard query defined above
		Do()              // execute
	if err != nil {
		// Handle error
		panic(err)
	}
	_ = searchResult
}