func main() { // Parse arguments kingpin.Version(Version) kingpin.CommandLine.Help = "Search for logs in a Logstash Elasticsearch index." kingpin.Parse() errorLog = log.New(os.Stderr, "ERROR ", log.Ltime|log.Lshortfile) if *verboseFlag || *debugFlag { infoLog = log.New(os.Stderr, "INFO ", log.Ltime|log.Lshortfile) } if *debugFlag { debugLog = log.New(os.Stderr, "TRACE ", log.Ltime|log.Lshortfile) } // Connect to the Elasticsearch cluster logInfo("Creating client...\n") client, err := elastic.NewClient( elastic.SetURL((*urlFlag).String()), elastic.SetSniff(false), elastic.SetHealthcheck(false), elastic.SetErrorLog(errorLog), elastic.SetInfoLog(infoLog), elastic.SetTraceLog(debugLog)) exitIfErr(err) if *tailFlag { Tail(client) } else { Search(client) } }
func (es *ElasticSearch) Conn() error { client, err := elastic.NewClient( elastic.SetURL(es.uri), elastic.SetSniff(false), elastic.SetHealthcheckInterval(10*time.Second), elastic.SetMaxRetries(5)) if err != nil { // Handle error return err } exists, err := client.IndexExists(es.index).Do() if err != nil { // Handle error return err } if !exists { // Create a new index. createIndex, err := client.CreateIndex(es.index).Do() if err != nil { // Handle error return err } if !createIndex.Acknowledged { // Not acknowledged } } es.client = client return nil }
func runIndices(cmd *Command, args []string) { var pattern = "" if len(args) > 0 { pattern = args[0] } // Get a client client, err := elastic.NewClient(elastic.SetURL(esUrl)) if err != nil { log.Fatal("%v", err) } indices, err := client.IndexNames() if err != nil { log.Fatal("%v", err) } // Sort by default sort.Strings(indices) for _, index := range indices { if len(pattern) > 0 { matched, err := regexp.MatchString(pattern, index) if err != nil { log.Fatal("invalid pattern") } if matched { fmt.Println(index) } } else { fmt.Println(index) } } }
func checkClient() { // Create a client and connect to http://192.1.199.81:9200 // client, err := elastic.NewClient(elastic.SetURL("http://192.1.199.81:9200")) client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200")) if err != nil { // Handle error panic(err) } // Ping the Elasticsearch server to get e.g. the version number info, code, err := client.Ping().Do() if err != nil { // Handle error panic(err) } fmt.Printf("Elasticsearch returned with code %d and version %s\n", code, info.Version.Number) // Getting the ES version number esversion, err := client.ElasticsearchVersion("http://127.0.0.1:9200") if err != nil { // Handle error panic(err) } fmt.Printf("Elasticsearch version %s\n", esversion) }
func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) { var esConfig ElasticSearchConfig opts, err := url.ParseQuery(uri.RawQuery) if err != nil { return nil, fmt.Errorf("failed to parser url's query string: %s", err) } // set the index for es,the default value is "heapster" esConfig.Index = ESIndex if len(opts["index"]) > 0 { esConfig.Index = opts["index"][0] } // If the ES cluster needs authentication, the username and secret // should be set in sink config.Else, set the Authenticate flag to false esConfig.NeedAuthen = false if len(opts["esUserName"]) > 0 && len(opts["esUserSecret"]) > 0 { esConfig.EsUserName = opts["esUserName"][0] esConfig.NeedAuthen = true } // set the URL endpoints of the ES's nodes. Notice that // when sniffing is enabled, these URLs are used to initially sniff the // cluster on startup. if len(opts["nodes"]) < 1 { return nil, fmt.Errorf("There is no node assigned for connecting ES cluster") } esConfig.EsNodes = append(esConfig.EsNodes, opts["nodes"]...) glog.V(2).Infof("configing elasticsearch sink with ES's nodes - %v", esConfig.EsNodes) var client *(elastic.Client) if esConfig.NeedAuthen == false { client, err = elastic.NewClient(elastic.SetURL(esConfig.EsNodes...)) } else { client, err = elastic.NewClient(elastic.SetBasicAuth(esConfig.EsUserName, esConfig.EsUserSecret), elastic.SetURL(esConfig.EsNodes...)) } if err != nil { return nil, fmt.Errorf("failed to create ElasticSearch client: %v", err) } esConfig.EsClient = client glog.V(2).Infof("elasticsearch sink configure successfully") return &esConfig, nil }
func elasticInsert() { // Create a client and connect to http://127.0.0.1:9200 client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200")) if err != nil { // Handle error panic(err) } // Use the IndexExists service to check if a specified index exists. exists, err := client.IndexExists("products").Do() if err != nil { // Handle error panic(err) } if !exists { // Create a new index. createIndex, err := client.CreateIndex("products").Do() if err != nil { // Handle error panic(err) } if !createIndex.Acknowledged { // Not acknowledged } } var product = "name" // Index a product (using JSON serialization) put1, err := client.Index(). Index("products"). Type("product"). Id("1"). BodyJson(product). Do() if err != nil { // Handle error panic(err) } fmt.Printf("Indexed product %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type) //Get product with specified ID get1, err := client.Get(). Index("products"). Type("product"). Id("1"). Do() if err != nil { // Handle error panic(err) } if get1.Found { fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type) } }
// InitClient sets up the elastic client. If the client has already been // initalized it is a noop func (e LogstashElasticHosts) InitClient() error { if lsClient == nil { var err error lsClient, err = elastic.NewClient(elastic.SetURL(e...), elastic.SetMaxRetries(10)) if err != nil { return err } } return nil }
func elasticDel() { client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200")) // Delete the index again _, err = client.DeleteIndex("products").Do() if err != nil { // Handle error panic(err) } }
func ExampleClient_NewClient_cluster() { // Obtain a client for an Elasticsearch cluster of two nodes, // running on 10.0.1.1 and 10.0.1.2. client, err := elastic.NewClient(elastic.SetURL("http://10.0.1.1:9200", "http://10.0.1.2:9200")) if err != nil { // Handle error panic(err) } _ = client }
func elasticQuery() { client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200")) // Flush to make sure the documents got written. _, err = client.Flush().Index("products").Do() if err != nil { panic(err) } // Search with a term query termQuery := elastic.NewTermQuery("name", "annual_fee") searchResult, err := client.Search(). Index("products"). // search in index "products" Query(&termQuery). // specify the query Sort("name", true). // sort by "name" field, ascending From(0).Size(10). // take documents 0-9 Pretty(true). // pretty print request and response JSON Do() // execute if err != nil { // Handle error panic(err) } // searchResult is of type SearchResult and returns hits, suggestions, // and all kinds of other information from Elasticsearch. fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis) // // TotalHits is another convenience function that works even when something goes wrong. fmt.Printf("Found a total of %d products\n", searchResult.TotalHits()) // Here's how you iterate through results with full control over each step. if searchResult.Hits != nil { fmt.Printf("Found a total of %d products\n", searchResult.Hits.TotalHits) // Iterate through results for _, hit := range searchResult.Hits.Hits { // hit.Index contains the name of the index // Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}). var p models.Product err := json.Unmarshal(*hit.Source, &p) if err != nil { // Deserialization failed } // Work with product fmt.Printf("The new product is %s: %s\n", p.Name, p.Description) } } else { // No hits fmt.Print("Found no products\n") } }
func (shell *Shell) refreshClient() { url := fmt.Sprint("http://", shell.prompt.Host, ":", strconv.Itoa(shell.prompt.Port), "/") util.LogInfo(fmt.Sprint("Connecting to ", url, "...")) client, err := elastic.NewClient( elastic.SetURL(url), ) if err == nil { shell.client = client } else { util.LogError(err.Error()) } }
func ExampleClusterStateService() { client, err := elastic.NewClient() if err != nil { panic(err) } // Get cluster state res, err := client.ClusterState().Metric("version").Do() if err != nil { panic(err) } fmt.Printf("Cluster %q has version %d", res.ClusterName, res.Version) }
func ExampleClient_NewClient_default() { // Obtain a client to the Elasticsearch instance on http://localhost:9200. client, err := elastic.NewClient() if err != nil { // Handle error fmt.Printf("connection failed: %v\n", err) } else { fmt.Println("connected") } _ = client // Output: // connected }
func ExampleGetTemplateService() { client, err := elastic.NewClient() if err != nil { panic(err) } // Get template stored under "my-search-template" resp, err := client.GetTemplate().Id("my-search-template").Do() if err != nil { panic(err) } fmt.Printf("search template is: %q\n", resp.Template) }
func ExampleSearchResult() { client, err := elastic.NewClient() if err != nil { panic(err) } // Do a search searchResult, err := client.Search().Index("twitter").Query(elastic.NewMatchAllQuery()).Do() if err != nil { panic(err) } // searchResult is of type SearchResult and returns hits, suggestions, // and all kinds of other information from Elasticsearch. fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis) // Each is a utility function that iterates over hits in a search result. // It makes sure you don't need to check for nil values in the response. // However, it ignores errors in serialization. If you want full control // over iterating the hits, see below. var ttyp Tweet for _, item := range searchResult.Each(reflect.TypeOf(ttyp)) { t := item.(Tweet) fmt.Printf("Tweet by %s: %s\n", t.User, t.Message) } fmt.Printf("Found a total of %d tweets\n", searchResult.TotalHits()) // Here's how you iterate hits with full control. if searchResult.Hits != nil { fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits) // Iterate through results for _, hit := range searchResult.Hits.Hits { // hit.Index contains the name of the index // Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}). var t Tweet err := json.Unmarshal(*hit.Source, &t) if err != nil { // Deserialization failed } // Work with tweet fmt.Printf("Tweet by %s: %s\n", t.User, t.Message) } } else { // No hits fmt.Print("Found no tweets\n") } }
func (f *Factory) New(uri *url.URL) bridge.RegistryAdapter { urls := "http://127.0.0.1:9200" if uri.Host != "" { urls = "http://" + uri.Host } client, err := elasticapi.NewClient(elasticapi.SetURL(urls)) if err != nil { log.Fatal("elastic: ", uri.Scheme) } return &ElasticAdapter{client: client} }
func ExampleSearchService() { // Get a client to the local Elasticsearch instance. client, err := elastic.NewClient() if err != nil { // Handle error panic(err) } // Search with a term query termQuery := elastic.NewTermQuery("user", "olivere") searchResult, err := client.Search(). Index("twitter"). // search in index "twitter" Query(&termQuery). // specify the query Sort("user", true). // sort by "user" field, ascending From(0).Size(10). // take documents 0-9 Pretty(true). // pretty print request and response JSON Do() // execute if err != nil { // Handle error panic(err) } // searchResult is of type SearchResult and returns hits, suggestions, // and all kinds of other information from Elasticsearch. fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis) // Number of hits if searchResult.Hits != nil { fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits) // Iterate through results for _, hit := range searchResult.Hits.Hits { // hit.Index contains the name of the index // Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}). var t Tweet err := json.Unmarshal(*hit.Source, &t) if err != nil { // Deserialization failed } // Work with tweet fmt.Printf("Tweet by %s: %s\n", t.User, t.Message) } } else { // No hits fmt.Print("Found no tweets\n") } }
func ExampleDeleteTemplateService() { client, err := elastic.NewClient() if err != nil { panic(err) } // Delete template resp, err := client.DeleteTemplate().Id("my-search-template").Do() if err != nil { panic(err) } if resp != nil && resp.Found { fmt.Println("template deleted") } }
func NewElasticHistory(host string, index string) *ElasticHistory { var err error history := new(ElasticHistory) history.client, err = elastic.NewClient(elastic.SetURL(host)) history.incomingMessages = make(chan *Message) history.index = strings.Replace(index, "#", "", -1) if err != nil { log.Panicf("Failed to connect to elasticsearch : %e", err) } go history.incomingLoop() return history }
func (a *Appbase) setupClient() error { var err error a.client, err = elastic.NewClient( elastic.SetURL(a.uri.String()), elastic.SetSniff(false), ) if err != nil { return err } a.bulkService = a.client.Bulk().Index(a.appName).Type(a.typename) return nil }
func ExampleClusterHealthService() { client, err := elastic.NewClient() if err != nil { panic(err) } // Get cluster health res, err := client.ClusterHealth().Index("twitter").Do() if err != nil { panic(err) } if res == nil { panic(err) } fmt.Printf("Cluster status is %q\n", res.Status) }
func ExampleClusterHealthService_WaitForGreen() { client, err := elastic.NewClient() if err != nil { panic(err) } // Wait for status green res, err := client.ClusterHealth().WaitForStatus("green").Timeout("15s").Do() if err != nil { panic(err) } if res.TimedOut { fmt.Printf("time out waiting for cluster status %q\n", "green") } else { fmt.Printf("cluster status is %q\n", res.Status) } }
func ExampleDeleteIndexService() { // Get a client to the local Elasticsearch instance. client, err := elastic.NewClient() if err != nil { // Handle error panic(err) } // Delete an index. deleteIndex, err := client.DeleteIndex("twitter").Do() if err != nil { // Handle error panic(err) } if !deleteIndex.Acknowledged { // Not acknowledged } }
func ExampleIndexExistsService() { // Get a client to the local Elasticsearch instance. client, err := elastic.NewClient() if err != nil { // Handle error panic(err) } // Use the IndexExists service to check if the index "twitter" exists. exists, err := client.IndexExists("twitter").Do() if err != nil { // Handle error panic(err) } if exists { // ... } }
func processor(ic <-chan string) { es, err := elastic.NewClient(elastic.SetURL("http://testbox02.chinacloudapp.cn:9200"), elastic.SetSniff(false)) if err != nil { log.Println(err) } for item := range ic { if item == "" { break } var m media if err = json.Unmarshal([]byte(item), &m); err == nil { es_push(es, "fsmedia", "media", m) } } if es != nil { es.Flush().Do() } }
func TestCreateElasticSearchConfig(t *testing.T) { url, err := url.Parse("?nodes=https://foo.com:20468&nodes=https://bar.com:20468&esUserName=test&esUserSecret=password&maxRetries=10&startupHealthcheckTimeout=30&sniff=false&healthCheck=false") if err != nil { t.Fatalf("Error when parsing URL: %s", err.Error()) } config, err := CreateElasticSearchConfig(url) if err != nil { t.Fatalf("Error when creating config: %s", err.Error()) } expectedClient, err := elastic.NewClient( elastic.SetURL("https://foo.com:20468", "https://bar.com:20468"), elastic.SetBasicAuth("test", "password"), elastic.SetMaxRetries(10), elastic.SetHealthcheckTimeoutStartup(30*time.Second), elastic.SetSniff(false), elastic.SetHealthcheck(false)) if err != nil { t.Fatalf("Error when creating client: %s", err.Error()) } actualClientRefl := reflect.ValueOf(config.EsClient).Elem() expectedClientRefl := reflect.ValueOf(expectedClient).Elem() if actualClientRefl.FieldByName("basicAuthUsername").String() != expectedClientRefl.FieldByName("basicAuthUsername").String() { t.Fatalf("basicAuthUsername is not equal") } if actualClientRefl.FieldByName("basicAuthUsername").String() != expectedClientRefl.FieldByName("basicAuthUsername").String() { t.Fatalf("basicAuthUsername is not equal") } if actualClientRefl.FieldByName("maxRetries").Int() != expectedClientRefl.FieldByName("maxRetries").Int() { t.Fatalf("maxRetries is not equal") } if actualClientRefl.FieldByName("healthcheckTimeoutStartup").Int() != expectedClientRefl.FieldByName("healthcheckTimeoutStartup").Int() { t.Fatalf("healthcheckTimeoutStartup is not equal") } if actualClientRefl.FieldByName("snifferEnabled").Bool() != expectedClientRefl.FieldByName("snifferEnabled").Bool() { t.Fatalf("snifferEnabled is not equal") } if actualClientRefl.FieldByName("healthcheckEnabled").Bool() != expectedClientRefl.FieldByName("healthcheckEnabled").Bool() { t.Fatalf("healthcheckEnabled is not equal") } }
func ExampleAggregations() { // Get a client to the local Elasticsearch instance. client, err := elastic.NewClient() if err != nil { // Handle error panic(err) } // Create an aggregation for users and a sub-aggregation for a date histogram of tweets (per year). timeline := elastic.NewTermsAggregation().Field("user").Size(10).OrderByCountDesc() histogram := elastic.NewDateHistogramAggregation().Field("created").Interval("year") timeline = timeline.SubAggregation("history", histogram) // Search with a term query searchResult, err := client.Search(). Index("twitter"). // search in index "twitter" Query(elastic.NewMatchAllQuery()). // return all results, but ... SearchType("count"). // ... do not return hits, just the count Aggregation("timeline", timeline). // add our aggregation to the query Pretty(true). // pretty print request and response JSON Do() // execute if err != nil { // Handle error panic(err) } // Access "timeline" aggregate in search result. agg, found := searchResult.Aggregations.Terms("timeline") if !found { log.Fatalf("we sould have a terms aggregation called %q", "timeline") } for _, userBucket := range agg.Buckets { // Every bucket should have the user field as key. user := userBucket.Key // The sub-aggregation history should have the number of tweets per year. histogram, found := userBucket.DateHistogram("history") if found { for _, year := range histogram.Buckets { fmt.Printf("user %q has %d tweets in %q\n", user, year.DocCount, year.KeyAsString) } } } }
func ExampleClient_NewClient_manyOptions() { // Obtain a client for an Elasticsearch cluster of two nodes, // running on 10.0.1.1 and 10.0.1.2. Do not run the sniffer. // Set the healthcheck interval to 10s. When requests fail, // retry 5 times. Print error messages to os.Stderr and informational // messages to os.Stdout. client, err := elastic.NewClient( elastic.SetURL("http://10.0.1.1:9200", "http://10.0.1.2:9200"), elastic.SetSniff(false), elastic.SetHealthcheckInterval(10*time.Second), elastic.SetMaxRetries(5), elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)), elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags))) if err != nil { // Handle error panic(err) } _ = client }
func main() { kingpin.Parse() // Create a client b, err := elastic.NewClient() Check(err) // Found the filesystem. nfs := pathfs.NewPathNodeFs(&ElasticFs{ Index: *cliIndex, Backend: b, FileSystem: pathfs.NewDefaultFileSystem(), }, nil) s, _, err := nodefs.MountRoot(*cliDir, nfs.Root(), nil) if err != nil { log.Fatalf("Mount fail: %v\n", err) } s.Serve() }
func ExampleWildcardQuery() { // Get a client to the local Elasticsearch instance. client, err := elastic.NewClient() if err != nil { // Handle error panic(err) } // Define wildcard query q := elastic.NewWildcardQuery("user", "oli*er?").Boost(1.2) searchResult, err := client.Search(). Index("twitter"). // search in index "twitter" Query(q). // use wildcard query defined above Do() // execute if err != nil { // Handle error panic(err) } _ = searchResult }