func (e *Elastic) GetAnnotations(start, end *time.Time, source, host, creationUser, owner, category string) (annotate.Annotations, error) { annotations := annotate.Annotations{} s := elastic.NewSearchSource() if start != nil && end != nil { startQ := elastic.NewRangeQuery(annotate.EndDate).Gte(start) endQ := elastic.NewRangeQuery(annotate.StartDate).Lte(end) s = s.Query(elastic.NewBoolQuery().Must(startQ, endQ)) } if source != "" { s = s.Query(elastic.NewTermQuery(annotate.Source, source)) } if host != "" { s = s.Query(elastic.NewTermQuery(annotate.Host, host)) } if creationUser != "" { s = s.Query(elastic.NewTermQuery(annotate.CreationUser, creationUser)) } if owner != "" { s = s.Query(elastic.NewTermQuery(annotate.Owner, owner)) } if category != "" { s = s.Query(elastic.NewTermQuery(annotate.Category, category)) } res, err := e.Search(e.index).Query(s).Size(e.maxResults).Do() if err != nil { return annotations, err } var aType annotate.Annotation for _, item := range res.Each(reflect.TypeOf(aType)) { a := item.(annotate.Annotation) annotations = append(annotations, a) } return annotations, nil }
func (e *Elastic) GetAnnotations(start, end *time.Time, fieldFilters ...FieldFilter) (annotate.Annotations, error) { if !e.initialized { return nil, unInitErr } annotations := annotate.Annotations{} filters := []elastic.Query{} if start != nil && end != nil { startQ := elastic.NewRangeQuery(annotate.EndDate).Gte(start) endQ := elastic.NewRangeQuery(annotate.StartDate).Lte(end) filters = append(filters, elastic.NewBoolQuery().Must(startQ, endQ)) } for _, filter := range fieldFilters { switch filter.Field { case annotate.Source, annotate.Host, annotate.CreationUser, annotate.Owner, annotate.Category: default: return annotations, fmt.Errorf("%v is not a field that can be filtered on", filter.Field) } var q elastic.Query switch filter.Verb { case Is, "": q = elastic.NewTermQuery(filter.Field, filter.Value) case Empty: // Can't detect empty on a analyzed field if filter.Field == annotate.Message { return annotations, fmt.Errorf("message field does not support empty searches") } q = elastic.NewTermQuery(filter.Field, "") default: return annotations, fmt.Errorf("%v is not a valid query verb", filter.Verb) } if filter.Not { q = elastic.NewBoolQuery().MustNot(q) } filters = append(filters, q) } var aType annotate.Annotation scroll := e.Scroll(e.index).Query(elastic.NewBoolQuery().Must(filters...)).Size(e.maxResults).Pretty(true) for { res, err := scroll.Do() if err == io.EOF { break } if err != nil { return annotations, err } for _, item := range res.Each(reflect.TypeOf(aType)) { a := item.(annotate.Annotation) annotations = append(annotations, a) } } return annotations, nil }
// FindGeoFencesExceptByUser returns all geofences around a lat/lon pair, excluding ones from the specified user. func FindGeoFencesExceptByUser(centerLat float64, centerLon float64, radius int64, excludeBy int) ([]models.Fence, error) { query := elastic.NewBoolQuery() query = query.MustNot(elastic.NewTermQuery("owner", excludeBy)) query.Filter(elastic.NewGeoDistanceQuery("center").Distance(fmt.Sprintf("%d m", radius)).Lat(centerLat).Lon(centerLon)) searchResult, err := ElasticInstance.Search(). Index(IndexGeoFences). Query(query). Do() // Check whether an error appeared or not. if err != nil { return nil, err } if searchResult.Hits != nil { fences := make([]int64, searchResult.TotalHits(), searchResult.TotalHits()) fmt.Printf("Found a total of %d GeoFences\n", searchResult.Hits.TotalHits) // Iterate through results for i, hit := range searchResult.Hits.Hits { stringID, _ := strconv.ParseInt(hit.Id, 10, 64) fences[i] = stringID } return models.FindFencesByIDs(fences) } fmt.Print("Found no fences\n") var empty []models.Fence return empty, nil }
func ScopeES(ts opentsdb.TagSet, q elastic.Query) elastic.Query { var filters []elastic.Query for tagKey, tagValue := range ts { filters = append(filters, elastic.NewTermQuery(tagKey, tagValue)) } filters = append(filters, q) b := elastic.NewBoolQuery().Must(filters...) return b }
func ExampleSearchService() { // Get a client to the local Elasticsearch instance. client, err := elastic.NewClient() if err != nil { // Handle error panic(err) } // Search with a term query termQuery := elastic.NewTermQuery("user", "olivere") searchResult, err := client.Search(). Index("twitter"). // search in index "twitter" Query(termQuery). // specify the query Sort("user", true). // sort by "user" field, ascending From(0).Size(10). // take documents 0-9 Pretty(true). // pretty print request and response JSON Do() // execute if err != nil { // Handle error panic(err) } // searchResult is of type SearchResult and returns hits, suggestions, // and all kinds of other information from Elasticsearch. fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis) // Number of hits if searchResult.Hits.TotalHits > 0 { fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits) // Iterate through results for _, hit := range searchResult.Hits.Hits { // hit.Index contains the name of the index // Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}). var t Tweet err := json.Unmarshal(*hit.Source, &t) if err != nil { // Deserialization failed } // Work with tweet fmt.Printf("Tweet by %s: %s\n", t.User, t.Message) } } else { // No hits fmt.Print("Found no tweets\n") } }
func TestStringtoNb(t *testing.T) { t1 := "40.7" t2 := 3.3 t3 := "100" t4 := "nothing to do here" t5 := "now-30m" t6 := "100mille" r1 := stringToNb(t1) r2 := stringToNb(t2) r3 := stringToNb(t3) r4 := stringToNb(t4) r5 := stringToNb(t5) r6 := stringToNb(t6) assert.Equal(t, 40.7, r1) assert.Equal(t, 3.3, r2) assert.Equal(t, 100, r3) assert.Equal(t, "nothing to do here", r4) assert.Equal(t, "now-30m", r5) assert.Equal(t, "100mille", r6) query1 := elastic.NewBoolQuery().Should( elastic.NewTermQuery("User", "Thomas"), ).Must( elastic.NewTermQuery("Status", stringToNb("500")), elastic.NewRangeQuery("Code").Gt(stringToNb("42.42")), ) queryWrong := elastic.NewBoolQuery().Should( elastic.NewTermQuery("User", "Thomas"), ).Must( elastic.NewTermQuery("Status", "500"), elastic.NewRangeQuery("Code").Gt("42.42"), ) query2 := elastic.NewBoolQuery().Should( elastic.NewTermQuery("User", "Thomas"), ).Must( elastic.NewTermQuery("Status", 500), elastic.NewRangeQuery("Code").Gt(42.42), ) assert.NotEqual(t, queryWrong, query2) assert.Equal(t, query1, query2) }
func (t *TestCase) search() { // Loop forever to check for connection issues for { // Get tweet with specified ID get1, err := t.client.Get(). Index(t.index). Type("tweet"). Id("1"). Do() if err != nil { //failf("Get failed: %v", err) t.runCh <- RunInfo{Success: false} continue } if !get1.Found { //log.Printf("Document %s not found\n", "1") //fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type) t.runCh <- RunInfo{Success: false} continue } // Search with a term query searchResult, err := t.client.Search(). Index(t.index). // search in index t.index Query(elastic.NewTermQuery("user", "olivere")). // specify the query Sort("user", true). // sort by "user" field, ascending From(0).Size(10). // take documents 0-9 Pretty(true). // pretty print request and response JSON Do() // execute if err != nil { //failf("Search failed: %v\n", err) t.runCh <- RunInfo{Success: false} continue } // searchResult is of type SearchResult and returns hits, suggestions, // and all kinds of other information from Elasticsearch. //fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis) // Number of hits if searchResult.Hits.TotalHits > 0 { //fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits) // Iterate through results for _, hit := range searchResult.Hits.Hits { // hit.Index contains the name of the index // Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}). var tweet Tweet err := json.Unmarshal(*hit.Source, &tweet) if err != nil { // Deserialization failed //failf("Deserialize failed: %v\n", err) t.runCh <- RunInfo{Success: false} continue } // Work with tweet //fmt.Printf("Tweet by %s: %s\n", t.User, t.Message) } } else { // No hits //fmt.Print("Found no tweets\n") } t.runCh <- RunInfo{Success: true} // Sleep some time time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond) } }
func Test_getQueries(t *testing.T) { eslog.InitSilent() // term filters var filters []interface{} = []interface{}{ map[interface{}]interface{}{"term": []interface{}{"test", "yes"}}, map[interface{}]interface{}{"term": []interface{}{"required", true}}, } realfilters := []elastic.Query{ elastic.NewTermQuery("test", "yes"), elastic.NewTermQuery("required", true), } testfilters, err := getQueries(filters) assert.Equal(t, err, nil, "Should be nil") assert.Equal(t, realfilters, testfilters) realfilters = []elastic.Query{ elastic.NewTermQuery("test", "yes"), elastic.NewTermQuery("required", false), } assert.NotEqual(t, realfilters, testfilters) //range filters filters = []interface{}{ map[interface{}]interface{}{"term": []interface{}{"value", 146}}, map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}}, map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte", "now-1h"}}, } realfilters = []elastic.Query{ elastic.NewTermQuery("value", 146), elastic.NewTermQuery("othervalue", "testTest"), elastic.NewRangeQuery("Timestamp").Gte("now-1h"), } testfilters, err = getQueries(filters) assert.Equal(t, err, nil, "Should be nil") assert.Equal(t, realfilters, testfilters) realfilters = []elastic.Query{ elastic.NewTermQuery("value", 146), elastic.NewTermQuery("othervalue", "testTest"), elastic.NewRangeQuery("Timestamp").Lt("now-1h"), } assert.NotEqual(t, realfilters, testfilters) filters = []interface{}{ map[interface{}]interface{}{"term": []interface{}{"value", 146}}, map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}}, map[interface{}]interface{}{"range": []interface{}{"Timestamp", "lt", "now-1h"}}, } testfilters, err = getQueries(filters) assert.Equal(t, err, nil, "Should be nil") assert.Equal(t, realfilters, testfilters) //non valid fields filters = []interface{}{ map[interface{}]interface{}{"term": []interface{}{145, "yes"}}, map[interface{}]interface{}{"term": []interface{}{"required", "yes"}}, } testfilters, err = getQueries(filters) assert.NotNil(t, err) filters = []interface{}{ map[interface{}]interface{}{"term": []interface{}{"yes"}}, map[interface{}]interface{}{"term": []interface{}{"required", 28}}, } testfilters, err = getQueries(filters) assert.NotNil(t, err) filters = []interface{}{ map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte"}}, map[interface{}]interface{}{"term": []interface{}{"required", 28}}, } testfilters, err = getQueries(filters) assert.NotNil(t, err) filters = []interface{}{ map[interface{}]interface{}{"range": []interface{}{42, "gte", "now-30m"}}, map[interface{}]interface{}{"term": []interface{}{"required", 28}}, } testfilters, err = getQueries(filters) assert.NotNil(t, err) }
func Test_boolquery(t *testing.T) { eslog.InitSilent() info := &config.QueryInfo{} myQuery, err := computeQuery(info) assert.NotNil(t, err) queryInfo := &config.QueryInfo{ Type: "boolquery", Clauses: map[string]interface{}{ "must": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"Value", 146.5}}, map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}}, map[interface{}]interface{}{"range": []interface{}{"Timestamp", "lt", "now-1h"}}, }, "must_not": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"status", "OK"}}, }, }, } realQuery := elastic.NewBoolQuery().Must( elastic.NewTermQuery("Value", 146.5), elastic.NewTermQuery("othervalue", "testTest"), elastic.NewRangeQuery("Timestamp").Lt("now-1h"), ).MustNot( elastic.NewTermQuery("status", "OK"), ) myQuery, err = computeQuery(queryInfo) assert.Nil(t, err) assert.Equal(t, realQuery, myQuery) queryInfo = &config.QueryInfo{ Type: "boolquery", Clauses: map[string]interface{}{ "must_not": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"Status", "Error"}}, map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gt", "now-30m"}}, }, }, } realQuery = elastic.NewBoolQuery().MustNot( elastic.NewTermQuery("Status", "Error"), ).MustNot( elastic.NewRangeQuery("Timestamp").Gt("now-30m"), ) myQuery, err = computeQuery(queryInfo) assert.Nil(t, err) assert.Equal(t, realQuery, myQuery) queryInfo = &config.QueryInfo{ Type: "boolquery", Clauses: map[string]interface{}{ "should": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"User", "Thomas"}}, }, "must": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}}, map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gt", "now-2h"}}, }, }, } realQuery = elastic.NewBoolQuery().Should( elastic.NewTermQuery("User", "Thomas"), ).Must( elastic.NewTermQuery("Status", "OK"), elastic.NewRangeQuery("Timestamp").Gt("now-2h"), ) myQuery, err = computeQuery(queryInfo) assert.Nil(t, err) assert.Equal(t, realQuery, myQuery) realQuery = elastic.NewBoolQuery().Should( elastic.NewTermQuery("User", "Tobias"), ).Must( elastic.NewTermQuery("Status", "OK"), elastic.NewRangeQuery("Timestamp").Gt("now-2h"), ) assert.NotEqual(t, realQuery, myQuery) queryInfo = &config.QueryInfo{ Type: "boolquery", Clauses: map[string]interface{}{ "should": []interface{}{ map[interface{}]interface{}{"plop": []interface{}{"Thomas"}}, }, "must": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}}, map[interface{}]interface{}{"hihi": []interface{}{"Timestamp", "gt", "now-2h"}}, }, }, } myQuery, err = computeQuery(queryInfo) assert.NotNil(t, err) // test filterQueries queryInfo = &config.QueryInfo{ Type: "boolquery", Clauses: map[string]interface{}{ "should": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"User", "Thomas"}}, }, "must": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}}, }, "filter": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"Method", "XPUT"}}, map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte", "now-1h"}}, }, }, } realQuery = elastic.NewBoolQuery().Should( elastic.NewTermQuery("User", "Thomas"), ).Must( elastic.NewTermQuery("Status", "OK"), ).Filter( elastic.NewTermQuery("Method", "XPUT"), elastic.NewRangeQuery("Timestamp").Gte("now-1h"), ) myQuery, err = computeQuery(queryInfo) assert.Nil(t, err) assert.Equal(t, realQuery, myQuery) queryInfo = &config.QueryInfo{ Type: "boolquery", Clauses: map[string]interface{}{ "must_not": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"Type", "plop"}}, }, "filter": []interface{}{ map[interface{}]interface{}{"term": []interface{}{"Method"}}, //missing 2nd parameter }, }, } myQuery, err = computeQuery(queryInfo) assert.NotNil(t, err) }
func Example() { errorlog := log.New(os.Stdout, "APP ", log.LstdFlags) // Obtain a client. You can also provide your own HTTP client here. client, err := elastic.NewClient(elastic.SetErrorLog(errorlog)) if err != nil { // Handle error panic(err) } // Trace request and response details like this //client.SetTracer(log.New(os.Stdout, "", 0)) // Ping the Elasticsearch server to get e.g. the version number info, code, err := client.Ping("http://127.0.0.1:9200").Do() if err != nil { // Handle error panic(err) } fmt.Printf("Elasticsearch returned with code %d and version %s", code, info.Version.Number) // Getting the ES version number is quite common, so there's a shortcut esversion, err := client.ElasticsearchVersion("http://127.0.0.1:9200") if err != nil { // Handle error panic(err) } fmt.Printf("Elasticsearch version %s", esversion) // Use the IndexExists service to check if a specified index exists. exists, err := client.IndexExists("twitter").Do() if err != nil { // Handle error panic(err) } if !exists { // Create a new index. createIndex, err := client.CreateIndex("twitter").Do() if err != nil { // Handle error panic(err) } if !createIndex.Acknowledged { // Not acknowledged } } // Index a tweet (using JSON serialization) tweet1 := Tweet{User: "******", Message: "Take Five", Retweets: 0} put1, err := client.Index(). Index("twitter"). Type("tweet"). Id("1"). BodyJson(tweet1). Do() if err != nil { // Handle error panic(err) } fmt.Printf("Indexed tweet %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type) // Index a second tweet (by string) tweet2 := `{"user" : "olivere", "message" : "It's a Raggy Waltz"}` put2, err := client.Index(). Index("twitter"). Type("tweet"). Id("2"). BodyString(tweet2). Do() if err != nil { // Handle error panic(err) } fmt.Printf("Indexed tweet %s to index %s, type %s\n", put2.Id, put2.Index, put2.Type) // Get tweet with specified ID get1, err := client.Get(). Index("twitter"). Type("tweet"). Id("1"). Do() if err != nil { // Handle error panic(err) } if get1.Found { fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type) } // Flush to make sure the documents got written. _, err = client.Flush().Index("twitter").Do() if err != nil { panic(err) } // Search with a term query termQuery := elastic.NewTermQuery("user", "olivere") searchResult, err := client.Search(). Index("twitter"). // search in index "twitter" Query(termQuery). // specify the query Sort("user", true). // sort by "user" field, ascending From(0).Size(10). // take documents 0-9 Pretty(true). // pretty print request and response JSON Do() // execute if err != nil { // Handle error panic(err) } // searchResult is of type SearchResult and returns hits, suggestions, // and all kinds of other information from Elasticsearch. fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis) // Each is a convenience function that iterates over hits in a search result. // It makes sure you don't need to check for nil values in the response. // However, it ignores errors in serialization. If you want full control // over iterating the hits, see below. var ttyp Tweet for _, item := range searchResult.Each(reflect.TypeOf(ttyp)) { t := item.(Tweet) fmt.Printf("Tweet by %s: %s\n", t.User, t.Message) } // TotalHits is another convenience function that works even when something goes wrong. fmt.Printf("Found a total of %d tweets\n", searchResult.TotalHits()) // Here's how you iterate through results with full control over each step. if searchResult.Hits.TotalHits > 0 { fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits) // Iterate through results for _, hit := range searchResult.Hits.Hits { // hit.Index contains the name of the index // Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}). var t Tweet err := json.Unmarshal(*hit.Source, &t) if err != nil { // Deserialization failed } // Work with tweet fmt.Printf("Tweet by %s: %s\n", t.User, t.Message) } } else { // No hits fmt.Print("Found no tweets\n") } // Update a tweet by the update API of Elasticsearch. // We just increment the number of retweets. script := elastic.NewScript("ctx._source.retweets += num").Param("num", 1) update, err := client.Update().Index("twitter").Type("tweet").Id("1"). Script(script). Upsert(map[string]interface{}{"retweets": 0}). Do() if err != nil { // Handle error panic(err) } fmt.Printf("New version of tweet %q is now %d", update.Id, update.Version) // ... // Delete an index. deleteIndex, err := client.DeleteIndex("twitter").Do() if err != nil { // Handle error panic(err) } if !deleteIndex.Acknowledged { // Not acknowledged } }