func (e *Elastic) GetAnnotations(start, end *time.Time, source, host, creationUser, owner, category string) (annotate.Annotations, error) {
annotations := annotate.Annotations{}
s := elastic.NewSearchSource()
if start != nil && end != nil {
startQ := elastic.NewRangeQuery(annotate.EndDate).Gte(start)
endQ := elastic.NewRangeQuery(annotate.StartDate).Lte(end)
s = s.Query(elastic.NewBoolQuery().Must(startQ, endQ))
}
if source != "" {
s = s.Query(elastic.NewTermQuery(annotate.Source, source))
}
if host != "" {
s = s.Query(elastic.NewTermQuery(annotate.Host, host))
}
if creationUser != "" {
s = s.Query(elastic.NewTermQuery(annotate.CreationUser, creationUser))
}
if owner != "" {
s = s.Query(elastic.NewTermQuery(annotate.Owner, owner))
}
if category != "" {
s = s.Query(elastic.NewTermQuery(annotate.Category, category))
}
res, err := e.Search(e.index).Query(s).Size(e.maxResults).Do()
if err != nil {
return annotations, err
}
var aType annotate.Annotation
for _, item := range res.Each(reflect.TypeOf(aType)) {
a := item.(annotate.Annotation)
annotations = append(annotations, a)
}
return annotations, nil
}
func (e *Elastic) GetAnnotations(start, end *time.Time, fieldFilters ...FieldFilter) (annotate.Annotations, error) {
if !e.initialized {
return nil, unInitErr
}
annotations := annotate.Annotations{}
filters := []elastic.Query{}
if start != nil && end != nil {
startQ := elastic.NewRangeQuery(annotate.EndDate).Gte(start)
endQ := elastic.NewRangeQuery(annotate.StartDate).Lte(end)
filters = append(filters, elastic.NewBoolQuery().Must(startQ, endQ))
}
for _, filter := range fieldFilters {
switch filter.Field {
case annotate.Source, annotate.Host, annotate.CreationUser, annotate.Owner, annotate.Category:
default:
return annotations, fmt.Errorf("%v is not a field that can be filtered on", filter.Field)
}
var q elastic.Query
switch filter.Verb {
case Is, "":
q = elastic.NewTermQuery(filter.Field, filter.Value)
case Empty:
// Can't detect empty on a analyzed field
if filter.Field == annotate.Message {
return annotations, fmt.Errorf("message field does not support empty searches")
}
q = elastic.NewTermQuery(filter.Field, "")
default:
return annotations, fmt.Errorf("%v is not a valid query verb", filter.Verb)
}
if filter.Not {
q = elastic.NewBoolQuery().MustNot(q)
}
filters = append(filters, q)
}
var aType annotate.Annotation
scroll := e.Scroll(e.index).Query(elastic.NewBoolQuery().Must(filters...)).Size(e.maxResults).Pretty(true)
for {
res, err := scroll.Do()
if err == io.EOF {
break
}
if err != nil {
return annotations, err
}
for _, item := range res.Each(reflect.TypeOf(aType)) {
a := item.(annotate.Annotation)
annotations = append(annotations, a)
}
}
return annotations, nil
}
// FindGeoFencesExceptByUser returns all geofences around a lat/lon pair, excluding ones from the specified user.
func FindGeoFencesExceptByUser(centerLat float64, centerLon float64, radius int64, excludeBy int) ([]models.Fence, error) {
query := elastic.NewBoolQuery()
query = query.MustNot(elastic.NewTermQuery("owner", excludeBy))
query.Filter(elastic.NewGeoDistanceQuery("center").Distance(fmt.Sprintf("%d m", radius)).Lat(centerLat).Lon(centerLon))
searchResult, err := ElasticInstance.Search().
Index(IndexGeoFences).
Query(query).
Do()
// Check whether an error appeared or not.
if err != nil {
return nil, err
}
if searchResult.Hits != nil {
fences := make([]int64, searchResult.TotalHits(), searchResult.TotalHits())
fmt.Printf("Found a total of %d GeoFences\n", searchResult.Hits.TotalHits)
// Iterate through results
for i, hit := range searchResult.Hits.Hits {
stringID, _ := strconv.ParseInt(hit.Id, 10, 64)
fences[i] = stringID
}
return models.FindFencesByIDs(fences)
}
fmt.Print("Found no fences\n")
var empty []models.Fence
return empty, nil
}
func ScopeES(ts opentsdb.TagSet, q elastic.Query) elastic.Query {
var filters []elastic.Query
for tagKey, tagValue := range ts {
filters = append(filters, elastic.NewTermQuery(tagKey, tagValue))
}
filters = append(filters, q)
b := elastic.NewBoolQuery().Must(filters...)
return b
}
func ExampleSearchService() {
// Get a client to the local Elasticsearch instance.
client, err := elastic.NewClient()
if err != nil {
// Handle error
panic(err)
}
// Search with a term query
termQuery := elastic.NewTermQuery("user", "olivere")
searchResult, err := client.Search().
Index("twitter"). // search in index "twitter"
Query(termQuery). // specify the query
Sort("user", true). // sort by "user" field, ascending
From(0).Size(10). // take documents 0-9
Pretty(true). // pretty print request and response JSON
Do() // execute
if err != nil {
// Handle error
panic(err)
}
// searchResult is of type SearchResult and returns hits, suggestions,
// and all kinds of other information from Elasticsearch.
fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)
// Number of hits
if searchResult.Hits.TotalHits > 0 {
fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)
// Iterate through results
for _, hit := range searchResult.Hits.Hits {
// hit.Index contains the name of the index
// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
var t Tweet
err := json.Unmarshal(*hit.Source, &t)
if err != nil {
// Deserialization failed
}
// Work with tweet
fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
}
} else {
// No hits
fmt.Print("Found no tweets\n")
}
}
func TestStringtoNb(t *testing.T) {
t1 := "40.7"
t2 := 3.3
t3 := "100"
t4 := "nothing to do here"
t5 := "now-30m"
t6 := "100mille"
r1 := stringToNb(t1)
r2 := stringToNb(t2)
r3 := stringToNb(t3)
r4 := stringToNb(t4)
r5 := stringToNb(t5)
r6 := stringToNb(t6)
assert.Equal(t, 40.7, r1)
assert.Equal(t, 3.3, r2)
assert.Equal(t, 100, r3)
assert.Equal(t, "nothing to do here", r4)
assert.Equal(t, "now-30m", r5)
assert.Equal(t, "100mille", r6)
query1 := elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Thomas"),
).Must(
elastic.NewTermQuery("Status", stringToNb("500")),
elastic.NewRangeQuery("Code").Gt(stringToNb("42.42")),
)
queryWrong := elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Thomas"),
).Must(
elastic.NewTermQuery("Status", "500"),
elastic.NewRangeQuery("Code").Gt("42.42"),
)
query2 := elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Thomas"),
).Must(
elastic.NewTermQuery("Status", 500),
elastic.NewRangeQuery("Code").Gt(42.42),
)
assert.NotEqual(t, queryWrong, query2)
assert.Equal(t, query1, query2)
}
func (t *TestCase) search() {
// Loop forever to check for connection issues
for {
// Get tweet with specified ID
get1, err := t.client.Get().
Index(t.index).
Type("tweet").
Id("1").
Do()
if err != nil {
//failf("Get failed: %v", err)
t.runCh <- RunInfo{Success: false}
continue
}
if !get1.Found {
//log.Printf("Document %s not found\n", "1")
//fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type)
t.runCh <- RunInfo{Success: false}
continue
}
// Search with a term query
searchResult, err := t.client.Search().
Index(t.index). // search in index t.index
Query(elastic.NewTermQuery("user", "olivere")). // specify the query
Sort("user", true). // sort by "user" field, ascending
From(0).Size(10). // take documents 0-9
Pretty(true). // pretty print request and response JSON
Do() // execute
if err != nil {
//failf("Search failed: %v\n", err)
t.runCh <- RunInfo{Success: false}
continue
}
// searchResult is of type SearchResult and returns hits, suggestions,
// and all kinds of other information from Elasticsearch.
//fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)
// Number of hits
if searchResult.Hits.TotalHits > 0 {
//fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)
// Iterate through results
for _, hit := range searchResult.Hits.Hits {
// hit.Index contains the name of the index
// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
var tweet Tweet
err := json.Unmarshal(*hit.Source, &tweet)
if err != nil {
// Deserialization failed
//failf("Deserialize failed: %v\n", err)
t.runCh <- RunInfo{Success: false}
continue
}
// Work with tweet
//fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
}
} else {
// No hits
//fmt.Print("Found no tweets\n")
}
t.runCh <- RunInfo{Success: true}
// Sleep some time
time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
}
}
func Test_getQueries(t *testing.T) {
eslog.InitSilent()
// term filters
var filters []interface{} = []interface{}{
map[interface{}]interface{}{"term": []interface{}{"test", "yes"}},
map[interface{}]interface{}{"term": []interface{}{"required", true}},
}
realfilters := []elastic.Query{
elastic.NewTermQuery("test", "yes"),
elastic.NewTermQuery("required", true),
}
testfilters, err := getQueries(filters)
assert.Equal(t, err, nil, "Should be nil")
assert.Equal(t, realfilters, testfilters)
realfilters = []elastic.Query{
elastic.NewTermQuery("test", "yes"),
elastic.NewTermQuery("required", false),
}
assert.NotEqual(t, realfilters, testfilters)
//range filters
filters = []interface{}{
map[interface{}]interface{}{"term": []interface{}{"value", 146}},
map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte", "now-1h"}},
}
realfilters = []elastic.Query{
elastic.NewTermQuery("value", 146),
elastic.NewTermQuery("othervalue", "testTest"),
elastic.NewRangeQuery("Timestamp").Gte("now-1h"),
}
testfilters, err = getQueries(filters)
assert.Equal(t, err, nil, "Should be nil")
assert.Equal(t, realfilters, testfilters)
realfilters = []elastic.Query{
elastic.NewTermQuery("value", 146),
elastic.NewTermQuery("othervalue", "testTest"),
elastic.NewRangeQuery("Timestamp").Lt("now-1h"),
}
assert.NotEqual(t, realfilters, testfilters)
filters = []interface{}{
map[interface{}]interface{}{"term": []interface{}{"value", 146}},
map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "lt", "now-1h"}},
}
testfilters, err = getQueries(filters)
assert.Equal(t, err, nil, "Should be nil")
assert.Equal(t, realfilters, testfilters)
//non valid fields
filters = []interface{}{
map[interface{}]interface{}{"term": []interface{}{145, "yes"}},
map[interface{}]interface{}{"term": []interface{}{"required", "yes"}},
}
testfilters, err = getQueries(filters)
assert.NotNil(t, err)
filters = []interface{}{
map[interface{}]interface{}{"term": []interface{}{"yes"}},
map[interface{}]interface{}{"term": []interface{}{"required", 28}},
}
testfilters, err = getQueries(filters)
assert.NotNil(t, err)
filters = []interface{}{
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte"}},
map[interface{}]interface{}{"term": []interface{}{"required", 28}},
}
testfilters, err = getQueries(filters)
assert.NotNil(t, err)
filters = []interface{}{
map[interface{}]interface{}{"range": []interface{}{42, "gte", "now-30m"}},
map[interface{}]interface{}{"term": []interface{}{"required", 28}},
}
testfilters, err = getQueries(filters)
assert.NotNil(t, err)
}
func Test_boolquery(t *testing.T) {
eslog.InitSilent()
info := &config.QueryInfo{}
myQuery, err := computeQuery(info)
assert.NotNil(t, err)
queryInfo := &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"must": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Value", 146.5}},
map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "lt", "now-1h"}},
},
"must_not": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"status", "OK"}},
},
},
}
realQuery := elastic.NewBoolQuery().Must(
elastic.NewTermQuery("Value", 146.5),
elastic.NewTermQuery("othervalue", "testTest"),
elastic.NewRangeQuery("Timestamp").Lt("now-1h"),
).MustNot(
elastic.NewTermQuery("status", "OK"),
)
myQuery, err = computeQuery(queryInfo)
assert.Nil(t, err)
assert.Equal(t, realQuery, myQuery)
queryInfo = &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"must_not": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Status", "Error"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gt", "now-30m"}},
},
},
}
realQuery = elastic.NewBoolQuery().MustNot(
elastic.NewTermQuery("Status", "Error"),
).MustNot(
elastic.NewRangeQuery("Timestamp").Gt("now-30m"),
)
myQuery, err = computeQuery(queryInfo)
assert.Nil(t, err)
assert.Equal(t, realQuery, myQuery)
queryInfo = &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"should": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"User", "Thomas"}},
},
"must": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gt", "now-2h"}},
},
},
}
realQuery = elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Thomas"),
).Must(
elastic.NewTermQuery("Status", "OK"),
elastic.NewRangeQuery("Timestamp").Gt("now-2h"),
)
myQuery, err = computeQuery(queryInfo)
assert.Nil(t, err)
assert.Equal(t, realQuery, myQuery)
realQuery = elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Tobias"),
).Must(
elastic.NewTermQuery("Status", "OK"),
elastic.NewRangeQuery("Timestamp").Gt("now-2h"),
)
assert.NotEqual(t, realQuery, myQuery)
queryInfo = &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"should": []interface{}{
map[interface{}]interface{}{"plop": []interface{}{"Thomas"}},
},
"must": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
map[interface{}]interface{}{"hihi": []interface{}{"Timestamp", "gt", "now-2h"}},
},
},
}
myQuery, err = computeQuery(queryInfo)
assert.NotNil(t, err)
// test filterQueries
queryInfo = &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"should": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"User", "Thomas"}},
},
"must": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
},
"filter": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Method", "XPUT"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte", "now-1h"}},
},
},
}
realQuery = elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Thomas"),
).Must(
elastic.NewTermQuery("Status", "OK"),
).Filter(
elastic.NewTermQuery("Method", "XPUT"),
elastic.NewRangeQuery("Timestamp").Gte("now-1h"),
)
myQuery, err = computeQuery(queryInfo)
assert.Nil(t, err)
assert.Equal(t, realQuery, myQuery)
queryInfo = &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"must_not": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Type", "plop"}},
},
"filter": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Method"}}, //missing 2nd parameter
},
},
}
myQuery, err = computeQuery(queryInfo)
assert.NotNil(t, err)
}
func Example() {
errorlog := log.New(os.Stdout, "APP ", log.LstdFlags)
// Obtain a client. You can also provide your own HTTP client here.
client, err := elastic.NewClient(elastic.SetErrorLog(errorlog))
if err != nil {
// Handle error
panic(err)
}
// Trace request and response details like this
//client.SetTracer(log.New(os.Stdout, "", 0))
// Ping the Elasticsearch server to get e.g. the version number
info, code, err := client.Ping("http://127.0.0.1:9200").Do()
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Elasticsearch returned with code %d and version %s", code, info.Version.Number)
// Getting the ES version number is quite common, so there's a shortcut
esversion, err := client.ElasticsearchVersion("http://127.0.0.1:9200")
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Elasticsearch version %s", esversion)
// Use the IndexExists service to check if a specified index exists.
exists, err := client.IndexExists("twitter").Do()
if err != nil {
// Handle error
panic(err)
}
if !exists {
// Create a new index.
createIndex, err := client.CreateIndex("twitter").Do()
if err != nil {
// Handle error
panic(err)
}
if !createIndex.Acknowledged {
// Not acknowledged
}
}
// Index a tweet (using JSON serialization)
tweet1 := Tweet{User: "******", Message: "Take Five", Retweets: 0}
put1, err := client.Index().
Index("twitter").
Type("tweet").
Id("1").
BodyJson(tweet1).
Do()
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Indexed tweet %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type)
// Index a second tweet (by string)
tweet2 := `{"user" : "olivere", "message" : "It's a Raggy Waltz"}`
put2, err := client.Index().
Index("twitter").
Type("tweet").
Id("2").
BodyString(tweet2).
Do()
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Indexed tweet %s to index %s, type %s\n", put2.Id, put2.Index, put2.Type)
// Get tweet with specified ID
get1, err := client.Get().
Index("twitter").
Type("tweet").
Id("1").
Do()
if err != nil {
// Handle error
panic(err)
}
if get1.Found {
fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type)
}
// Flush to make sure the documents got written.
_, err = client.Flush().Index("twitter").Do()
if err != nil {
panic(err)
}
// Search with a term query
termQuery := elastic.NewTermQuery("user", "olivere")
searchResult, err := client.Search().
Index("twitter"). // search in index "twitter"
Query(termQuery). // specify the query
Sort("user", true). // sort by "user" field, ascending
From(0).Size(10). // take documents 0-9
Pretty(true). // pretty print request and response JSON
Do() // execute
if err != nil {
// Handle error
panic(err)
}
// searchResult is of type SearchResult and returns hits, suggestions,
// and all kinds of other information from Elasticsearch.
fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)
// Each is a convenience function that iterates over hits in a search result.
// It makes sure you don't need to check for nil values in the response.
// However, it ignores errors in serialization. If you want full control
// over iterating the hits, see below.
var ttyp Tweet
for _, item := range searchResult.Each(reflect.TypeOf(ttyp)) {
t := item.(Tweet)
fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
}
// TotalHits is another convenience function that works even when something goes wrong.
fmt.Printf("Found a total of %d tweets\n", searchResult.TotalHits())
// Here's how you iterate through results with full control over each step.
if searchResult.Hits.TotalHits > 0 {
fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)
// Iterate through results
for _, hit := range searchResult.Hits.Hits {
// hit.Index contains the name of the index
// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
var t Tweet
err := json.Unmarshal(*hit.Source, &t)
if err != nil {
// Deserialization failed
}
// Work with tweet
fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
}
} else {
// No hits
fmt.Print("Found no tweets\n")
}
// Update a tweet by the update API of Elasticsearch.
// We just increment the number of retweets.
script := elastic.NewScript("ctx._source.retweets += num").Param("num", 1)
update, err := client.Update().Index("twitter").Type("tweet").Id("1").
Script(script).
Upsert(map[string]interface{}{"retweets": 0}).
Do()
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("New version of tweet %q is now %d", update.Id, update.Version)
// ...
// Delete an index.
deleteIndex, err := client.DeleteIndex("twitter").Do()
if err != nil {
// Handle error
panic(err)
}
if !deleteIndex.Acknowledged {
// Not acknowledged
}
}
