func (e *Elastic) GetAnnotations(start, end *time.Time, source, host, creationUser, owner, category string) (annotate.Annotations, error) {
annotations := annotate.Annotations{}
s := elastic.NewSearchSource()
if start != nil && end != nil {
startQ := elastic.NewRangeQuery(annotate.EndDate).Gte(start)
endQ := elastic.NewRangeQuery(annotate.StartDate).Lte(end)
s = s.Query(elastic.NewBoolQuery().Must(startQ, endQ))
}
if source != "" {
s = s.Query(elastic.NewTermQuery(annotate.Source, source))
}
if host != "" {
s = s.Query(elastic.NewTermQuery(annotate.Host, host))
}
if creationUser != "" {
s = s.Query(elastic.NewTermQuery(annotate.CreationUser, creationUser))
}
if owner != "" {
s = s.Query(elastic.NewTermQuery(annotate.Owner, owner))
}
if category != "" {
s = s.Query(elastic.NewTermQuery(annotate.Category, category))
}
res, err := e.Search(e.index).Query(s).Size(e.maxResults).Do()
if err != nil {
return annotations, err
}
var aType annotate.Annotation
for _, item := range res.Each(reflect.TypeOf(aType)) {
a := item.(annotate.Annotation)
annotations = append(annotations, a)
}
return annotations, nil
}
func (e *Elastic) GetAnnotations(start, end *time.Time, fieldFilters ...FieldFilter) (annotate.Annotations, error) {
if !e.initialized {
return nil, unInitErr
}
annotations := annotate.Annotations{}
filters := []elastic.Query{}
if start != nil && end != nil {
startQ := elastic.NewRangeQuery(annotate.EndDate).Gte(start)
endQ := elastic.NewRangeQuery(annotate.StartDate).Lte(end)
filters = append(filters, elastic.NewBoolQuery().Must(startQ, endQ))
}
for _, filter := range fieldFilters {
switch filter.Field {
case annotate.Source, annotate.Host, annotate.CreationUser, annotate.Owner, annotate.Category:
default:
return annotations, fmt.Errorf("%v is not a field that can be filtered on", filter.Field)
}
var q elastic.Query
switch filter.Verb {
case Is, "":
q = elastic.NewTermQuery(filter.Field, filter.Value)
case Empty:
// Can't detect empty on a analyzed field
if filter.Field == annotate.Message {
return annotations, fmt.Errorf("message field does not support empty searches")
}
q = elastic.NewTermQuery(filter.Field, "")
default:
return annotations, fmt.Errorf("%v is not a valid query verb", filter.Verb)
}
if filter.Not {
q = elastic.NewBoolQuery().MustNot(q)
}
filters = append(filters, q)
}
var aType annotate.Annotation
scroll := e.Scroll(e.index).Query(elastic.NewBoolQuery().Must(filters...)).Size(e.maxResults).Pretty(true)
for {
res, err := scroll.Do()
if err == io.EOF {
break
}
if err != nil {
return annotations, err
}
for _, item := range res.Each(reflect.TypeOf(aType)) {
a := item.(annotate.Annotation)
annotations = append(annotations, a)
}
}
return annotations, nil
}
func Test_getRangeQuery(t *testing.T) {
eslog.InitSilent()
var result elastic.Query
var err error
test := []interface{}{"errorcode", "gte", "500"}
real := elastic.NewRangeQuery("errorcode").Gte(500)
result, err = getRangeQuery(test)
assert.Equal(t, err, nil)
assert.Equal(t, real, result)
test = []interface{}{"timestamp", "lt", "now-1h"}
real = elastic.NewRangeQuery("timestamp").Lt("now-1h")
result, err = getRangeQuery(test)
assert.Equal(t, err, nil, "Should be nil")
assert.Equal(t, real, result)
test = []interface{}{"code", "lt", "800", "gte", "500"}
real = elastic.NewRangeQuery("code").Lt(800).Gte(500)
result, err = getRangeQuery(test)
assert.Equal(t, err, nil, "Should be nil")
assert.Equal(t, real, result)
test = []interface{}{"timestamp", "lt", "now-1h", "gte", "2d"}
real = elastic.NewRangeQuery("timestamp").Lt("now-1h").Gte("2d")
result, err = getRangeQuery(test)
assert.Equal(t, err, nil, "Should be nil")
assert.Equal(t, real, result)
test = []interface{}{"code", "lt", "800", "pouet", "500"}
result, err = getRangeQuery(test)
assert.NotNil(t, err)
test = []interface{}{"code", "lt", "800", "gte"}
result, err = getRangeQuery(test)
assert.NotNil(t, err)
test = []interface{}{"test", "lt", "800", "tge", "500"}
result, err = getRangeQuery(test)
assert.NotNil(t, err)
test = []interface{}{"testshort"}
result, err = getRangeQuery(test)
assert.NotNil(t, err)
test = []interface{}{}
result, err = getRangeQuery(test)
assert.NotNil(t, err)
}
// ESBaseQuery builds the base query that both ESCount and ESStat share
func ESBaseQuery(now time.Time, indexer ESIndexer, l ElasticHosts, filter elastic.Query, sduration, eduration string, size int) (*ElasticRequest, error) {
start, err := opentsdb.ParseDuration(sduration)
if err != nil {
return nil, err
}
var end opentsdb.Duration
if eduration != "" {
end, err = opentsdb.ParseDuration(eduration)
if err != nil {
return nil, err
}
}
st := now.Add(time.Duration(-start))
en := now.Add(time.Duration(-end))
r := ElasticRequest{
Indexer: indexer,
Start: &st,
End: &en,
Source: elastic.NewSearchSource().Size(size),
}
var q elastic.Query
q = elastic.NewRangeQuery(indexer.TimeField).Gte(st).Lte(en)
r.Source = r.Source.Query(elastic.NewBoolQuery().Must(q, filter))
return &r, nil
}
func ESLTE(e *State, T miniprofiler.Timer, key string, lte float64) (*Results, error) {
var r Results
q := ESQuery{
Query: elastic.NewRangeQuery(key).Lte(lte),
}
r.Results = append(r.Results, &Result{Value: q})
return &r, nil
}
func TestStringtoNb(t *testing.T) {
t1 := "40.7"
t2 := 3.3
t3 := "100"
t4 := "nothing to do here"
t5 := "now-30m"
t6 := "100mille"
r1 := stringToNb(t1)
r2 := stringToNb(t2)
r3 := stringToNb(t3)
r4 := stringToNb(t4)
r5 := stringToNb(t5)
r6 := stringToNb(t6)
assert.Equal(t, 40.7, r1)
assert.Equal(t, 3.3, r2)
assert.Equal(t, 100, r3)
assert.Equal(t, "nothing to do here", r4)
assert.Equal(t, "now-30m", r5)
assert.Equal(t, "100mille", r6)
query1 := elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Thomas"),
).Must(
elastic.NewTermQuery("Status", stringToNb("500")),
elastic.NewRangeQuery("Code").Gt(stringToNb("42.42")),
)
queryWrong := elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Thomas"),
).Must(
elastic.NewTermQuery("Status", "500"),
elastic.NewRangeQuery("Code").Gt("42.42"),
)
query2 := elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Thomas"),
).Must(
elastic.NewTermQuery("Status", 500),
elastic.NewRangeQuery("Code").Gt(42.42),
)
assert.NotEqual(t, queryWrong, query2)
assert.Equal(t, query1, query2)
}
func Test_getQueries(t *testing.T) {
eslog.InitSilent()
// term filters
var filters []interface{} = []interface{}{
map[interface{}]interface{}{"term": []interface{}{"test", "yes"}},
map[interface{}]interface{}{"term": []interface{}{"required", true}},
}
realfilters := []elastic.Query{
elastic.NewTermQuery("test", "yes"),
elastic.NewTermQuery("required", true),
}
testfilters, err := getQueries(filters)
assert.Equal(t, err, nil, "Should be nil")
assert.Equal(t, realfilters, testfilters)
realfilters = []elastic.Query{
elastic.NewTermQuery("test", "yes"),
elastic.NewTermQuery("required", false),
}
assert.NotEqual(t, realfilters, testfilters)
//range filters
filters = []interface{}{
map[interface{}]interface{}{"term": []interface{}{"value", 146}},
map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte", "now-1h"}},
}
realfilters = []elastic.Query{
elastic.NewTermQuery("value", 146),
elastic.NewTermQuery("othervalue", "testTest"),
elastic.NewRangeQuery("Timestamp").Gte("now-1h"),
}
testfilters, err = getQueries(filters)
assert.Equal(t, err, nil, "Should be nil")
assert.Equal(t, realfilters, testfilters)
realfilters = []elastic.Query{
elastic.NewTermQuery("value", 146),
elastic.NewTermQuery("othervalue", "testTest"),
elastic.NewRangeQuery("Timestamp").Lt("now-1h"),
}
assert.NotEqual(t, realfilters, testfilters)
filters = []interface{}{
map[interface{}]interface{}{"term": []interface{}{"value", 146}},
map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "lt", "now-1h"}},
}
testfilters, err = getQueries(filters)
assert.Equal(t, err, nil, "Should be nil")
assert.Equal(t, realfilters, testfilters)
//non valid fields
filters = []interface{}{
map[interface{}]interface{}{"term": []interface{}{145, "yes"}},
map[interface{}]interface{}{"term": []interface{}{"required", "yes"}},
}
testfilters, err = getQueries(filters)
assert.NotNil(t, err)
filters = []interface{}{
map[interface{}]interface{}{"term": []interface{}{"yes"}},
map[interface{}]interface{}{"term": []interface{}{"required", 28}},
}
testfilters, err = getQueries(filters)
assert.NotNil(t, err)
filters = []interface{}{
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte"}},
map[interface{}]interface{}{"term": []interface{}{"required", 28}},
}
testfilters, err = getQueries(filters)
assert.NotNil(t, err)
filters = []interface{}{
map[interface{}]interface{}{"range": []interface{}{42, "gte", "now-30m"}},
map[interface{}]interface{}{"term": []interface{}{"required", 28}},
}
testfilters, err = getQueries(filters)
assert.NotNil(t, err)
}
func Test_boolquery(t *testing.T) {
eslog.InitSilent()
info := &config.QueryInfo{}
myQuery, err := computeQuery(info)
assert.NotNil(t, err)
queryInfo := &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"must": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Value", 146.5}},
map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "lt", "now-1h"}},
},
"must_not": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"status", "OK"}},
},
},
}
realQuery := elastic.NewBoolQuery().Must(
elastic.NewTermQuery("Value", 146.5),
elastic.NewTermQuery("othervalue", "testTest"),
elastic.NewRangeQuery("Timestamp").Lt("now-1h"),
).MustNot(
elastic.NewTermQuery("status", "OK"),
)
myQuery, err = computeQuery(queryInfo)
assert.Nil(t, err)
assert.Equal(t, realQuery, myQuery)
queryInfo = &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"must_not": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Status", "Error"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gt", "now-30m"}},
},
},
}
realQuery = elastic.NewBoolQuery().MustNot(
elastic.NewTermQuery("Status", "Error"),
).MustNot(
elastic.NewRangeQuery("Timestamp").Gt("now-30m"),
)
myQuery, err = computeQuery(queryInfo)
assert.Nil(t, err)
assert.Equal(t, realQuery, myQuery)
queryInfo = &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"should": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"User", "Thomas"}},
},
"must": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gt", "now-2h"}},
},
},
}
realQuery = elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Thomas"),
).Must(
elastic.NewTermQuery("Status", "OK"),
elastic.NewRangeQuery("Timestamp").Gt("now-2h"),
)
myQuery, err = computeQuery(queryInfo)
assert.Nil(t, err)
assert.Equal(t, realQuery, myQuery)
realQuery = elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Tobias"),
).Must(
elastic.NewTermQuery("Status", "OK"),
elastic.NewRangeQuery("Timestamp").Gt("now-2h"),
)
assert.NotEqual(t, realQuery, myQuery)
queryInfo = &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"should": []interface{}{
map[interface{}]interface{}{"plop": []interface{}{"Thomas"}},
},
"must": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
map[interface{}]interface{}{"hihi": []interface{}{"Timestamp", "gt", "now-2h"}},
},
},
}
myQuery, err = computeQuery(queryInfo)
assert.NotNil(t, err)
// test filterQueries
queryInfo = &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"should": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"User", "Thomas"}},
},
"must": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
},
"filter": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Method", "XPUT"}},
map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte", "now-1h"}},
},
},
}
realQuery = elastic.NewBoolQuery().Should(
elastic.NewTermQuery("User", "Thomas"),
).Must(
elastic.NewTermQuery("Status", "OK"),
).Filter(
elastic.NewTermQuery("Method", "XPUT"),
elastic.NewRangeQuery("Timestamp").Gte("now-1h"),
)
myQuery, err = computeQuery(queryInfo)
assert.Nil(t, err)
assert.Equal(t, realQuery, myQuery)
queryInfo = &config.QueryInfo{
Type: "boolquery",
Clauses: map[string]interface{}{
"must_not": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Type", "plop"}},
},
"filter": []interface{}{
map[interface{}]interface{}{"term": []interface{}{"Method"}}, //missing 2nd parameter
},
},
}
myQuery, err = computeQuery(queryInfo)
assert.NotNil(t, err)
}
// Searches for mssages
func (e *Elastic) GetMessages(q *storage.SearchQuery) (result *storage.SearchResult, err error) {
var (
filters []es.Query = []es.Query{}
query *es.BoolQuery = es.NewBoolQuery()
tsRange *es.RangeQuery
msg *storage.Message
)
if len(q.Query) > 0 {
filters = append(filters, es.NewQueryStringQuery(q.Query))
} else {
filters = append(filters, es.NewMatchAllQuery())
}
if q.From.Second() > 0 && q.To.Second() > 0 {
tsRange = es.NewRangeQuery("timestamp")
tsRange = tsRange.From(q.From)
tsRange = tsRange.To(q.To)
filters = append(filters, tsRange)
} else if q.From.Second() > 0 && q.To.Second() <= 0 {
tsRange = es.NewRangeQuery("timestamp")
tsRange = tsRange.From(q.From)
filters = append(filters, tsRange)
} else if q.To.Second() > 0 && q.From.Second() <= 0 {
tsRange = es.NewRangeQuery("timestamp")
tsRange = tsRange.To(q.To)
filters = append(filters, tsRange)
}
rs, err := e.client.
Search(e.indexName).
Type(e.typeName).
Query(query.Filter(filters...)).
Sort("timestamp", false).
From(q.Offset).
Size(q.Limit).
Do()
if err != nil {
return nil, err
}
result = &storage.SearchResult{
Total: rs.TotalHits(),
TookMs: rs.TookInMillis,
Limit: q.Limit,
Offset: q.Offset,
Messages: []storage.Message{},
}
// rs.Each() is not used because we need to add message id manually
if rs.Hits != nil && rs.Hits.Hits != nil && len(rs.Hits.Hits) > 0 {
for _, hit := range rs.Hits.Hits {
msg = new(storage.Message)
err = json.Unmarshal(*hit.Source, msg)
if err != nil {
continue
}
msg.Id = hit.Id
result.Messages = append(result.Messages, *msg)
}
}
return result, nil
}
