예제 #1
0
func (e *Elastic) GetAnnotations(start, end *time.Time, fieldFilters ...FieldFilter) (annotate.Annotations, error) {
	if !e.initialized {
		return nil, unInitErr
	}
	annotations := annotate.Annotations{}
	filters := []elastic.Query{}
	if start != nil && end != nil {
		startQ := elastic.NewRangeQuery(annotate.EndDate).Gte(start)
		endQ := elastic.NewRangeQuery(annotate.StartDate).Lte(end)
		filters = append(filters, elastic.NewBoolQuery().Must(startQ, endQ))
	}
	for _, filter := range fieldFilters {
		switch filter.Field {
		case annotate.Source, annotate.Host, annotate.CreationUser, annotate.Owner, annotate.Category:
		default:
			return annotations, fmt.Errorf("%v is not a field that can be filtered on", filter.Field)
		}
		var q elastic.Query
		switch filter.Verb {
		case Is, "":
			q = elastic.NewTermQuery(filter.Field, filter.Value)
		case Empty:
			// Can't detect empty on a analyzed field
			if filter.Field == annotate.Message {
				return annotations, fmt.Errorf("message field does not support empty searches")
			}
			q = elastic.NewTermQuery(filter.Field, "")
		default:
			return annotations, fmt.Errorf("%v is not a valid query verb", filter.Verb)
		}
		if filter.Not {
			q = elastic.NewBoolQuery().MustNot(q)
		}
		filters = append(filters, q)
	}

	var aType annotate.Annotation
	scroll := e.Scroll(e.index).Query(elastic.NewBoolQuery().Must(filters...)).Size(e.maxResults).Pretty(true)
	for {
		res, err := scroll.Do()
		if err == io.EOF {
			break
		}
		if err != nil {
			return annotations, err
		}
		for _, item := range res.Each(reflect.TypeOf(aType)) {
			a := item.(annotate.Annotation)
			annotations = append(annotations, a)
		}
	}
	return annotations, nil
}
예제 #2
0
파일: elastic.go 프로젝트: noblehng/bosun
// ESBaseQuery builds the base query that both ESCount and ESStat share
func ESBaseQuery(now time.Time, indexer ESIndexer, l ElasticHosts, filter elastic.Query, sduration, eduration string, size int) (*ElasticRequest, error) {
	start, err := opentsdb.ParseDuration(sduration)
	if err != nil {
		return nil, err
	}
	var end opentsdb.Duration
	if eduration != "" {
		end, err = opentsdb.ParseDuration(eduration)
		if err != nil {
			return nil, err
		}
	}
	st := now.Add(time.Duration(-start))
	en := now.Add(time.Duration(-end))
	r := ElasticRequest{
		Indexer: indexer,
		Start:   &st,
		End:     &en,
		Source:  elastic.NewSearchSource().Size(size),
	}
	var q elastic.Query
	q = elastic.NewRangeQuery(indexer.TimeField).Gte(st).Lte(en)
	r.Source = r.Source.Query(elastic.NewBoolQuery().Must(q, filter))
	return &r, nil
}
예제 #3
0
파일: backend.go 프로젝트: noblehng/bosun
func (e *Elastic) GetAnnotations(start, end *time.Time, source, host, creationUser, owner, category string) (annotate.Annotations, error) {
	annotations := annotate.Annotations{}
	s := elastic.NewSearchSource()
	if start != nil && end != nil {
		startQ := elastic.NewRangeQuery(annotate.EndDate).Gte(start)
		endQ := elastic.NewRangeQuery(annotate.StartDate).Lte(end)
		s = s.Query(elastic.NewBoolQuery().Must(startQ, endQ))
	}
	if source != "" {
		s = s.Query(elastic.NewTermQuery(annotate.Source, source))
	}
	if host != "" {
		s = s.Query(elastic.NewTermQuery(annotate.Host, host))
	}
	if creationUser != "" {
		s = s.Query(elastic.NewTermQuery(annotate.CreationUser, creationUser))
	}
	if owner != "" {
		s = s.Query(elastic.NewTermQuery(annotate.Owner, owner))
	}
	if category != "" {
		s = s.Query(elastic.NewTermQuery(annotate.Category, category))
	}
	res, err := e.Search(e.index).Query(s).Size(e.maxResults).Do()
	if err != nil {
		return annotations, err
	}
	var aType annotate.Annotation
	for _, item := range res.Each(reflect.TypeOf(aType)) {
		a := item.(annotate.Annotation)
		annotations = append(annotations, a)
	}
	return annotations, nil
}
예제 #4
0
// FindGeoFencesExceptByUser returns all geofences around a lat/lon pair, excluding ones from the specified user.
func FindGeoFencesExceptByUser(centerLat float64, centerLon float64, radius int64, excludeBy int) ([]models.Fence, error) {
	query := elastic.NewBoolQuery()
	query = query.MustNot(elastic.NewTermQuery("owner", excludeBy))
	query.Filter(elastic.NewGeoDistanceQuery("center").Distance(fmt.Sprintf("%d m", radius)).Lat(centerLat).Lon(centerLon))

	searchResult, err := ElasticInstance.Search().
		Index(IndexGeoFences).
		Query(query).
		Do()

	// Check whether an error appeared or not.
	if err != nil {
		return nil, err
	}

	if searchResult.Hits != nil {
		fences := make([]int64, searchResult.TotalHits(), searchResult.TotalHits())
		fmt.Printf("Found a total of %d GeoFences\n", searchResult.Hits.TotalHits)

		// Iterate through results
		for i, hit := range searchResult.Hits.Hits {
			stringID, _ := strconv.ParseInt(hit.Id, 10, 64)
			fences[i] = stringID
		}
		return models.FindFencesByIDs(fences)
	}

	fmt.Print("Found no fences\n")
	var empty []models.Fence
	return empty, nil
}
예제 #5
0
func ESNot(e *State, T miniprofiler.Timer, query ESQuery) (*Results, error) {
	var r Results
	q := ESQuery{
		Query: elastic.NewBoolQuery().MustNot(query.Query),
	}
	r.Results = append(r.Results, &Result{Value: q})
	return &r, nil
}
예제 #6
0
파일: elastic.go 프로젝트: noblehng/bosun
func ScopeES(ts opentsdb.TagSet, q elastic.Query) elastic.Query {
	var filters []elastic.Query
	for tagKey, tagValue := range ts {
		filters = append(filters, elastic.NewTermQuery(tagKey, tagValue))
	}
	filters = append(filters, q)
	b := elastic.NewBoolQuery().Must(filters...)
	return b
}
예제 #7
0
func TestStringtoNb(t *testing.T) {
	t1 := "40.7"
	t2 := 3.3
	t3 := "100"
	t4 := "nothing to do here"
	t5 := "now-30m"
	t6 := "100mille"

	r1 := stringToNb(t1)
	r2 := stringToNb(t2)
	r3 := stringToNb(t3)
	r4 := stringToNb(t4)
	r5 := stringToNb(t5)
	r6 := stringToNb(t6)
	assert.Equal(t, 40.7, r1)
	assert.Equal(t, 3.3, r2)
	assert.Equal(t, 100, r3)
	assert.Equal(t, "nothing to do here", r4)
	assert.Equal(t, "now-30m", r5)
	assert.Equal(t, "100mille", r6)

	query1 := elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Thomas"),
	).Must(
		elastic.NewTermQuery("Status", stringToNb("500")),
		elastic.NewRangeQuery("Code").Gt(stringToNb("42.42")),
	)

	queryWrong := elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Thomas"),
	).Must(
		elastic.NewTermQuery("Status", "500"),
		elastic.NewRangeQuery("Code").Gt("42.42"),
	)

	query2 := elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Thomas"),
	).Must(
		elastic.NewTermQuery("Status", 500),
		elastic.NewRangeQuery("Code").Gt(42.42),
	)
	assert.NotEqual(t, queryWrong, query2)
	assert.Equal(t, query1, query2)
}
예제 #8
0
파일: elastic.go 프로젝트: noblehng/bosun
func ESOr(e *State, T miniprofiler.Timer, esqueries ...ESQuery) (*Results, error) {
	var r Results
	queries := make([]elastic.Query, len(esqueries))
	for i, q := range esqueries {
		queries[i] = q.Query
	}
	q := ESQuery{
		Query: elastic.NewBoolQuery().Should(queries...).MinimumNumberShouldMatch(1),
	}
	r.Results = append(r.Results, &Result{Value: q})
	return &r, nil
}
예제 #9
0
func Test_boolquery(t *testing.T) {
	eslog.InitSilent()
	info := &config.QueryInfo{}
	myQuery, err := computeQuery(info)
	assert.NotNil(t, err)

	queryInfo := &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"must": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Value", 146.5}},
				map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}},
				map[interface{}]interface{}{"range": []interface{}{"Timestamp", "lt", "now-1h"}},
			},
			"must_not": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"status", "OK"}},
			},
		},
	}
	realQuery := elastic.NewBoolQuery().Must(
		elastic.NewTermQuery("Value", 146.5),
		elastic.NewTermQuery("othervalue", "testTest"),
		elastic.NewRangeQuery("Timestamp").Lt("now-1h"),
	).MustNot(
		elastic.NewTermQuery("status", "OK"),
	)
	myQuery, err = computeQuery(queryInfo)
	assert.Nil(t, err)
	assert.Equal(t, realQuery, myQuery)

	queryInfo = &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"must_not": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Status", "Error"}},
				map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gt", "now-30m"}},
			},
		},
	}

	realQuery = elastic.NewBoolQuery().MustNot(
		elastic.NewTermQuery("Status", "Error"),
	).MustNot(
		elastic.NewRangeQuery("Timestamp").Gt("now-30m"),
	)

	myQuery, err = computeQuery(queryInfo)
	assert.Nil(t, err)
	assert.Equal(t, realQuery, myQuery)

	queryInfo = &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"should": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"User", "Thomas"}},
			},
			"must": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
				map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gt", "now-2h"}},
			},
		},
	}

	realQuery = elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Thomas"),
	).Must(
		elastic.NewTermQuery("Status", "OK"),
		elastic.NewRangeQuery("Timestamp").Gt("now-2h"),
	)

	myQuery, err = computeQuery(queryInfo)
	assert.Nil(t, err)
	assert.Equal(t, realQuery, myQuery)

	realQuery = elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Tobias"),
	).Must(
		elastic.NewTermQuery("Status", "OK"),
		elastic.NewRangeQuery("Timestamp").Gt("now-2h"),
	)
	assert.NotEqual(t, realQuery, myQuery)

	queryInfo = &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"should": []interface{}{
				map[interface{}]interface{}{"plop": []interface{}{"Thomas"}},
			},
			"must": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
				map[interface{}]interface{}{"hihi": []interface{}{"Timestamp", "gt", "now-2h"}},
			},
		},
	}
	myQuery, err = computeQuery(queryInfo)
	assert.NotNil(t, err)

	// test filterQueries
	queryInfo = &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"should": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"User", "Thomas"}},
			},
			"must": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
			},
			"filter": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Method", "XPUT"}},
				map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte", "now-1h"}},
			},
		},
	}

	realQuery = elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Thomas"),
	).Must(
		elastic.NewTermQuery("Status", "OK"),
	).Filter(
		elastic.NewTermQuery("Method", "XPUT"),
		elastic.NewRangeQuery("Timestamp").Gte("now-1h"),
	)

	myQuery, err = computeQuery(queryInfo)
	assert.Nil(t, err)
	assert.Equal(t, realQuery, myQuery)

	queryInfo = &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"must_not": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Type", "plop"}},
			},
			"filter": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Method"}}, //missing 2nd parameter
			},
		},
	}
	myQuery, err = computeQuery(queryInfo)
	assert.NotNil(t, err)
}
예제 #10
0
파일: elastic.go 프로젝트: endeveit/recause
// Searches for mssages
func (e *Elastic) GetMessages(q *storage.SearchQuery) (result *storage.SearchResult, err error) {
	var (
		filters []es.Query    = []es.Query{}
		query   *es.BoolQuery = es.NewBoolQuery()
		tsRange *es.RangeQuery
		msg     *storage.Message
	)

	if len(q.Query) > 0 {
		filters = append(filters, es.NewQueryStringQuery(q.Query))
	} else {
		filters = append(filters, es.NewMatchAllQuery())
	}

	if q.From.Second() > 0 && q.To.Second() > 0 {
		tsRange = es.NewRangeQuery("timestamp")
		tsRange = tsRange.From(q.From)
		tsRange = tsRange.To(q.To)
		filters = append(filters, tsRange)
	} else if q.From.Second() > 0 && q.To.Second() <= 0 {
		tsRange = es.NewRangeQuery("timestamp")
		tsRange = tsRange.From(q.From)
		filters = append(filters, tsRange)
	} else if q.To.Second() > 0 && q.From.Second() <= 0 {
		tsRange = es.NewRangeQuery("timestamp")
		tsRange = tsRange.To(q.To)
		filters = append(filters, tsRange)
	}

	rs, err := e.client.
		Search(e.indexName).
		Type(e.typeName).
		Query(query.Filter(filters...)).
		Sort("timestamp", false).
		From(q.Offset).
		Size(q.Limit).
		Do()

	if err != nil {
		return nil, err
	}

	result = &storage.SearchResult{
		Total:    rs.TotalHits(),
		TookMs:   rs.TookInMillis,
		Limit:    q.Limit,
		Offset:   q.Offset,
		Messages: []storage.Message{},
	}

	// rs.Each() is not used because we need to add message id manually
	if rs.Hits != nil && rs.Hits.Hits != nil && len(rs.Hits.Hits) > 0 {
		for _, hit := range rs.Hits.Hits {
			msg = new(storage.Message)
			err = json.Unmarshal(*hit.Source, msg)

			if err != nil {
				continue
			}

			msg.Id = hit.Id

			result.Messages = append(result.Messages, *msg)
		}
	}

	return result, nil
}