Example #1
0
func (e *Elastic) GetAnnotations(start, end *time.Time, source, host, creationUser, owner, category string) (annotate.Annotations, error) {
	annotations := annotate.Annotations{}
	s := elastic.NewSearchSource()
	if start != nil && end != nil {
		startQ := elastic.NewRangeQuery(annotate.EndDate).Gte(start)
		endQ := elastic.NewRangeQuery(annotate.StartDate).Lte(end)
		s = s.Query(elastic.NewBoolQuery().Must(startQ, endQ))
	}
	if source != "" {
		s = s.Query(elastic.NewTermQuery(annotate.Source, source))
	}
	if host != "" {
		s = s.Query(elastic.NewTermQuery(annotate.Host, host))
	}
	if creationUser != "" {
		s = s.Query(elastic.NewTermQuery(annotate.CreationUser, creationUser))
	}
	if owner != "" {
		s = s.Query(elastic.NewTermQuery(annotate.Owner, owner))
	}
	if category != "" {
		s = s.Query(elastic.NewTermQuery(annotate.Category, category))
	}
	res, err := e.Search(e.index).Query(s).Size(e.maxResults).Do()
	if err != nil {
		return annotations, err
	}
	var aType annotate.Annotation
	for _, item := range res.Each(reflect.TypeOf(aType)) {
		a := item.(annotate.Annotation)
		annotations = append(annotations, a)
	}
	return annotations, nil
}
Example #2
0
func (e *Elastic) GetAnnotations(start, end *time.Time, fieldFilters ...FieldFilter) (annotate.Annotations, error) {
	if !e.initialized {
		return nil, unInitErr
	}
	annotations := annotate.Annotations{}
	filters := []elastic.Query{}
	if start != nil && end != nil {
		startQ := elastic.NewRangeQuery(annotate.EndDate).Gte(start)
		endQ := elastic.NewRangeQuery(annotate.StartDate).Lte(end)
		filters = append(filters, elastic.NewBoolQuery().Must(startQ, endQ))
	}
	for _, filter := range fieldFilters {
		switch filter.Field {
		case annotate.Source, annotate.Host, annotate.CreationUser, annotate.Owner, annotate.Category:
		default:
			return annotations, fmt.Errorf("%v is not a field that can be filtered on", filter.Field)
		}
		var q elastic.Query
		switch filter.Verb {
		case Is, "":
			q = elastic.NewTermQuery(filter.Field, filter.Value)
		case Empty:
			// Can't detect empty on a analyzed field
			if filter.Field == annotate.Message {
				return annotations, fmt.Errorf("message field does not support empty searches")
			}
			q = elastic.NewTermQuery(filter.Field, "")
		default:
			return annotations, fmt.Errorf("%v is not a valid query verb", filter.Verb)
		}
		if filter.Not {
			q = elastic.NewBoolQuery().MustNot(q)
		}
		filters = append(filters, q)
	}

	var aType annotate.Annotation
	scroll := e.Scroll(e.index).Query(elastic.NewBoolQuery().Must(filters...)).Size(e.maxResults).Pretty(true)
	for {
		res, err := scroll.Do()
		if err == io.EOF {
			break
		}
		if err != nil {
			return annotations, err
		}
		for _, item := range res.Each(reflect.TypeOf(aType)) {
			a := item.(annotate.Annotation)
			annotations = append(annotations, a)
		}
	}
	return annotations, nil
}
Example #3
0
// FindGeoFencesExceptByUser returns all geofences around a lat/lon pair, excluding ones from the specified user.
func FindGeoFencesExceptByUser(centerLat float64, centerLon float64, radius int64, excludeBy int) ([]models.Fence, error) {
	query := elastic.NewBoolQuery()
	query = query.MustNot(elastic.NewTermQuery("owner", excludeBy))
	query.Filter(elastic.NewGeoDistanceQuery("center").Distance(fmt.Sprintf("%d m", radius)).Lat(centerLat).Lon(centerLon))

	searchResult, err := ElasticInstance.Search().
		Index(IndexGeoFences).
		Query(query).
		Do()

	// Check whether an error appeared or not.
	if err != nil {
		return nil, err
	}

	if searchResult.Hits != nil {
		fences := make([]int64, searchResult.TotalHits(), searchResult.TotalHits())
		fmt.Printf("Found a total of %d GeoFences\n", searchResult.Hits.TotalHits)

		// Iterate through results
		for i, hit := range searchResult.Hits.Hits {
			stringID, _ := strconv.ParseInt(hit.Id, 10, 64)
			fences[i] = stringID
		}
		return models.FindFencesByIDs(fences)
	}

	fmt.Print("Found no fences\n")
	var empty []models.Fence
	return empty, nil
}
Example #4
0
func ScopeES(ts opentsdb.TagSet, q elastic.Query) elastic.Query {
	var filters []elastic.Query
	for tagKey, tagValue := range ts {
		filters = append(filters, elastic.NewTermQuery(tagKey, tagValue))
	}
	filters = append(filters, q)
	b := elastic.NewBoolQuery().Must(filters...)
	return b
}
Example #5
0
func ExampleSearchService() {
	// Get a client to the local Elasticsearch instance.
	client, err := elastic.NewClient()
	if err != nil {
		// Handle error
		panic(err)
	}

	// Search with a term query
	termQuery := elastic.NewTermQuery("user", "olivere")
	searchResult, err := client.Search().
		Index("twitter").   // search in index "twitter"
		Query(termQuery).   // specify the query
		Sort("user", true). // sort by "user" field, ascending
		From(0).Size(10).   // take documents 0-9
		Pretty(true).       // pretty print request and response JSON
		Do()                // execute
	if err != nil {
		// Handle error
		panic(err)
	}

	// searchResult is of type SearchResult and returns hits, suggestions,
	// and all kinds of other information from Elasticsearch.
	fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)

	// Number of hits
	if searchResult.Hits.TotalHits > 0 {
		fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)

		// Iterate through results
		for _, hit := range searchResult.Hits.Hits {
			// hit.Index contains the name of the index

			// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
			var t Tweet
			err := json.Unmarshal(*hit.Source, &t)
			if err != nil {
				// Deserialization failed
			}

			// Work with tweet
			fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
		}
	} else {
		// No hits
		fmt.Print("Found no tweets\n")
	}
}
Example #6
0
func TestStringtoNb(t *testing.T) {
	t1 := "40.7"
	t2 := 3.3
	t3 := "100"
	t4 := "nothing to do here"
	t5 := "now-30m"
	t6 := "100mille"

	r1 := stringToNb(t1)
	r2 := stringToNb(t2)
	r3 := stringToNb(t3)
	r4 := stringToNb(t4)
	r5 := stringToNb(t5)
	r6 := stringToNb(t6)
	assert.Equal(t, 40.7, r1)
	assert.Equal(t, 3.3, r2)
	assert.Equal(t, 100, r3)
	assert.Equal(t, "nothing to do here", r4)
	assert.Equal(t, "now-30m", r5)
	assert.Equal(t, "100mille", r6)

	query1 := elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Thomas"),
	).Must(
		elastic.NewTermQuery("Status", stringToNb("500")),
		elastic.NewRangeQuery("Code").Gt(stringToNb("42.42")),
	)

	queryWrong := elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Thomas"),
	).Must(
		elastic.NewTermQuery("Status", "500"),
		elastic.NewRangeQuery("Code").Gt("42.42"),
	)

	query2 := elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Thomas"),
	).Must(
		elastic.NewTermQuery("Status", 500),
		elastic.NewRangeQuery("Code").Gt(42.42),
	)
	assert.NotEqual(t, queryWrong, query2)
	assert.Equal(t, query1, query2)
}
func (t *TestCase) search() {
	// Loop forever to check for connection issues
	for {
		// Get tweet with specified ID
		get1, err := t.client.Get().
			Index(t.index).
			Type("tweet").
			Id("1").
			Do()
		if err != nil {
			//failf("Get failed: %v", err)
			t.runCh <- RunInfo{Success: false}
			continue
		}
		if !get1.Found {
			//log.Printf("Document %s not found\n", "1")
			//fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type)
			t.runCh <- RunInfo{Success: false}
			continue
		}

		// Search with a term query
		searchResult, err := t.client.Search().
			Index(t.index).                                 // search in index t.index
			Query(elastic.NewTermQuery("user", "olivere")). // specify the query
			Sort("user", true).                             // sort by "user" field, ascending
			From(0).Size(10).                               // take documents 0-9
			Pretty(true).                                   // pretty print request and response JSON
			Do()                                            // execute
		if err != nil {
			//failf("Search failed: %v\n", err)
			t.runCh <- RunInfo{Success: false}
			continue
		}

		// searchResult is of type SearchResult and returns hits, suggestions,
		// and all kinds of other information from Elasticsearch.
		//fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)

		// Number of hits
		if searchResult.Hits.TotalHits > 0 {
			//fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)

			// Iterate through results
			for _, hit := range searchResult.Hits.Hits {
				// hit.Index contains the name of the index

				// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
				var tweet Tweet
				err := json.Unmarshal(*hit.Source, &tweet)
				if err != nil {
					// Deserialization failed
					//failf("Deserialize failed: %v\n", err)
					t.runCh <- RunInfo{Success: false}
					continue
				}

				// Work with tweet
				//fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
			}
		} else {
			// No hits
			//fmt.Print("Found no tweets\n")
		}

		t.runCh <- RunInfo{Success: true}

		// Sleep some time
		time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
	}
}
Example #8
0
func Test_getQueries(t *testing.T) {
	eslog.InitSilent()

	// term filters
	var filters []interface{} = []interface{}{
		map[interface{}]interface{}{"term": []interface{}{"test", "yes"}},
		map[interface{}]interface{}{"term": []interface{}{"required", true}},
	}

	realfilters := []elastic.Query{
		elastic.NewTermQuery("test", "yes"),
		elastic.NewTermQuery("required", true),
	}

	testfilters, err := getQueries(filters)
	assert.Equal(t, err, nil, "Should be nil")
	assert.Equal(t, realfilters, testfilters)

	realfilters = []elastic.Query{
		elastic.NewTermQuery("test", "yes"),
		elastic.NewTermQuery("required", false),
	}
	assert.NotEqual(t, realfilters, testfilters)

	//range filters
	filters = []interface{}{
		map[interface{}]interface{}{"term": []interface{}{"value", 146}},
		map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}},
		map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte", "now-1h"}},
	}

	realfilters = []elastic.Query{
		elastic.NewTermQuery("value", 146),
		elastic.NewTermQuery("othervalue", "testTest"),
		elastic.NewRangeQuery("Timestamp").Gte("now-1h"),
	}

	testfilters, err = getQueries(filters)
	assert.Equal(t, err, nil, "Should be nil")
	assert.Equal(t, realfilters, testfilters)

	realfilters = []elastic.Query{
		elastic.NewTermQuery("value", 146),
		elastic.NewTermQuery("othervalue", "testTest"),
		elastic.NewRangeQuery("Timestamp").Lt("now-1h"),
	}

	assert.NotEqual(t, realfilters, testfilters)

	filters = []interface{}{
		map[interface{}]interface{}{"term": []interface{}{"value", 146}},
		map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}},
		map[interface{}]interface{}{"range": []interface{}{"Timestamp", "lt", "now-1h"}},
	}
	testfilters, err = getQueries(filters)
	assert.Equal(t, err, nil, "Should be nil")
	assert.Equal(t, realfilters, testfilters)

	//non valid fields
	filters = []interface{}{
		map[interface{}]interface{}{"term": []interface{}{145, "yes"}},
		map[interface{}]interface{}{"term": []interface{}{"required", "yes"}},
	}
	testfilters, err = getQueries(filters)
	assert.NotNil(t, err)

	filters = []interface{}{
		map[interface{}]interface{}{"term": []interface{}{"yes"}},
		map[interface{}]interface{}{"term": []interface{}{"required", 28}},
	}
	testfilters, err = getQueries(filters)
	assert.NotNil(t, err)

	filters = []interface{}{
		map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte"}},
		map[interface{}]interface{}{"term": []interface{}{"required", 28}},
	}

	testfilters, err = getQueries(filters)
	assert.NotNil(t, err)

	filters = []interface{}{
		map[interface{}]interface{}{"range": []interface{}{42, "gte", "now-30m"}},
		map[interface{}]interface{}{"term": []interface{}{"required", 28}},
	}

	testfilters, err = getQueries(filters)
	assert.NotNil(t, err)
}
Example #9
0
func Test_boolquery(t *testing.T) {
	eslog.InitSilent()
	info := &config.QueryInfo{}
	myQuery, err := computeQuery(info)
	assert.NotNil(t, err)

	queryInfo := &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"must": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Value", 146.5}},
				map[interface{}]interface{}{"term": []interface{}{"othervalue", "testTest"}},
				map[interface{}]interface{}{"range": []interface{}{"Timestamp", "lt", "now-1h"}},
			},
			"must_not": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"status", "OK"}},
			},
		},
	}
	realQuery := elastic.NewBoolQuery().Must(
		elastic.NewTermQuery("Value", 146.5),
		elastic.NewTermQuery("othervalue", "testTest"),
		elastic.NewRangeQuery("Timestamp").Lt("now-1h"),
	).MustNot(
		elastic.NewTermQuery("status", "OK"),
	)
	myQuery, err = computeQuery(queryInfo)
	assert.Nil(t, err)
	assert.Equal(t, realQuery, myQuery)

	queryInfo = &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"must_not": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Status", "Error"}},
				map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gt", "now-30m"}},
			},
		},
	}

	realQuery = elastic.NewBoolQuery().MustNot(
		elastic.NewTermQuery("Status", "Error"),
	).MustNot(
		elastic.NewRangeQuery("Timestamp").Gt("now-30m"),
	)

	myQuery, err = computeQuery(queryInfo)
	assert.Nil(t, err)
	assert.Equal(t, realQuery, myQuery)

	queryInfo = &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"should": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"User", "Thomas"}},
			},
			"must": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
				map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gt", "now-2h"}},
			},
		},
	}

	realQuery = elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Thomas"),
	).Must(
		elastic.NewTermQuery("Status", "OK"),
		elastic.NewRangeQuery("Timestamp").Gt("now-2h"),
	)

	myQuery, err = computeQuery(queryInfo)
	assert.Nil(t, err)
	assert.Equal(t, realQuery, myQuery)

	realQuery = elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Tobias"),
	).Must(
		elastic.NewTermQuery("Status", "OK"),
		elastic.NewRangeQuery("Timestamp").Gt("now-2h"),
	)
	assert.NotEqual(t, realQuery, myQuery)

	queryInfo = &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"should": []interface{}{
				map[interface{}]interface{}{"plop": []interface{}{"Thomas"}},
			},
			"must": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
				map[interface{}]interface{}{"hihi": []interface{}{"Timestamp", "gt", "now-2h"}},
			},
		},
	}
	myQuery, err = computeQuery(queryInfo)
	assert.NotNil(t, err)

	// test filterQueries
	queryInfo = &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"should": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"User", "Thomas"}},
			},
			"must": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Status", "OK"}},
			},
			"filter": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Method", "XPUT"}},
				map[interface{}]interface{}{"range": []interface{}{"Timestamp", "gte", "now-1h"}},
			},
		},
	}

	realQuery = elastic.NewBoolQuery().Should(
		elastic.NewTermQuery("User", "Thomas"),
	).Must(
		elastic.NewTermQuery("Status", "OK"),
	).Filter(
		elastic.NewTermQuery("Method", "XPUT"),
		elastic.NewRangeQuery("Timestamp").Gte("now-1h"),
	)

	myQuery, err = computeQuery(queryInfo)
	assert.Nil(t, err)
	assert.Equal(t, realQuery, myQuery)

	queryInfo = &config.QueryInfo{
		Type: "boolquery",
		Clauses: map[string]interface{}{
			"must_not": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Type", "plop"}},
			},
			"filter": []interface{}{
				map[interface{}]interface{}{"term": []interface{}{"Method"}}, //missing 2nd parameter
			},
		},
	}
	myQuery, err = computeQuery(queryInfo)
	assert.NotNil(t, err)
}
Example #10
0
func Example() {
	errorlog := log.New(os.Stdout, "APP ", log.LstdFlags)

	// Obtain a client. You can also provide your own HTTP client here.
	client, err := elastic.NewClient(elastic.SetErrorLog(errorlog))
	if err != nil {
		// Handle error
		panic(err)
	}

	// Trace request and response details like this
	//client.SetTracer(log.New(os.Stdout, "", 0))

	// Ping the Elasticsearch server to get e.g. the version number
	info, code, err := client.Ping("http://127.0.0.1:9200").Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	fmt.Printf("Elasticsearch returned with code %d and version %s", code, info.Version.Number)

	// Getting the ES version number is quite common, so there's a shortcut
	esversion, err := client.ElasticsearchVersion("http://127.0.0.1:9200")
	if err != nil {
		// Handle error
		panic(err)
	}
	fmt.Printf("Elasticsearch version %s", esversion)

	// Use the IndexExists service to check if a specified index exists.
	exists, err := client.IndexExists("twitter").Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	if !exists {
		// Create a new index.
		createIndex, err := client.CreateIndex("twitter").Do()
		if err != nil {
			// Handle error
			panic(err)
		}
		if !createIndex.Acknowledged {
			// Not acknowledged
		}
	}

	// Index a tweet (using JSON serialization)
	tweet1 := Tweet{User: "olivere", Message: "Take Five", Retweets: 0}
	put1, err := client.Index().
		Index("twitter").
		Type("tweet").
		Id("1").
		BodyJson(tweet1).
		Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	fmt.Printf("Indexed tweet %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type)

	// Index a second tweet (by string)
	tweet2 := `{"user" : "olivere", "message" : "It's a Raggy Waltz"}`
	put2, err := client.Index().
		Index("twitter").
		Type("tweet").
		Id("2").
		BodyString(tweet2).
		Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	fmt.Printf("Indexed tweet %s to index %s, type %s\n", put2.Id, put2.Index, put2.Type)

	// Get tweet with specified ID
	get1, err := client.Get().
		Index("twitter").
		Type("tweet").
		Id("1").
		Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	if get1.Found {
		fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type)
	}

	// Flush to make sure the documents got written.
	_, err = client.Flush().Index("twitter").Do()
	if err != nil {
		panic(err)
	}

	// Search with a term query
	termQuery := elastic.NewTermQuery("user", "olivere")
	searchResult, err := client.Search().
		Index("twitter").   // search in index "twitter"
		Query(termQuery).   // specify the query
		Sort("user", true). // sort by "user" field, ascending
		From(0).Size(10).   // take documents 0-9
		Pretty(true).       // pretty print request and response JSON
		Do()                // execute
	if err != nil {
		// Handle error
		panic(err)
	}

	// searchResult is of type SearchResult and returns hits, suggestions,
	// and all kinds of other information from Elasticsearch.
	fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)

	// Each is a convenience function that iterates over hits in a search result.
	// It makes sure you don't need to check for nil values in the response.
	// However, it ignores errors in serialization. If you want full control
	// over iterating the hits, see below.
	var ttyp Tweet
	for _, item := range searchResult.Each(reflect.TypeOf(ttyp)) {
		t := item.(Tweet)
		fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
	}
	// TotalHits is another convenience function that works even when something goes wrong.
	fmt.Printf("Found a total of %d tweets\n", searchResult.TotalHits())

	// Here's how you iterate through results with full control over each step.
	if searchResult.Hits.TotalHits > 0 {
		fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)

		// Iterate through results
		for _, hit := range searchResult.Hits.Hits {
			// hit.Index contains the name of the index

			// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
			var t Tweet
			err := json.Unmarshal(*hit.Source, &t)
			if err != nil {
				// Deserialization failed
			}

			// Work with tweet
			fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
		}
	} else {
		// No hits
		fmt.Print("Found no tweets\n")
	}

	// Update a tweet by the update API of Elasticsearch.
	// We just increment the number of retweets.
	script := elastic.NewScript("ctx._source.retweets += num").Param("num", 1)
	update, err := client.Update().Index("twitter").Type("tweet").Id("1").
		Script(script).
		Upsert(map[string]interface{}{"retweets": 0}).
		Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	fmt.Printf("New version of tweet %q is now %d", update.Id, update.Version)

	// ...

	// Delete an index.
	deleteIndex, err := client.DeleteIndex("twitter").Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	if !deleteIndex.Acknowledged {
		// Not acknowledged
	}
}