func findOffersFromText(index bleve.Index, query string, ids []string) (
[]datedOffer, error) {
if query == "" {
return nil, nil
}
datedOffers := []datedOffer{}
q, err := makeSearchQuery(query, ids)
if err != nil {
return nil, err
}
rq := bleve.NewSearchRequest(q)
rq.Size = 20000
rq.Fields = []string{"date"}
res, err := index.Search(rq)
if err != nil {
return nil, err
}
for _, doc := range res.Hits {
date, ok := doc.Fields["date"].(string)
if !ok {
return nil, fmt.Errorf("could not retrieve date for %s", doc.ID)
}
datedOffers = append(datedOffers, datedOffer{
Date: date,
Id: doc.ID,
})
}
return datedOffers, nil
}
func (self *Wiki) Search(queryString string) ([]*SearchResult, error) {
query := bleve.NewMatchQuery(queryString)
search := bleve.NewSearchRequest(query)
search.Highlight = bleve.NewHighlight()
searchResult, err := self.Index.Search(search)
if err != nil {
return nil, err
}
results := []*SearchResult{}
for _, result := range searchResult.Hits {
section, article := self.Find(result.ID)
if article == nil && section == nil {
return nil, fmt.Errorf("%v section or article not found", result.ID)
}
text := ""
for _, values := range result.Fragments {
for _, value := range values {
text += value
}
}
name := ""
if section != nil {
name = section.Name
}
if article != nil {
name = article.Name
}
results = append(results, &SearchResult{Path: result.ID, Name: name, Text: text})
}
return results, nil
}
func search(cfg *Config) error {
store, err := OpenStore(cfg.Store())
if err != nil {
return err
}
index, err := bleve.Open(cfg.Index())
if err != nil {
return err
}
defer index.Close()
q, err := makeSearchQuery(*searchQuery, nil)
if err != nil {
return err
}
rq := bleve.NewSearchRequest(q)
rq.Size = 100
ids := []string{}
for {
res, err := index.Search(rq)
if err != nil {
return err
}
for _, doc := range res.Hits {
ids = append(ids, doc.ID)
}
if len(res.Hits) < rq.Size {
break
}
rq.From += rq.Size
}
return printOffers(store, ids)
}
// Search method lookup for records using a query
func (i *bleveIndexer) Search(q string) (records []indexer.Record) {
query := bleve.NewQueryStringQuery(q)
request := bleve.NewSearchRequest(query)
request.Highlight = bleve.NewHighlight()
result, err := i.bleve.Search(request)
if err != nil { // an empty query would cause this
return
}
for _, match := range result.Hits {
rec := i.Record(match.ID)
loaded := rec.Load()
if !loaded {
continue
}
if len(match.Fragments["Body"]) > 0 {
rec.SetBody([]byte(match.Fragments["Body"][0]))
}
records = append(records, rec)
}
return
}
func main() {
// open a new index
mapping := bleve.NewIndexMapping()
index, err := bleve.New("example.bleve", mapping)
if err != nil {
fmt.Println(err)
return
}
data := struct {
Name string
}{
Name: "text",
}
// index some data
index.Index("id", data)
// search for some text
query := bleve.NewMatchQuery("text")
search := bleve.NewSearchRequest(query)
searchResults, err := index.Search(search)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(searchResults)
}
func (docstore *DocStoreExt) Search(collection, queryString string) ([]byte, error) {
js := []byte{}
query := bleve.NewQueryStringQuery(queryString)
searchRequest := bleve.NewSearchRequest(query)
searchResult, err := docstore.index.Search(searchRequest)
if err != nil {
return nil, err
}
for index, sr := range searchResult.Hits {
jsPart := []byte{}
_, err := docstore.fetchDoc(collection, sr.ID, &jsPart)
if err != nil {
return nil, err
}
js = append(js, addID(jsPart, sr.ID)...)
if index != len(searchResult.Hits)-1 {
js = append(js, []byte(",")...)
}
}
if len(searchResult.Hits) > 0 {
js = js[0 : len(js)-1]
}
js = append(js, []byte("]")...)
// "_meta": searchResult,
// "data": docs,
// TODO(tsileo) returns meta along with argument
return js, nil
}
func main() {
// Open an index if it already exists.
index, err := bleve.Open("example.bleve")
if err != nil {
mapping := bleve.NewIndexMapping()
index, err = bleve.New("example.bleve", mapping)
if err != nil {
fmt.Println(err)
return
}
}
data := struct {
Name string
}{
Name: "text",
}
// Index some data.
index.Index("id", data)
// Search for some text
query := bleve.NewMatchQuery("text")
search := bleve.NewSearchRequest(query)
searchResults, err := index.Search(search)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(searchResults)
}
// Search is an http.HandlerFunc that accepts a Parse Cloud Code Webhook request.
//
// The expected query parameter is 'q'
func (i *Indexer) Search(w http.ResponseWriter, r *http.Request) {
req, err := webhookRequest(r, i.webhookKey)
if err != nil {
writeErr(w, err)
return
}
rawq := req.Params["q"]
if rawq == nil {
writeErr(w, fmt.Errorf("no term provided"))
return
}
q, ok := rawq.(string)
if q == "" || !ok {
writeErr(w, fmt.Errorf("no term provided"))
return
}
query := bleve.NewQueryStringQuery(q)
search := bleve.NewSearchRequest(query)
searchResults, err := i.index.Search(search)
if err != nil {
writeErr(w, err)
return
}
spew.Dump(searchResults)
ids := []string{}
for _, h := range searchResults.Hits {
ids = append(ids, h.ID)
}
err = json.NewEncoder(w).Encode(Response{
Success: ids,
})
if err != nil {
log.Println("error encoding response:", err)
}
}
func TestFindByAnything(t *testing.T) {
db, eventList := dbCreate()
idx := idxCreate()
indexEvents(idx, eventList)
// We are looking to an Event with some string which match with dotGo
query := bleve.NewMatchQuery("dotGo")
searchRequest := bleve.NewSearchRequest(query)
searchResult, err := idx.Search(searchRequest)
if err != nil {
t.Error("Something wrong happen with the search", err, ballotX)
} else {
t.Log("Should search the query", checkMark)
}
if searchResult.Total != 1 {
t.Error("Only 1 result are expected, got ", searchResult.Total, ballotX)
} else {
t.Log("Should return only one result", checkMark)
}
event := &Event{}
db.First(&event, &searchResult.Hits[0].ID)
if event.Name != "dotGo 2015" {
t.Error("Expected \"dotGo 2015\", Receive: ", event.Name)
} else {
t.Log("Should return an event with the name equal a", event.Name, checkMark)
}
idxDestroy()
dbDestroy()
}
func jobTags(w http.ResponseWriter, r *http.Request) {
query := bleve.NewMatchAllQuery()
searchRequest := bleve.NewSearchRequest(query)
searchRequest.Fields = []string{"tags"}
tagsFacet := bleve.NewFacetRequest("tags", 99999)
searchRequest.AddFacet("tags", tagsFacet)
jobsSearchResults, err := doRequest("jobs", searchRequest)
if err != nil {
respond.With(w, r, http.StatusInternalServerError, err)
return
}
tags := []string{}
for _, t := range jobsSearchResults.Facets["tags"].Terms {
tags = append(tags, fmt.Sprintf("%s", t.Term))
}
respond.With(w, r, http.StatusOK, struct {
Total int `json:"total"`
Tags []string `json:"tags"`
}{
Total: len(tags),
Tags: tags,
})
}
// listPoints returns the location of offers satisfying specified full-text
// query. If query is empty, it returns all locations. If not nil, spatial is
// exploited as a cache to fetch indexed offers and their locations, which
// avoid store lookups.
func listPoints(store *Store, index bleve.Index, spatial *SpatialIndex,
query string) ([]Point, error) {
var ids []string
if query == "" {
if spatial != nil {
ids = spatial.List()
} else {
list, err := store.List()
if err != nil {
return nil, err
}
ids = list
}
} else {
q, err := makeSearchQuery(query, nil)
if err != nil {
return nil, err
}
rq := bleve.NewSearchRequest(q)
rq.Size = 20000
res, err := index.Search(rq)
if err != nil {
return nil, err
}
for _, doc := range res.Hits {
ids = append(ids, doc.ID)
}
}
points := make([]Point, 0, len(ids))
for _, id := range ids {
var p *Point
if spatial != nil {
offer := spatial.Get(id)
if offer != nil {
p = &offer.Point
}
}
if p == nil {
loc, _, err := store.GetLocation(id)
if err != nil {
return nil, err
}
if loc == nil {
continue
}
p = &Point{
Lat: loc.Lat,
Lon: loc.Lon,
}
}
points = append(points, *p)
}
return points, nil
}
func (be *BleveEngine) Search(query string) (interface{}, error) {
index, _ := bleve.Open(INDEX)
bleveQuery := bleve.NewQueryStringQuery(query)
searchRequest := bleve.NewSearchRequest(bleveQuery)
searchResults, err := index.Search(searchRequest)
if err != nil {
return nil, err
}
return searchResults, nil
}
//See https://godoc.org/github.com/blevesearch/bleve#SearchResult
func Search(keyword string) (*bleve.SearchResult, error) {
fmt.Println(keyword)
query := bleve.NewMatchQuery(keyword)
request := bleve.NewSearchRequest(query)
request.Size = 20
//request.Fields = []string{"Name", "Text", "Size", "IsDir", "ModTime"}
result, err := LimeIndex.Search(request)
if err != nil {
return nil, err
}
return result, nil
}
// START OMIT
func main() {
index, err := bleve.Open("people.bleve") // HLOPEN
if err != nil {
log.Fatal(err)
}
query := bleve.NewTermQuery("marty") // HLQUERY
request := bleve.NewSearchRequest(query) // HLREQ
result, err := index.Search(request) // HLSEARCH
if err != nil {
log.Fatal(err)
}
fmt.Println(result)
}
// this test reproduces bug #87
// https://github.com/blevesearch/bleve/issues/87
// because of which, it will deadlock
func TestBeerSearchBug87(t *testing.T) {
defer os.RemoveAll("beer-search-test.bleve")
mapping, err := buildIndexMapping()
if err != nil {
t.Fatal(err)
}
index, err := bleve.New("beer-search-test.bleve", mapping)
if err != nil {
t.Fatal(err)
}
defer index.Close()
// start indexing documents in the background
go func() {
// open the directory
dirEntries, err := ioutil.ReadDir("data/")
if err != nil {
t.Fatal(err)
}
for _, dirEntry := range dirEntries {
filename := dirEntry.Name()
jsonBytes, err := ioutil.ReadFile("data/" + filename)
if err != nil {
t.Fatal(err)
}
ext := filepath.Ext(filename)
docId := filename[:(len(filename) - len(ext))]
index.Index(docId, jsonBytes)
}
}()
// give indexing a head start
time.Sleep(1 * time.Second)
// start querying
for i := 0; i < 1000; i++ {
time.Sleep(1 * time.Millisecond)
termQuery := bleve.NewTermQuery("shock").SetField("name")
termSearchRequest := bleve.NewSearchRequest(termQuery)
// termSearchRequest.AddFacet("styles", bleve.NewFacetRequest("style", 3))
termSearchRequest.Fields = []string{"abv"}
_, err := index.Search(termSearchRequest)
if err != nil {
t.Error(err)
}
}
}
// Search performs a search of the index using the given query. Returns IDs of documents
// which satisfy all queries. Returns Doc IDs in sorted order, ascending.
func (i *Index) Search(q string) (DocIDs, error) {
query := bleve.NewQueryStringQuery(q)
searchRequest := bleve.NewSearchRequest(query)
searchRequest.Size = maxSearchHitSize
searchResults, err := i.Alias.Search(searchRequest)
if err != nil {
return nil, err
}
docIDs := make(DocIDs, 0, len(searchResults.Hits))
for _, d := range searchResults.Hits {
docIDs = append(docIDs, DocID(d.ID))
}
sort.Sort(docIDs)
return docIDs, nil
}
// Find all comments matching text
// @return result.Result<[]*Comment, error>
func CommentsWithContent(repoPath, content string) result.Result {
return openIndex(repoPath, func(repo *git.Repository, index bleve.Index) result.Result {
query := bleve.NewQueryStringQuery(content)
request := bleve.NewSearchRequest(query)
return result.NewResult(index.Search(request)).FlatMap(func(match interface{}) result.Result {
hits := match.(*bleve.SearchResult).Hits
comments := make([]*gc.Comment, len(hits))
for idx, hit := range hits {
gc.CommentByID(repo, hit.ID).FlatMap(func(comment interface{}) result.Result {
comments[idx] = comment.(*gc.Comment)
return result.Result{}
})
}
return result.NewSuccess(comments)
})
})
}
func (cache *BleveCache) Search(ctx context.Context, term string, options SearchOptions) ([]*RecipeLink, error) {
q := bleve.NewQueryStringQuery(term)
req := bleve.NewSearchRequest(q)
res, err := cache.index.Search(req)
if err != nil {
return nil, err
}
links := make([]*RecipeLink, len(res.Hits))
for i, hit := range res.Hits {
filePath := cache.dbPath + "/" + hit.ID + ".json"
links[i], err = loadRecipeLink(filePath)
if err != nil {
return links, err
}
}
return links, err
}
// START OMIT
func main() {
index, err := bleve.Open("gophercon.bleve")
if err != nil {
log.Fatal(err)
}
q := bleve.NewMatchAllQuery()
req := bleve.NewSearchRequest(q)
req.Size = 0
req.AddFacet("sections", // HL
bleve.NewFacetRequest("section", 50)) // HL
res, err := index.Search(req)
if err != nil {
log.Fatal(err)
}
fmt.Println(res)
}
// Search the swish index for a given query.
func searchBleve(queryString string) (Results, error) {
index, err := openIndex()
if err != nil {
LogError(err)
return Results{}, err
}
defer index.Close()
newQueryString := ""
for _, tmp := range strings.Split(queryString, " ") {
word := strings.TrimSpace(tmp)
if word[0] == '-' || word[0] == '+' {
newQueryString += " " + word
} else if word[0] == '~' {
// Remove prefix to make term optional
newQueryString += " " + word[1:]
} else {
newQueryString += " +" + word
}
}
query := bleve.NewQueryStringQuery(newQueryString[1:]) // Remove leading space
search := bleve.NewSearchRequest(query)
search.Size = Config.MaxResults
searchResults, err := index.Search(search)
if err != nil {
println("Invalid query string: '" + newQueryString[1:] + "'")
LogError(err)
return Results{}, err
}
var ids []Scroll
for _, match := range searchResults.Hits {
id := Id(match.ID)
content, err := readScroll(id)
TryLogError(err)
scroll := Parse(string(id), content)
ids = append(ids, scroll)
}
return Results{ids[:len(searchResults.Hits)], int(searchResults.Total)}, nil
}
func (brs *BleveRecipeStore) All() ([]model.Recipe, error) {
recipes := []model.Recipe{}
q := bleve.NewMatchAllQuery()
src := bleve.NewSearchRequest(q)
res, err := brs.idx.Search(src)
for _, hit := range res.Hits {
data, err := brs.idx.GetInternal([]byte(hit.ID))
if err != nil {
panic(err)
}
recipe := model.Recipe{}
json.Unmarshal(data, &recipe)
recipes = append(recipes, recipe)
}
return recipes, err
}
// search executes a search for rooms or buildings.
func (s *Server) search(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
q := query.Get("q")
typeFilter := query.Get("type")
results := []*models.Index{}
if idx, ok := s.idIndex[q]; ok {
results = append(results, idx)
} else {
query := bleve.NewBooleanQuery()
if len(q) > 0 {
/*fuzzy_query := bleve.NewFuzzyQuery(q)
fuzzy_query.FuzzinessVal = 3
queryShould = append(queryShould, fuzzy_query)
queryShould = append(queryShould, bleve.NewRegexpQuery("[a-zA-Z0-9_]*"+q+"[a-zA-Z0-9_]*"))
queryShould = append(queryShould, bleve.NewQueryStringQuery(q))*/
query.AddShould(bleve.NewQueryStringQuery(q))
}
if typeFilter != "all" {
termQuery := bleve.NewTermQuery(typeFilter)
query.AddMust(termQuery)
}
searchRequest := bleve.NewSearchRequest(query)
searchRequest.Size = 25
searchResult, err := s.index.Search(searchRequest)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
for _, result := range []*search.DocumentMatch(searchResult.Hits) {
results = append(results, s.idIndex[result.ID])
}
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results)
}
// this test reproduces bug #87
// https://github.com/blevesearch/bleve/issues/87
// because of which, it will deadlock
func TestBeerSearchBug87(t *testing.T) {
defer os.RemoveAll("beer-search-test.bleve")
mapping, err := buildIndexMapping()
if err != nil {
t.Fatal(err)
}
index, err := bleve.New("beer-search-test.bleve", mapping)
if err != nil {
t.Fatal(err)
}
defer index.Close()
var wg sync.WaitGroup
wg.Add(1)
// start indexing documents in the background
go func() {
for jf := range walkDirectory("data/", t) {
docId := jf.filename[0:strings.LastIndex(jf.filename, ".")]
err = index.Index(docId, jf.contents)
if err != nil {
t.Error(err)
}
}
wg.Done()
}()
for i := 0; i < 50; i++ {
time.Sleep(1 * time.Second)
termQuery := bleve.NewTermQuery("shock").SetField("name")
termSearchRequest := bleve.NewSearchRequest(termQuery)
// termSearchRequest.AddFacet("styles", bleve.NewFacetRequest("style", 3))
termSearchRequest.Fields = []string{"abv"}
_, err := index.Search(termSearchRequest)
if err != nil {
t.Error(err)
}
}
wg.Wait()
}
// START OMIT
func main() {
index, err := bleve.Open("gophercon.bleve")
if err != nil {
log.Fatal(err)
}
qString := `+content:debugger ` // HLQUERY
qString += `title:"delve" ` // HLQUERY
qString += `title:go~2 ` // HLQUERY
qString += `-content:rust ` // HLQUERY
qString += `word_count:>30` // HLQUERY
q := bleve.NewQueryStringQuery(qString)
req := bleve.NewSearchRequest(q)
req.Highlight = bleve.NewHighlightWithStyle("html")
req.Fields = []string{"title", "author", "content", "word_count"}
res, err := index.Search(req)
if err != nil {
log.Fatal(err)
}
fmt.Println(res)
}
func freelancers(w http.ResponseWriter, r *http.Request) {
searchRequest := bleve.NewSearchRequest(bleve.NewMatchAllQuery())
searchRequest.Fields = []string{"*"}
freelnacersSearchResults, err := doRequest("freelancers", searchRequest)
if err != nil {
respond.With(w, r, http.StatusInternalServerError, err)
return
}
freelancers := []interface{}{}
for _, hit := range freelnacersSearchResults.Hits {
freelancers = append(freelancers, hit.Fields)
}
respond.With(w, r, http.StatusOK, struct {
Total int `json:"total"`
Items []interface{} `json:"items"`
}{
Total: len(freelancers),
Items: freelancers,
})
}
func query(term, highlight string, index bleve.Index, u content.User, feedIds []data.FeedId, paging ...int) (ua []content.UserArticle, err error) {
var query bleve.Query
query = bleve.NewQueryStringQuery(term)
if len(feedIds) > 0 {
queries := make([]bleve.Query, len(feedIds))
conjunct := make([]bleve.Query, 2)
for i, id := range feedIds {
q := bleve.NewTermQuery(strconv.FormatInt(int64(id), 10))
q.SetField("FeedId")
queries[i] = q
}
disjunct := bleve.NewDisjunctionQuery(queries)
conjunct[0] = query
conjunct[1] = disjunct
query = bleve.NewConjunctionQuery(conjunct)
}
searchRequest := bleve.NewSearchRequest(query)
if highlight != "" {
searchRequest.Highlight = bleve.NewHighlightWithStyle(highlight)
}
limit, offset := pagingLimit(paging)
searchRequest.Size = limit
searchRequest.From = offset
searchResult, err := index.Search(searchRequest)
if err != nil {
return
}
if len(searchResult.Hits) == 0 {
return
}
articleIds := []data.ArticleId{}
hitMap := map[data.ArticleId]*search.DocumentMatch{}
for _, hit := range searchResult.Hits {
if articleId, err := strconv.ParseInt(hit.ID, 10, 64); err == nil {
id := data.ArticleId(articleId)
articleIds = append(articleIds, id)
hitMap[id] = hit
}
}
ua = u.ArticlesById(articleIds)
if u.HasErr() {
return ua, u.Err()
}
for i := range ua {
data := ua[i].Data()
hit := hitMap[data.Id]
if len(hit.Fragments) > 0 {
data.Hit.Fragments = hit.Fragments
ua[i].Data(data)
}
}
return
}
func Search(q string) (*bleve.SearchResult, error) {
query := bleve.NewQueryStringQuery(q)
req := bleve.NewSearchRequest(query)
req.Highlight = bleve.NewHighlightWithStyle("html_ex")
return index.Search(req)
}
func Example() {
INDEX_DIR := "gojieba.bleve"
messages := []struct {
Id string
Body string
}{
{
Id: "1",
Body: "你好",
},
{
Id: "2",
Body: "交代",
},
{
Id: "3",
Body: "长江大桥",
},
}
indexMapping := bleve.NewIndexMapping()
os.RemoveAll(INDEX_DIR)
// clean index when example finished
defer os.RemoveAll(INDEX_DIR)
err := indexMapping.AddCustomTokenizer("gojieba",
map[string]interface{}{
"dictpath": gojieba.DICT_PATH,
"hmmpath": gojieba.HMM_PATH,
"userdictpath": gojieba.USER_DICT_PATH,
"idf": gojieba.IDF_PATH,
"stop_words": gojieba.STOP_WORDS_PATH,
"type": "gojieba",
},
)
if err != nil {
panic(err)
}
err = indexMapping.AddCustomAnalyzer("gojieba",
map[string]interface{}{
"type": "gojieba",
"tokenizer": "gojieba",
},
)
if err != nil {
panic(err)
}
indexMapping.DefaultAnalyzer = "gojieba"
index, err := bleve.New(INDEX_DIR, indexMapping)
if err != nil {
panic(err)
}
for _, msg := range messages {
if err := index.Index(msg.Id, msg); err != nil {
panic(err)
}
}
querys := []string{
"你好世界",
"亲口交代",
"长江",
}
for _, q := range querys {
req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(q))
req.Highlight = bleve.NewHighlight()
res, err := index.Search(req)
if err != nil {
panic(err)
}
fmt.Println(prettify(res))
}
// Output:
// [{"id":"1","score":0.27650412875470115}]
// [{"id":"2","score":0.27650412875470115}]
// [{"id":"3","score":0.7027325540540822}]
}
func runConfig(conf string, tar string, cpu string) []*Line {
if cpu != "" {
f, err := os.Create(cpu)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
start := time.Now()
wikiReader, err := blevebench.NewWikiReader(*source)
if err != nil {
log.Fatal(err)
}
defer wikiReader.Close()
mapping := blevebench.BuildArticleMapping()
benchConfig := blevebench.LoadConfigFile(conf)
fmt.Printf("Using Index Type: %s\n", benchConfig.IndexType)
fmt.Printf("Using KV store: %s\n", benchConfig.KVStore)
fmt.Printf("Using KV config: %#v\n", benchConfig.KVConfig)
index, err := bleve.NewUsing(tar, mapping, benchConfig.IndexType, benchConfig.KVStore, benchConfig.KVConfig)
if err != nil {
log.Fatal(err)
}
_, store, err := index.Advanced()
if err != nil {
log.Fatal(err)
}
itr := *count / (*level)
lines := NewLines(itr, len(typename), conf, typename)
tot := 0
// print header
fmt.Printf("elapsed,docs,avg_single_doc_ms,avg_batched_doc_ms,query_water_matches,first_query_water_ms,avg_repeated%d_query_water_ms", *qrepeat)
printOtherHeader(store)
fmt.Printf("\n")
singleCount := 0
var singleTime time.Duration
batchCount := 0
var batchTime time.Duration
batch := index.NewBatch()
for i := 1; i < (*count)+1; i++ {
leveli := i % *level
a, err := wikiReader.Next()
if err != nil {
log.Fatal(err)
}
if leveli < *batchSize {
// index single
singleStart := time.Now()
err = index.Index(a.Title, a)
if err != nil {
log.Fatalf("error indexing: %v", err)
}
duration := time.Since(singleStart)
singleCount++
singleTime += duration
} else {
// add to batch
batch.Index(a.Title, a)
// if batch is full index it
if batch.Size() == *batchSize {
batchStart := time.Now()
err := index.Batch(batch)
if err != nil {
log.Fatalf("error executing batch: %v", err)
}
duration := time.Since(batchStart)
batchCount++
batchTime += duration
// reset batch
batch = index.NewBatch()
}
}
if leveli == 0 {
// run some queries
termQueryCount := 0
termQueryStart := time.Now()
termQuery := bleve.NewTermQuery("water")
termQuery.SetField("text")
termSearch := bleve.NewSearchRequest(termQuery)
searchResults, err := index.Search(termSearch)
if err != nil {
log.Fatalf("error searching: %v", err)
}
termQueryCount++
termQueryTime := time.Since(termQueryStart)
firstQueryTime := float64(termQueryTime)
for termQueryCount < *qrepeat {
termQueryStart = time.Now()
searchResults, err = index.Search(termSearch)
if err != nil {
log.Fatal(err)
}
termQueryCount++
termQueryTime += time.Since(termQueryStart)
}
// print stats
avgSingleDocTime := float64(singleTime) / float64(singleCount)
avgBatchTime := float64(batchTime) / float64(batchCount)
avgBatchDocTime := float64(avgBatchTime) / float64(*batchSize)
avgQueryTime := float64(termQueryTime) / float64(termQueryCount)
elapsedTime := time.Since(start) / time.Millisecond
fmt.Printf("%d,%d,%f,%f,%d,%f,%f", elapsedTime, i, avgSingleDocTime/float64(time.Millisecond), avgBatchDocTime/float64(time.Millisecond), searchResults.Total, firstQueryTime/float64(time.Millisecond), avgQueryTime/float64(time.Millisecond))
printOther(store)
lines[0].Pt[tot].Y = avgSingleDocTime / float64(time.Millisecond)
lines[0].Pt[tot].X = float64(i)
lines[1].Pt[tot].Y = avgBatchDocTime / float64(time.Millisecond)
lines[1].Pt[tot].X = float64(i)
lines[2].Pt[tot].Y = firstQueryTime / float64(time.Millisecond)
lines[2].Pt[tot].X = float64(i)
lines[3].Pt[tot].Y = avgQueryTime / float64(time.Millisecond)
lines[3].Pt[tot].X = float64(i)
tot++
fmt.Printf("\n")
// reset stats
singleCount = 0
singleTime = 0
batchCount = 0
batchTime = 0
// dump mem stats if requested
if *memprofile != "" {
f, err := os.Create(strconv.Itoa(i) + "-" + *memprofile)
if err != nil {
log.Fatal(err)
}
pprof.WriteHeapProfile(f)
}
}
}
return lines
}
func TestBeerSearchAll(t *testing.T) {
defer os.RemoveAll("beer-search-test.bleve")
mapping, err := buildIndexMapping()
if err != nil {
t.Fatal(err)
}
index, err := bleve.New("beer-search-test.bleve", mapping)
if err != nil {
t.Fatal(err)
}
defer index.Close()
for jf := range walkDirectory("data/", t) {
docId := jf.filename[0:strings.LastIndex(jf.filename, ".")]
err = index.Index(docId, jf.contents)
if err != nil {
t.Error(err)
}
}
expectedCount := uint64(7303)
actualCount := index.DocCount()
if actualCount != expectedCount {
t.Errorf("expected %d documents, got %d", expectedCount, actualCount)
}
// run a term search
termQuery := bleve.NewTermQuery("shock").SetField("name")
termSearchRequest := bleve.NewSearchRequest(termQuery)
termSearchResult, err := index.Search(termSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount := uint64(1)
if termSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total)
} else {
expectedResultId := "anheuser_busch-shock_top"
if termSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID)
}
}
// run a match phrase search
matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food")
matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery)
matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if matchPhraseSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total)
} else {
expectedResultId := "great_divide_brewing-wild_raspberry_ale"
if matchPhraseSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID)
}
}
// run a syntax query
syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley")
syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery)
syntaxSearchResult, err := index.Search(syntaxSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if syntaxSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total)
} else {
expectedResultId := "iron_city_brewing_co-ic_light"
if syntaxSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID)
}
}
// run a numeric range search
queryMin := 50.0
numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv")
numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery)
numericSearchResult, err := index.Search(numericSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if numericSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total)
} else {
expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale"
if numericSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID)
}
}
// run a date range search
queryStartDate := "2011-10-04"
dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated")
dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery)
dateSearchResult, err := index.Search(dateSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(2)
if dateSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total)
} else {
expectedResultId := "brasserie_du_bouffay-ambr"
if dateSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID)
}
}
// run a prefix search
prefixQuery := bleve.NewPrefixQuery("adir").SetField("name")
prefixSearchRequest := bleve.NewSearchRequest(prefixQuery)
prefixSearchResult, err := index.Search(prefixSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if prefixSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total)
} else {
expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager"
if prefixSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID)
}
}
}
