func (pages *Pages) QueryTerm(field string, term string) *Pages { q := bleve.NewTermQuery(term) q.SetField(field) pages.queries = append(pages.queries, q) pages.clear() return pages }
func (q *allMatchQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() } analyzerName := m.AnalyzerNameForPath(field) analyzer := m.AnalyzerNamed(analyzerName) tokens := analyzer.Analyze([]byte(q.Match)) if len(tokens) == 0 { noneQuery := bleve.NewMatchNoneQuery() return noneQuery.Searcher(i, m, explain) } tqs := make([]query.Query, len(tokens)) for i, token := range tokens { tq := bleve.NewTermQuery(string(token.Term)) tq.SetField(field) tq.SetBoost(q.BoostVal) tqs[i] = tq } allQuery := bleve.NewConjunctionQuery(tqs...) allQuery.SetBoost(q.BoostVal) return allQuery.Searcher(i, m, explain) }
// START OMIT func main() { index, err := bleve.Open("people.bleve") // HLOPEN if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") // HLQUERY request := bleve.NewSearchRequest(query) // HLREQ result, err := index.Search(request) // HLSEARCH if err != nil { log.Fatal(err) } fmt.Println(result) }
// this test reproduces bug #87 // https://github.com/blevesearch/bleve/issues/87 // because of which, it will deadlock func TestBeerSearchBug87(t *testing.T) { defer os.RemoveAll("beer-search-test.bleve") mapping, err := buildIndexMapping() if err != nil { t.Fatal(err) } index, err := bleve.New("beer-search-test.bleve", mapping) if err != nil { t.Fatal(err) } defer index.Close() // start indexing documents in the background go func() { // open the directory dirEntries, err := ioutil.ReadDir("data/") if err != nil { t.Fatal(err) } for _, dirEntry := range dirEntries { filename := dirEntry.Name() jsonBytes, err := ioutil.ReadFile("data/" + filename) if err != nil { t.Fatal(err) } ext := filepath.Ext(filename) docId := filename[:(len(filename) - len(ext))] index.Index(docId, jsonBytes) } }() // give indexing a head start time.Sleep(1 * time.Second) // start querying for i := 0; i < 1000; i++ { time.Sleep(1 * time.Millisecond) termQuery := bleve.NewTermQuery("shock").SetField("name") termSearchRequest := bleve.NewSearchRequest(termQuery) // termSearchRequest.AddFacet("styles", bleve.NewFacetRequest("style", 3)) termSearchRequest.Fields = []string{"abv"} _, err := index.Search(termSearchRequest) if err != nil { t.Error(err) } } }
// this test reproduces bug #87 // https://github.com/blevesearch/bleve/issues/87 // because of which, it will deadlock func TestBeerSearchBug87(t *testing.T) { defer os.RemoveAll("beer-search-test.bleve") mapping, err := buildIndexMapping() if err != nil { t.Fatal(err) } index, err := bleve.New("beer-search-test.bleve", mapping) if err != nil { t.Fatal(err) } defer index.Close() var wg sync.WaitGroup wg.Add(1) // start indexing documents in the background go func() { for jf := range walkDirectory("data/", t) { docId := jf.filename[0:strings.LastIndex(jf.filename, ".")] err = index.Index(docId, jf.contents) if err != nil { t.Error(err) } } wg.Done() }() for i := 0; i < 50; i++ { time.Sleep(1 * time.Second) termQuery := bleve.NewTermQuery("shock").SetField("name") termSearchRequest := bleve.NewSearchRequest(termQuery) // termSearchRequest.AddFacet("styles", bleve.NewFacetRequest("style", 3)) termSearchRequest.Fields = []string{"abv"} _, err := index.Search(termSearchRequest) if err != nil { t.Error(err) } } wg.Wait() }
// search executes a search for rooms or buildings. func (s *Server) search(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() q := query.Get("q") typeFilter := query.Get("type") results := []*models.Index{} if idx, ok := s.idIndex[q]; ok { results = append(results, idx) } else { query := bleve.NewBooleanQuery() if len(q) > 0 { /*fuzzy_query := bleve.NewFuzzyQuery(q) fuzzy_query.FuzzinessVal = 3 queryShould = append(queryShould, fuzzy_query) queryShould = append(queryShould, bleve.NewRegexpQuery("[a-zA-Z0-9_]*"+q+"[a-zA-Z0-9_]*")) queryShould = append(queryShould, bleve.NewQueryStringQuery(q))*/ query.AddShould(bleve.NewQueryStringQuery(q)) } if typeFilter != "all" { termQuery := bleve.NewTermQuery(typeFilter) query.AddMust(termQuery) } searchRequest := bleve.NewSearchRequest(query) searchRequest.Size = 25 searchResult, err := s.index.Search(searchRequest) if err != nil { http.Error(w, err.Error(), 500) return } for _, result := range []*search.DocumentMatch(searchResult.Hits) { results = append(results, s.idIndex[result.ID]) } } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(results) }
func buildQuery(args []string) query.Query { var q query.Query switch qtype { case "prefix": pquery := bleve.NewPrefixQuery(strings.Join(args[1:], " ")) if qfield != "" { pquery.SetField(qfield) } q = pquery case "term": pquery := bleve.NewTermQuery(strings.Join(args[1:], " ")) if qfield != "" { pquery.SetField(qfield) } q = pquery default: // build a search with the provided parameters queryString := strings.Join(args[1:], " ") q = bleve.NewQueryStringQuery(queryString) } return q }
func query(term, highlight string, index bleve.Index, u content.User, feedIds []data.FeedId, paging ...int) (ua []content.UserArticle, err error) { var query bleve.Query query = bleve.NewQueryStringQuery(term) if len(feedIds) > 0 { queries := make([]bleve.Query, len(feedIds)) conjunct := make([]bleve.Query, 2) for i, id := range feedIds { q := bleve.NewTermQuery(strconv.FormatInt(int64(id), 10)) q.SetField("FeedId") queries[i] = q } disjunct := bleve.NewDisjunctionQuery(queries) conjunct[0] = query conjunct[1] = disjunct query = bleve.NewConjunctionQuery(conjunct) } searchRequest := bleve.NewSearchRequest(query) if highlight != "" { searchRequest.Highlight = bleve.NewHighlightWithStyle(highlight) } limit, offset := pagingLimit(paging) searchRequest.Size = limit searchRequest.From = offset searchResult, err := index.Search(searchRequest) if err != nil { return } if len(searchResult.Hits) == 0 { return } articleIds := []data.ArticleId{} hitMap := map[data.ArticleId]*search.DocumentMatch{} for _, hit := range searchResult.Hits { if articleId, err := strconv.ParseInt(hit.ID, 10, 64); err == nil { id := data.ArticleId(articleId) articleIds = append(articleIds, id) hitMap[id] = hit } } ua = u.ArticlesById(articleIds) if u.HasErr() { return ua, u.Err() } for i := range ua { data := ua[i].Data() hit := hitMap[data.Id] if len(hit.Fragments) > 0 { data.Hit.Fragments = hit.Fragments ua[i].Data(data) } } return }
func TestBeerSearchAll(t *testing.T) { defer os.RemoveAll("beer-search-test.bleve") mapping, err := buildIndexMapping() if err != nil { t.Fatal(err) } index, err := bleve.New("beer-search-test.bleve", mapping) if err != nil { t.Fatal(err) } defer index.Close() for jf := range walkDirectory("data/", t) { docId := jf.filename[0:strings.LastIndex(jf.filename, ".")] err = index.Index(docId, jf.contents) if err != nil { t.Error(err) } } expectedCount := uint64(7303) actualCount := index.DocCount() if actualCount != expectedCount { t.Errorf("expected %d documents, got %d", expectedCount, actualCount) } // run a term search termQuery := bleve.NewTermQuery("shock").SetField("name") termSearchRequest := bleve.NewSearchRequest(termQuery) termSearchResult, err := index.Search(termSearchRequest) if err != nil { t.Error(err) } expectedResultCount := uint64(1) if termSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total) } else { expectedResultId := "anheuser_busch-shock_top" if termSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID) } } // run a match phrase search matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food") matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery) matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest) if err != nil { t.Error(err) } expectedResultCount = uint64(1) if matchPhraseSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total) } else { expectedResultId := "great_divide_brewing-wild_raspberry_ale" if matchPhraseSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID) } } // run a syntax query syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley") syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery) syntaxSearchResult, err := index.Search(syntaxSearchRequest) if err != nil { t.Error(err) } expectedResultCount = uint64(1) if syntaxSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total) } else { expectedResultId := "iron_city_brewing_co-ic_light" if syntaxSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID) } } // run a numeric range search queryMin := 50.0 numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv") numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery) numericSearchResult, err := index.Search(numericSearchRequest) if err != nil { t.Error(err) } expectedResultCount = uint64(1) if numericSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total) } else { expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale" if numericSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID) } } // run a date range search queryStartDate := "2011-10-04" dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated") dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery) dateSearchResult, err := index.Search(dateSearchRequest) if err != nil { t.Error(err) } expectedResultCount = uint64(2) if dateSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total) } else { expectedResultId := "brasserie_du_bouffay-ambr" if dateSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID) } } // run a prefix search prefixQuery := bleve.NewPrefixQuery("adir").SetField("name") prefixSearchRequest := bleve.NewSearchRequest(prefixQuery) prefixSearchResult, err := index.Search(prefixSearchRequest) if err != nil { t.Error(err) } expectedResultCount = uint64(1) if prefixSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total) } else { expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager" if prefixSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID) } } }
func runConfig(conf string, tar string, cpu string) []*Line { if cpu != "" { f, err := os.Create(cpu) if err != nil { log.Fatal(err) } pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } start := time.Now() wikiReader, err := blevebench.NewWikiReader(*source) if err != nil { log.Fatal(err) } defer wikiReader.Close() mapping := blevebench.BuildArticleMapping() benchConfig := blevebench.LoadConfigFile(conf) fmt.Printf("Using Index Type: %s\n", benchConfig.IndexType) fmt.Printf("Using KV store: %s\n", benchConfig.KVStore) fmt.Printf("Using KV config: %#v\n", benchConfig.KVConfig) index, err := bleve.NewUsing(tar, mapping, benchConfig.IndexType, benchConfig.KVStore, benchConfig.KVConfig) if err != nil { log.Fatal(err) } _, store, err := index.Advanced() if err != nil { log.Fatal(err) } itr := *count / (*level) lines := NewLines(itr, len(typename), conf, typename) tot := 0 // print header fmt.Printf("elapsed,docs,avg_single_doc_ms,avg_batched_doc_ms,query_water_matches,first_query_water_ms,avg_repeated%d_query_water_ms", *qrepeat) printOtherHeader(store) fmt.Printf("\n") singleCount := 0 var singleTime time.Duration batchCount := 0 var batchTime time.Duration batch := index.NewBatch() for i := 1; i < (*count)+1; i++ { leveli := i % *level a, err := wikiReader.Next() if err != nil { log.Fatal(err) } if leveli < *batchSize { // index single singleStart := time.Now() err = index.Index(a.Title, a) if err != nil { log.Fatalf("error indexing: %v", err) } duration := time.Since(singleStart) singleCount++ singleTime += duration } else { // add to batch batch.Index(a.Title, a) // if batch is full index it if batch.Size() == *batchSize { batchStart := time.Now() err := index.Batch(batch) if err != nil { log.Fatalf("error executing batch: %v", err) } duration := time.Since(batchStart) batchCount++ batchTime += duration // reset batch batch = index.NewBatch() } } if leveli == 0 { // run some queries termQueryCount := 0 termQueryStart := time.Now() termQuery := bleve.NewTermQuery("water") termQuery.SetField("text") termSearch := bleve.NewSearchRequest(termQuery) searchResults, err := index.Search(termSearch) if err != nil { log.Fatalf("error searching: %v", err) } termQueryCount++ termQueryTime := time.Since(termQueryStart) firstQueryTime := float64(termQueryTime) for termQueryCount < *qrepeat { termQueryStart = time.Now() searchResults, err = index.Search(termSearch) if err != nil { log.Fatal(err) } termQueryCount++ termQueryTime += time.Since(termQueryStart) } // print stats avgSingleDocTime := float64(singleTime) / float64(singleCount) avgBatchTime := float64(batchTime) / float64(batchCount) avgBatchDocTime := float64(avgBatchTime) / float64(*batchSize) avgQueryTime := float64(termQueryTime) / float64(termQueryCount) elapsedTime := time.Since(start) / time.Millisecond fmt.Printf("%d,%d,%f,%f,%d,%f,%f", elapsedTime, i, avgSingleDocTime/float64(time.Millisecond), avgBatchDocTime/float64(time.Millisecond), searchResults.Total, firstQueryTime/float64(time.Millisecond), avgQueryTime/float64(time.Millisecond)) printOther(store) lines[0].Pt[tot].Y = avgSingleDocTime / float64(time.Millisecond) lines[0].Pt[tot].X = float64(i) lines[1].Pt[tot].Y = avgBatchDocTime / float64(time.Millisecond) lines[1].Pt[tot].X = float64(i) lines[2].Pt[tot].Y = firstQueryTime / float64(time.Millisecond) lines[2].Pt[tot].X = float64(i) lines[3].Pt[tot].Y = avgQueryTime / float64(time.Millisecond) lines[3].Pt[tot].X = float64(i) tot++ fmt.Printf("\n") // reset stats singleCount = 0 singleTime = 0 batchCount = 0 batchTime = 0 // dump mem stats if requested if *memprofile != "" { f, err := os.Create(strconv.Itoa(i) + "-" + *memprofile) if err != nil { log.Fatal(err) } pprof.WriteHeapProfile(f) } } } return lines }
func TestBeerSearchAll(t *testing.T) { defer os.RemoveAll("beer-search-test.bleve") mapping, err := buildIndexMapping() if err != nil { t.Fatal(err) } index, err := bleve.New("beer-search-test.bleve", mapping) if err != nil { t.Fatal(err) } defer index.Close() // open the directory dirEntries, err := ioutil.ReadDir("data/") if err != nil { t.Fatal(err) } indexBatchSize := 100 batch := index.NewBatch() batchCount := 0 for _, dirEntry := range dirEntries { filename := dirEntry.Name() // read the bytes jsonBytes, err := ioutil.ReadFile("data/" + filename) if err != nil { t.Fatal(err) } // // shred them into a document ext := filepath.Ext(filename) docId := filename[:(len(filename) - len(ext))] batch.Index(docId, jsonBytes) batchCount++ if batchCount >= indexBatchSize { err = index.Batch(batch) if err != nil { t.Fatal(err) } batch = index.NewBatch() batchCount = 0 } } // flush the last batch if batchCount > 0 { err = index.Batch(batch) if err != nil { t.Fatal(err) } } expectedCount := uint64(7303) actualCount, err := index.DocCount() if err != nil { t.Error(err) } if actualCount != expectedCount { t.Errorf("expected %d documents, got %d", expectedCount, actualCount) } // run a term search termQuery := bleve.NewTermQuery("shock").SetField("name") termSearchRequest := bleve.NewSearchRequest(termQuery) termSearchResult, err := index.Search(termSearchRequest) if err != nil { t.Error(err) } expectedResultCount := uint64(1) if termSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total) } else { expectedResultId := "anheuser_busch-shock_top" if termSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID) } } // run a match phrase search matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food") matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery) matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest) if err != nil { t.Error(err) } expectedResultCount = uint64(1) if matchPhraseSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total) } else { expectedResultId := "great_divide_brewing-wild_raspberry_ale" if matchPhraseSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID) } } // run a syntax query syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley") syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery) syntaxSearchResult, err := index.Search(syntaxSearchRequest) if err != nil { t.Error(err) } expectedResultCount = uint64(1) if syntaxSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total) } else { expectedResultId := "iron_city_brewing_co-ic_light" if syntaxSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID) } } // run a numeric range search queryMin := 50.0 numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv") numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery) numericSearchResult, err := index.Search(numericSearchRequest) if err != nil { t.Error(err) } expectedResultCount = uint64(1) if numericSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total) } else { expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale" if numericSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID) } } // run a date range search queryStartDate := "2011-10-04" dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated") dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery) dateSearchResult, err := index.Search(dateSearchRequest) if err != nil { t.Error(err) } expectedResultCount = uint64(2) if dateSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total) } else { expectedResultId := "brasserie_du_bouffay-ambr" if dateSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID) } } // run a prefix search prefixQuery := bleve.NewPrefixQuery("adir").SetField("name") prefixSearchRequest := bleve.NewSearchRequest(prefixQuery) prefixSearchResult, err := index.Search(prefixSearchRequest) if err != nil { t.Error(err) } expectedResultCount = uint64(1) if prefixSearchResult.Total != expectedResultCount { t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total) } else { expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager" if prefixSearchResult.Hits[0].ID != expectedResultId { t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID) } } }
func buildMatchQuery() *bleve.SearchRequest { termQuery := bleve.NewTermQuery("water") termQuery.SetField("text") termSearch := bleve.NewSearchRequest(termQuery) return termSearch }
func main() { flag.Parse() if *cpuprofile != "" { f, err := os.Create(*cpuprofile) if err != nil { log.Fatal(err) } err = pprof.StartCPUProfile(f) if err != nil { log.Fatal(err) } defer pprof.StopCPUProfile() } if *indexPath == "" { log.Fatal("Specify index to query") } if flag.NArg() < 1 { log.Fatal("Specify search query") } // open index index, err := bleve.Open(*indexPath) if err != nil { log.Fatal(err) } defer func() { cerr := index.Close() if cerr != nil { log.Fatalf("error closing index: %v", err) } }() for i := 0; i < *repeat; i++ { var query bleve.Query switch *qtype { case "prefix": pquery := bleve.NewPrefixQuery(strings.Join(flag.Args(), " ")) if *qfield != "" { pquery.SetField(*qfield) } query = pquery case "term": pquery := bleve.NewTermQuery(strings.Join(flag.Args(), " ")) if *qfield != "" { pquery.SetField(*qfield) } query = pquery default: // build a search with the provided parameters queryString := strings.Join(flag.Args(), " ") query = bleve.NewQueryStringQuery(queryString) } searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain) // enable highlights if requested if *includeHighlights { searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi") } if *includeStoredFields { searchRequest.Fields = []string{"*"} } // execute the search searchResult, err := index.Search(searchRequest) if err != nil { log.Fatalf("search error: %v", err) } fmt.Println(searchResult) } }