func BlevePIndexQuerySamples() []cbgt.Documentation {
return []cbgt.Documentation{
cbgt.Documentation{
Text: "A simple bleve query POST body:",
JSON: &struct {
*cbgt.QueryCtlParams
*bleve.SearchRequest
}{
nil,
&bleve.SearchRequest{
From: 0,
Size: 10,
Query: bleve.NewQueryStringQuery("a sample query"),
},
},
},
cbgt.Documentation{
Text: `An example POST body using from/size for results paging,
using ctl for a timeout and for "at_plus" consistency level.
On consistency, the index must have incorporated at least mutation
sequence-number 123 for partition (vbucket) 0 and mutation
sequence-number 234 for partition (vbucket) 1 (where vbucket 1
should have a vbucketUUID of a0b1c2):`,
JSON: &struct {
*cbgt.QueryCtlParams
*bleve.SearchRequest
}{
&cbgt.QueryCtlParams{
Ctl: cbgt.QueryCtl{
Timeout: cbgt.QUERY_CTL_DEFAULT_TIMEOUT_MS,
Consistency: &cbgt.ConsistencyParams{
Level: "at_plus",
Vectors: map[string]cbgt.ConsistencyVector{
"customerIndex": cbgt.ConsistencyVector{
"0": 123,
"1/a0b1c2": 234,
},
},
},
},
},
&bleve.SearchRequest{
From: 20,
Size: 10,
Fields: []string{"*"},
Query: bleve.NewQueryStringQuery("alice smith"),
Highlight: bleve.NewHighlight(),
Explain: true,
},
},
},
}
}
func (pages *Pages) QueryString(field string, str string) *Pages {
q := bleve.NewQueryStringQuery(str)
q.SetField(field)
pages.queries = append(pages.queries, q)
pages.clear()
return pages
}
// Search is an http.HandlerFunc that accepts a Parse Cloud Code Webhook request.
//
// The expected query parameter is 'q'
func (i *Indexer) Search(w http.ResponseWriter, r *http.Request) {
req, err := webhookRequest(r, i.webhookKey)
if err != nil {
writeErr(w, err)
return
}
rawq := req.Params["q"]
if rawq == nil {
writeErr(w, fmt.Errorf("no term provided"))
return
}
q, ok := rawq.(string)
if q == "" || !ok {
writeErr(w, fmt.Errorf("no term provided"))
return
}
query := bleve.NewQueryStringQuery(q)
search := bleve.NewSearchRequest(query)
searchResults, err := i.index.Search(search)
if err != nil {
writeErr(w, err)
return
}
spew.Dump(searchResults)
ids := []string{}
for _, h := range searchResults.Hits {
ids = append(ids, h.ID)
}
err = json.NewEncoder(w).Encode(Response{
Success: ids,
})
if err != nil {
log.Println("error encoding response:", err)
}
}
func (docstore *DocStoreExt) Search(collection, queryString string) ([]byte, error) {
js := []byte{}
query := bleve.NewQueryStringQuery(queryString)
searchRequest := bleve.NewSearchRequest(query)
searchResult, err := docstore.index.Search(searchRequest)
if err != nil {
return nil, err
}
for index, sr := range searchResult.Hits {
jsPart := []byte{}
_, err := docstore.fetchDoc(collection, sr.ID, &jsPart)
if err != nil {
return nil, err
}
js = append(js, addID(jsPart, sr.ID)...)
if index != len(searchResult.Hits)-1 {
js = append(js, []byte(",")...)
}
}
if len(searchResult.Hits) > 0 {
js = js[0 : len(js)-1]
}
js = append(js, []byte("]")...)
// "_meta": searchResult,
// "data": docs,
// TODO(tsileo) returns meta along with argument
return js, nil
}
// Search method lookup for records using a query
func (i *bleveIndexer) Search(q string) (records []indexer.Record) {
query := bleve.NewQueryStringQuery(q)
request := bleve.NewSearchRequest(query)
request.Highlight = bleve.NewHighlight()
result, err := i.bleve.Search(request)
if err != nil { // an empty query would cause this
return
}
for _, match := range result.Hits {
rec := i.Record(match.ID)
loaded := rec.Load()
if !loaded {
continue
}
if len(match.Fragments["Body"]) > 0 {
rec.SetBody([]byte(match.Fragments["Body"][0]))
}
records = append(records, rec)
}
return
}
func (b *Index) Search(queryStr string, offset, limit int, highlights bool) (*bleve.SearchResult, error) {
query := bleve.NewQueryStringQuery(queryStr)
req := bleve.NewSearchRequestOptions(query, limit, offset, false)
if highlights {
req.Highlight = bleve.NewHighlightWithStyle("ansi")
}
return b.index.Search(req)
}
func main() {
flag.Parse()
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
err = pprof.StartCPUProfile(f)
if err != nil {
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
if *indexPath == "" {
log.Fatal("Specify index to query")
}
if flag.NArg() < 1 {
log.Fatal("Specify search query")
}
// open index
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}
defer func() {
cerr := index.Close()
if cerr != nil {
log.Fatalf("error closing index: %v", err)
}
}()
for i := 0; i < *repeat; i++ {
// build a search with the provided parameters
queryString := strings.Join(flag.Args(), " ")
query := bleve.NewQueryStringQuery(queryString)
searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain)
// enable highlights if requested
if *includeHighlights {
searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi")
}
// execute the search
searchResult, err := index.Search(searchRequest)
if err != nil {
log.Fatalf("search error: %v", err)
}
fmt.Println(searchResult)
}
}
func (be *BleveEngine) Search(query string) (interface{}, error) {
index, _ := bleve.Open(INDEX)
bleveQuery := bleve.NewQueryStringQuery(query)
searchRequest := bleve.NewSearchRequest(bleveQuery)
searchResults, err := index.Search(searchRequest)
if err != nil {
return nil, err
}
return searchResults, nil
}
// Search performs a search of the index using the given query. Returns IDs of documents
// which satisfy all queries. Returns Doc IDs in sorted order, ascending.
func (i *Index) Search(q string) (DocIDs, error) {
query := bleve.NewQueryStringQuery(q)
searchRequest := bleve.NewSearchRequest(query)
searchRequest.Size = maxSearchHitSize
searchResults, err := i.Alias.Search(searchRequest)
if err != nil {
return nil, err
}
docIDs := make(DocIDs, 0, len(searchResults.Hits))
for _, d := range searchResults.Hits {
docIDs = append(docIDs, DocID(d.ID))
}
sort.Sort(docIDs)
return docIDs, nil
}
// Find all comments matching text
// @return result.Result<[]*Comment, error>
func CommentsWithContent(repoPath, content string) result.Result {
return openIndex(repoPath, func(repo *git.Repository, index bleve.Index) result.Result {
query := bleve.NewQueryStringQuery(content)
request := bleve.NewSearchRequest(query)
return result.NewResult(index.Search(request)).FlatMap(func(match interface{}) result.Result {
hits := match.(*bleve.SearchResult).Hits
comments := make([]*gc.Comment, len(hits))
for idx, hit := range hits {
gc.CommentByID(repo, hit.ID).FlatMap(func(comment interface{}) result.Result {
comments[idx] = comment.(*gc.Comment)
return result.Result{}
})
}
return result.NewSuccess(comments)
})
})
}
func (cache *BleveCache) Search(ctx context.Context, term string, options SearchOptions) ([]*RecipeLink, error) {
q := bleve.NewQueryStringQuery(term)
req := bleve.NewSearchRequest(q)
res, err := cache.index.Search(req)
if err != nil {
return nil, err
}
links := make([]*RecipeLink, len(res.Hits))
for i, hit := range res.Hits {
filePath := cache.dbPath + "/" + hit.ID + ".json"
links[i], err = loadRecipeLink(filePath)
if err != nil {
return links, err
}
}
return links, err
}
// Search the swish index for a given query.
func searchBleve(queryString string) (Results, error) {
index, err := openIndex()
if err != nil {
LogError(err)
return Results{}, err
}
defer index.Close()
newQueryString := ""
for _, tmp := range strings.Split(queryString, " ") {
word := strings.TrimSpace(tmp)
if word[0] == '-' || word[0] == '+' {
newQueryString += " " + word
} else if word[0] == '~' {
// Remove prefix to make term optional
newQueryString += " " + word[1:]
} else {
newQueryString += " +" + word
}
}
query := bleve.NewQueryStringQuery(newQueryString[1:]) // Remove leading space
search := bleve.NewSearchRequest(query)
search.Size = Config.MaxResults
searchResults, err := index.Search(search)
if err != nil {
println("Invalid query string: '" + newQueryString[1:] + "'")
LogError(err)
return Results{}, err
}
var ids []Scroll
for _, match := range searchResults.Hits {
id := Id(match.ID)
content, err := readScroll(id)
TryLogError(err)
scroll := Parse(string(id), content)
ids = append(ids, scroll)
}
return Results{ids[:len(searchResults.Hits)], int(searchResults.Total)}, nil
}
// search executes a search for rooms or buildings.
func (s *Server) search(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
q := query.Get("q")
typeFilter := query.Get("type")
results := []*models.Index{}
if idx, ok := s.idIndex[q]; ok {
results = append(results, idx)
} else {
query := bleve.NewBooleanQuery()
if len(q) > 0 {
/*fuzzy_query := bleve.NewFuzzyQuery(q)
fuzzy_query.FuzzinessVal = 3
queryShould = append(queryShould, fuzzy_query)
queryShould = append(queryShould, bleve.NewRegexpQuery("[a-zA-Z0-9_]*"+q+"[a-zA-Z0-9_]*"))
queryShould = append(queryShould, bleve.NewQueryStringQuery(q))*/
query.AddShould(bleve.NewQueryStringQuery(q))
}
if typeFilter != "all" {
termQuery := bleve.NewTermQuery(typeFilter)
query.AddMust(termQuery)
}
searchRequest := bleve.NewSearchRequest(query)
searchRequest.Size = 25
searchResult, err := s.index.Search(searchRequest)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
for _, result := range []*search.DocumentMatch(searchResult.Hits) {
results = append(results, s.idIndex[result.ID])
}
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results)
}
// START OMIT
func main() {
index, err := bleve.Open("gophercon.bleve")
if err != nil {
log.Fatal(err)
}
qString := `+content:debugger ` // HLQUERY
qString += `title:"delve" ` // HLQUERY
qString += `title:go~2 ` // HLQUERY
qString += `-content:rust ` // HLQUERY
qString += `word_count:>30` // HLQUERY
q := bleve.NewQueryStringQuery(qString)
req := bleve.NewSearchRequest(q)
req.Highlight = bleve.NewHighlightWithStyle("html")
req.Fields = []string{"title", "author", "content", "word_count"}
res, err := index.Search(req)
if err != nil {
log.Fatal(err)
}
fmt.Println(res)
}
func buildQuery(args []string) query.Query {
var q query.Query
switch qtype {
case "prefix":
pquery := bleve.NewPrefixQuery(strings.Join(args[1:], " "))
if qfield != "" {
pquery.SetField(qfield)
}
q = pquery
case "term":
pquery := bleve.NewTermQuery(strings.Join(args[1:], " "))
if qfield != "" {
pquery.SetField(qfield)
}
q = pquery
default:
// build a search with the provided parameters
queryString := strings.Join(args[1:], " ")
q = bleve.NewQueryStringQuery(queryString)
}
return q
}
func main() {
flag.Parse()
if *indexPath == "" {
log.Fatal("specify index to query")
}
if flag.NArg() < 1 {
log.Fatal("Specify search query")
}
// open index
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}
defer index.Close()
// build a search with the provided parameters
queryString := strings.Join(flag.Args(), " ")
query := bleve.NewQueryStringQuery(queryString)
searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain)
// enable highlights if requested
if *includeHighlights {
searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi")
}
// execute the search
searchResult, err := index.Search(searchRequest)
if err != nil {
log.Fatalf("search error: %v", err)
}
fmt.Println(searchResult)
}
func main() {
flag.Parse()
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
err = pprof.StartCPUProfile(f)
if err != nil {
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
if *indexPath == "" {
log.Fatal("Specify index to query")
}
if flag.NArg() < 1 {
log.Fatal("Specify search query")
}
// open index
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}
defer func() {
cerr := index.Close()
if cerr != nil {
log.Fatalf("error closing index: %v", err)
}
}()
for i := 0; i < *repeat; i++ {
var query bleve.Query
switch *qtype {
case "prefix":
pquery := bleve.NewPrefixQuery(strings.Join(flag.Args(), " "))
if *qfield != "" {
pquery.SetField(*qfield)
}
query = pquery
case "term":
pquery := bleve.NewTermQuery(strings.Join(flag.Args(), " "))
if *qfield != "" {
pquery.SetField(*qfield)
}
query = pquery
default:
// build a search with the provided parameters
queryString := strings.Join(flag.Args(), " ")
query = bleve.NewQueryStringQuery(queryString)
}
searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain)
// enable highlights if requested
if *includeHighlights {
searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi")
}
if *includeStoredFields {
searchRequest.Fields = []string{"*"}
}
// execute the search
searchResult, err := index.Search(searchRequest)
if err != nil {
log.Fatalf("search error: %v", err)
}
fmt.Println(searchResult)
}
}
func query(term, highlight string, index bleve.Index, u content.User, feedIds []data.FeedId, paging ...int) (ua []content.UserArticle, err error) {
var query bleve.Query
query = bleve.NewQueryStringQuery(term)
if len(feedIds) > 0 {
queries := make([]bleve.Query, len(feedIds))
conjunct := make([]bleve.Query, 2)
for i, id := range feedIds {
q := bleve.NewTermQuery(strconv.FormatInt(int64(id), 10))
q.SetField("FeedId")
queries[i] = q
}
disjunct := bleve.NewDisjunctionQuery(queries)
conjunct[0] = query
conjunct[1] = disjunct
query = bleve.NewConjunctionQuery(conjunct)
}
searchRequest := bleve.NewSearchRequest(query)
if highlight != "" {
searchRequest.Highlight = bleve.NewHighlightWithStyle(highlight)
}
limit, offset := pagingLimit(paging)
searchRequest.Size = limit
searchRequest.From = offset
searchResult, err := index.Search(searchRequest)
if err != nil {
return
}
if len(searchResult.Hits) == 0 {
return
}
articleIds := []data.ArticleId{}
hitMap := map[data.ArticleId]*search.DocumentMatch{}
for _, hit := range searchResult.Hits {
if articleId, err := strconv.ParseInt(hit.ID, 10, 64); err == nil {
id := data.ArticleId(articleId)
articleIds = append(articleIds, id)
hitMap[id] = hit
}
}
ua = u.ArticlesById(articleIds)
if u.HasErr() {
return ua, u.Err()
}
for i := range ua {
data := ua[i].Data()
hit := hitMap[data.Id]
if len(hit.Fragments) > 0 {
data.Hit.Fragments = hit.Fragments
ua[i].Data(data)
}
}
return
}
func Search(q string) (*bleve.SearchResult, error) {
query := bleve.NewQueryStringQuery(q)
req := bleve.NewSearchRequest(query)
req.Highlight = bleve.NewHighlightWithStyle("html_ex")
return index.Search(req)
}
func Example() {
INDEX_DIR := "gojieba.bleve"
messages := []struct {
Id string
Body string
}{
{
Id: "1",
Body: "你好",
},
{
Id: "2",
Body: "交代",
},
{
Id: "3",
Body: "长江大桥",
},
}
indexMapping := bleve.NewIndexMapping()
os.RemoveAll(INDEX_DIR)
// clean index when example finished
defer os.RemoveAll(INDEX_DIR)
err := indexMapping.AddCustomTokenizer("gojieba",
map[string]interface{}{
"dictpath": gojieba.DICT_PATH,
"hmmpath": gojieba.HMM_PATH,
"userdictpath": gojieba.USER_DICT_PATH,
"idf": gojieba.IDF_PATH,
"stop_words": gojieba.STOP_WORDS_PATH,
"type": "gojieba",
},
)
if err != nil {
panic(err)
}
err = indexMapping.AddCustomAnalyzer("gojieba",
map[string]interface{}{
"type": "gojieba",
"tokenizer": "gojieba",
},
)
if err != nil {
panic(err)
}
indexMapping.DefaultAnalyzer = "gojieba"
index, err := bleve.New(INDEX_DIR, indexMapping)
if err != nil {
panic(err)
}
for _, msg := range messages {
if err := index.Index(msg.Id, msg); err != nil {
panic(err)
}
}
querys := []string{
"你好世界",
"亲口交代",
"长江",
}
for _, q := range querys {
req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(q))
req.Highlight = bleve.NewHighlight()
res, err := index.Search(req)
if err != nil {
panic(err)
}
fmt.Println(prettify(res))
}
// Output:
// [{"id":"1","score":0.27650412875470115}]
// [{"id":"2","score":0.27650412875470115}]
// [{"id":"3","score":0.7027325540540822}]
}
func searchHandler(w http.ResponseWriter, r *http.Request) {
d := map[string]interface{}{
"Path": path.Base(*zimPath),
}
if !idx {
templates["searchNoIdx"].Execute(w, d)
return
}
if r.Method == "GET" {
templates["search"].Execute(w, d)
return
}
q := r.FormValue("search_data")
if q == "" {
templates["search"].Execute(w, d)
return
}
query := bleve.NewQueryStringQuery(q)
search := bleve.NewSearchRequest(query)
search.Fields = []string{"Title"}
sr, err := index.Search(search)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
if sr.Total > 0 {
d["Info"] = fmt.Sprintf("%d matches for query [%s], took %s", sr.Total, q, sr.Took)
// Constructs a list of Hits
var l []map[string]string
for _, h := range sr.Hits {
idx, err := strconv.Atoi(h.ID)
if err != nil {
log.Println(err.Error())
continue
}
a, err := Z.ArticleAtURLIdx(uint32(idx))
if err != nil {
continue
}
l = append(l, map[string]string{
"Score": strconv.FormatFloat(h.Score, 'f', 1, 64),
"Title": a.Title,
"URL": "/zim/" + a.FullURL()})
}
d["Hits"] = l
} else {
d["Info"] = fmt.Sprintf("No match for [%s], took %s", q, sr.Took)
d["Hits"] = 0
}
templates["searchResult"].Execute(w, d)
}
func TestBeerSearchAll(t *testing.T) {
defer os.RemoveAll("beer-search-test.bleve")
mapping, err := buildIndexMapping()
if err != nil {
t.Fatal(err)
}
index, err := bleve.New("beer-search-test.bleve", mapping)
if err != nil {
t.Fatal(err)
}
defer index.Close()
// open the directory
dirEntries, err := ioutil.ReadDir("data/")
if err != nil {
t.Fatal(err)
}
indexBatchSize := 100
batch := index.NewBatch()
batchCount := 0
for _, dirEntry := range dirEntries {
filename := dirEntry.Name()
// read the bytes
jsonBytes, err := ioutil.ReadFile("data/" + filename)
if err != nil {
t.Fatal(err)
}
// // shred them into a document
ext := filepath.Ext(filename)
docId := filename[:(len(filename) - len(ext))]
batch.Index(docId, jsonBytes)
batchCount++
if batchCount >= indexBatchSize {
err = index.Batch(batch)
if err != nil {
t.Fatal(err)
}
batch = index.NewBatch()
batchCount = 0
}
}
// flush the last batch
if batchCount > 0 {
err = index.Batch(batch)
if err != nil {
t.Fatal(err)
}
}
expectedCount := uint64(7303)
actualCount, err := index.DocCount()
if err != nil {
t.Error(err)
}
if actualCount != expectedCount {
t.Errorf("expected %d documents, got %d", expectedCount, actualCount)
}
// run a term search
termQuery := bleve.NewTermQuery("shock").SetField("name")
termSearchRequest := bleve.NewSearchRequest(termQuery)
termSearchResult, err := index.Search(termSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount := uint64(1)
if termSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total)
} else {
expectedResultId := "anheuser_busch-shock_top"
if termSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID)
}
}
// run a match phrase search
matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food")
matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery)
matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if matchPhraseSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total)
} else {
expectedResultId := "great_divide_brewing-wild_raspberry_ale"
if matchPhraseSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID)
}
}
// run a syntax query
syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley")
syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery)
syntaxSearchResult, err := index.Search(syntaxSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if syntaxSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total)
} else {
expectedResultId := "iron_city_brewing_co-ic_light"
if syntaxSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID)
}
}
// run a numeric range search
queryMin := 50.0
numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv")
numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery)
numericSearchResult, err := index.Search(numericSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if numericSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total)
} else {
expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale"
if numericSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID)
}
}
// run a date range search
queryStartDate := "2011-10-04"
dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated")
dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery)
dateSearchResult, err := index.Search(dateSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(2)
if dateSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total)
} else {
expectedResultId := "brasserie_du_bouffay-ambr"
if dateSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID)
}
}
// run a prefix search
prefixQuery := bleve.NewPrefixQuery("adir").SetField("name")
prefixSearchRequest := bleve.NewSearchRequest(prefixQuery)
prefixSearchResult, err := index.Search(prefixSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if prefixSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total)
} else {
expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager"
if prefixSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID)
}
}
}
func resultsHandler(w http.ResponseWriter, r *http.Request) {
var (
pageHTML = "results-search.html"
pageInclude = "results-search.include"
)
urlQuery := r.URL.Query()
err := r.ParseForm()
if err != nil {
responseLogger(r, http.StatusBadRequest, err)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(fmt.Sprintf("error in POST: %s", err)))
return
}
// Collect the submissions fields.
submission := make(map[string]interface{})
// Basic Search results
if r.Method == "GET" {
for k, v := range urlQuery {
if k == "all_ids" {
if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil {
submission[k] = b
}
} else if k == "from" || k == "size" || k == "total" {
if i, err := strconv.Atoi(strings.Join(v, "")); err == nil {
submission[k] = i
}
} else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" {
submission[k] = strings.Join(v, "")
}
}
}
// Advanced Search results
if r.Method == "POST" {
for k, v := range r.Form {
if k == "all_ids" {
if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil {
submission[k] = b
}
} else if k == "from" || k == "size" || k == "total" {
if i, err := strconv.Atoi(strings.Join(v, "")); err == nil {
submission[k] = i
}
} else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" {
submission[k] = strings.Join(v, "")
}
}
}
q, err := mapToSearchQuery(submission)
if err != nil {
responseLogger(r, http.StatusBadRequest, err)
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(fmt.Sprintf("%s", err)))
return
}
//
// Note: Add logic to handle basic and advanced search...
//
// q NewQueryStringQuery
// q_required NewQueryStringQuery with a + prefix for each strings.Fields(q_required) value
// q_exact NewMatchPhraseQuery
// q_excluded NewQueryStringQuery with a - prefix for each strings.Feilds(q_excluded) value
//
var conQry []bleve.Query
if q.Q != "" {
conQry = append(conQry, bleve.NewQueryStringQuery(q.Q))
}
if q.QExact != "" {
conQry = append(conQry, bleve.NewMatchPhraseQuery(q.QExact))
}
var terms []string
for _, s := range strings.Fields(q.QRequired) {
terms = append(terms, fmt.Sprintf("+%s", strings.TrimSpace(s)))
}
for _, s := range strings.Fields(q.QExcluded) {
terms = append(terms, fmt.Sprintf("-%s", strings.TrimSpace(s)))
}
if len(terms) > 0 {
qString := strings.Join(terms, " ")
conQry = append(conQry, bleve.NewQueryStringQuery(qString))
}
qry := bleve.NewConjunctionQuery(conQry)
if q.Size == 0 {
q.Size = 10
}
searchRequest := bleve.NewSearchRequestOptions(qry, q.Size, q.From, q.Explain)
if searchRequest == nil {
responseLogger(r, http.StatusBadRequest, fmt.Errorf("Can't build new search request options %+v, %s", qry, err))
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(fmt.Sprintf("%s", err)))
return
}
searchRequest.Highlight = bleve.NewHighlight()
searchRequest.Highlight.AddField("title")
searchRequest.Highlight.AddField("content_description")
searchRequest.Highlight.AddField("subjects")
searchRequest.Highlight.AddField("subjects_function")
searchRequest.Highlight.AddField("subjects_topical")
searchRequest.Highlight.AddField("extents")
subjectFacet := bleve.NewFacetRequest("subjects", 3)
searchRequest.AddFacet("subjects", subjectFacet)
subjectTopicalFacet := bleve.NewFacetRequest("subjects_topical", 3)
searchRequest.AddFacet("subjects_topical", subjectTopicalFacet)
subjectFunctionFacet := bleve.NewFacetRequest("subjects_function", 3)
searchRequest.AddFacet("subjects_function", subjectFunctionFacet)
// Return all fields
searchRequest.Fields = []string{
"title",
"identifier",
"content_description",
"content_condition",
"resource_type",
"access_restrictions",
"access_restrictions_note",
"use_restrictins",
"use_restrictons_note",
"dates",
"date_expression",
"extents",
"subjects",
"subjects_function",
"subjects_topical",
"linked_agents_creators",
"linked_agents_subjects",
"link_agents_sources",
"digital_objects.title",
"digital_objects.file_uris",
"related_resources",
"deaccessions",
"accession_date",
"created",
}
searchResults, err := index.Search(searchRequest)
if err != nil {
responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Bleve results error %v, %s", qry, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(fmt.Sprintf("%s", err)))
return
}
// q (ciat.SearchQuery) performs double duty as both the structure for query submission as well
// as carring the results to support paging and other types of navigation through
// the query set. Results are a query with the bleve.SearchReults merged
q.AttachSearchResults(searchResults)
pageHTML = "results-search.html"
pageInclude = "results-search.include"
// Load my templates and setup to execute them
tmpl, err := tmplfn.Assemble(tmplFuncs, path.Join(templatesDir, pageHTML), path.Join(templatesDir, pageInclude))
if err != nil {
responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Template Errors: %s, %s, %s\n", pageHTML, pageInclude, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(fmt.Sprintf("Template errors: %s", err)))
return
}
// Render the page
w.Header().Set("Content-Type", "text/html")
var buf bytes.Buffer
err = tmpl.Execute(&buf, q)
//err = tmpl.Execute(w, q)
if err != nil {
responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Can't render %s, %s/%s, %s", templatesDir, pageHTML, pageInclude, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Template error"))
return
}
//NOTE: This bit of ugliness is here because I need to allow <mark> elements and ellipis in the results fragments
w.Write(bytes.Replace(bytes.Replace(bytes.Replace(buf.Bytes(), []byte("<mark>"), []byte("<mark>"), -1), []byte("</mark>"), []byte("</mark>"), -1), []byte(`…`), []byte(`…`), -1))
}
func TestBeerSearchAll(t *testing.T) {
defer os.RemoveAll("beer-search-test.bleve")
mapping, err := buildIndexMapping()
if err != nil {
t.Fatal(err)
}
index, err := bleve.New("beer-search-test.bleve", mapping)
if err != nil {
t.Fatal(err)
}
defer index.Close()
for jf := range walkDirectory("data/", t) {
docId := jf.filename[0:strings.LastIndex(jf.filename, ".")]
err = index.Index(docId, jf.contents)
if err != nil {
t.Error(err)
}
}
expectedCount := uint64(7303)
actualCount := index.DocCount()
if actualCount != expectedCount {
t.Errorf("expected %d documents, got %d", expectedCount, actualCount)
}
// run a term search
termQuery := bleve.NewTermQuery("shock").SetField("name")
termSearchRequest := bleve.NewSearchRequest(termQuery)
termSearchResult, err := index.Search(termSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount := uint64(1)
if termSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total)
} else {
expectedResultId := "anheuser_busch-shock_top"
if termSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID)
}
}
// run a match phrase search
matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food")
matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery)
matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if matchPhraseSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total)
} else {
expectedResultId := "great_divide_brewing-wild_raspberry_ale"
if matchPhraseSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID)
}
}
// run a syntax query
syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley")
syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery)
syntaxSearchResult, err := index.Search(syntaxSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if syntaxSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total)
} else {
expectedResultId := "iron_city_brewing_co-ic_light"
if syntaxSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID)
}
}
// run a numeric range search
queryMin := 50.0
numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv")
numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery)
numericSearchResult, err := index.Search(numericSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if numericSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total)
} else {
expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale"
if numericSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID)
}
}
// run a date range search
queryStartDate := "2011-10-04"
dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated")
dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery)
dateSearchResult, err := index.Search(dateSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(2)
if dateSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total)
} else {
expectedResultId := "brasserie_du_bouffay-ambr"
if dateSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID)
}
}
// run a prefix search
prefixQuery := bleve.NewPrefixQuery("adir").SetField("name")
prefixSearchRequest := bleve.NewSearchRequest(prefixQuery)
prefixSearchResult, err := index.Search(prefixSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if prefixSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total)
} else {
expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager"
if prefixSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID)
}
}
}
func Example_beleveSearch() {
// open a new index
indexMapping := bleve.NewIndexMapping()
err := indexMapping.AddCustomTokenizer("jieba",
map[string]interface{}{
"file": "../dict.txt",
"type": "jieba",
})
if err != nil {
log.Fatal(err)
}
// create a custom analyzer
err = indexMapping.AddCustomAnalyzer("jieba",
map[string]interface{}{
"type": "custom",
"tokenizer": "jieba",
"token_filters": []string{
"possessive_en",
"to_lower",
"stop_en",
},
})
if err != nil {
log.Fatal(err)
}
indexMapping.DefaultAnalyzer = "jieba"
cacheDir := "jieba.beleve"
os.RemoveAll(cacheDir)
index, err := bleve.New(cacheDir, indexMapping)
if err != nil {
log.Fatal(err)
}
docs := []struct {
Title string
Name string
}{
{
Title: "Doc 1",
Name: "This is the first document we’ve added",
},
{
Title: "Doc 2",
Name: "The second one 你 中文测试中文 is even more interesting! 吃水果",
},
{
Title: "Doc 3",
Name: "买水果然后来世博园。",
},
{
Title: "Doc 4",
Name: "工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作",
},
{
Title: "Doc 5",
Name: "咱俩交换一下吧。",
},
}
// index docs
for _, doc := range docs {
index.Index(doc.Title, doc)
}
// search for some text
for _, keyword := range []string{"水果世博园", "你", "first", "中文", "交换机", "交换"} {
query := bleve.NewQueryStringQuery(keyword)
search := bleve.NewSearchRequest(query)
search.Highlight = bleve.NewHighlight()
searchResults, err := index.Search(search)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Result of \"%s\": %d matches:\n", keyword, searchResults.Total)
for i, hit := range searchResults.Hits {
rv := fmt.Sprintf("%d. %s, (%f)\n", i+searchResults.Request.From+1, hit.ID, hit.Score)
for fragmentField, fragments := range hit.Fragments {
rv += fmt.Sprintf("%s: ", fragmentField)
for _, fragment := range fragments {
rv += fmt.Sprintf("%s", fragment)
}
}
fmt.Printf("%s\n", rv)
}
}
// Output:
// Result of "水果世博园": 2 matches:
// 1. Doc 3, (1.099550)
// Name: 买<span class="highlight">水果</span>然后来<span class="highlight">世博</span>园。
// 2. Doc 2, (0.031941)
// Name: The second one 你 中文测试中文 is even more interesting! 吃<span class="highlight">水果</span>
// Result of "你": 1 matches:
// 1. Doc 2, (0.391161)
// Name: The second one <span class="highlight">你</span> 中文测试中文 is even more interesting! 吃水果
// Result of "first": 1 matches:
// 1. Doc 1, (0.512150)
// Name: This is the <span class="highlight">first</span> document we’ve added
// Result of "中文": 1 matches:
// 1. Doc 2, (0.553186)
// Name: The second one 你 <span class="highlight">中文</span>测试<span class="highlight">中文</span> is even more interesting! 吃水果
// Result of "交换机": 2 matches:
// 1. Doc 4, (0.608495)
// Name: 工信处女干事每月经过下属科室都要亲口交代24口<span class="highlight">交换机</span>等技术性器件的安装工作
// 2. Doc 5, (0.086700)
// Name: 咱俩<span class="highlight">交换</span>一下吧。
// Result of "交换": 2 matches:
// 1. Doc 5, (0.534158)
// Name: 咱俩<span class="highlight">交换</span>一下吧。
// 2. Doc 4, (0.296297)
// Name: 工信处女干事每月经过下属科室都要亲口交代24口<span class="highlight">交换</span>机等技术性器件的安装工作
}