func (self *Wiki) Search(queryString string) ([]*SearchResult, error) {
query := bleve.NewMatchQuery(queryString)
search := bleve.NewSearchRequest(query)
search.Highlight = bleve.NewHighlight()
searchResult, err := self.Index.Search(search)
if err != nil {
return nil, err
}
results := []*SearchResult{}
for _, result := range searchResult.Hits {
section, article := self.Find(result.ID)
if article == nil && section == nil {
return nil, fmt.Errorf("%v section or article not found", result.ID)
}
text := ""
for _, values := range result.Fragments {
for _, value := range values {
text += value
}
}
name := ""
if section != nil {
name = section.Name
}
if article != nil {
name = article.Name
}
results = append(results, &SearchResult{Path: result.ID, Name: name, Text: text})
}
return results, nil
}
// Search method lookup for records using a query
func (i *bleveIndexer) Search(q string) (records []indexer.Record) {
query := bleve.NewQueryStringQuery(q)
request := bleve.NewSearchRequest(query)
request.Highlight = bleve.NewHighlight()
result, err := i.bleve.Search(request)
if err != nil { // an empty query would cause this
return
}
for _, match := range result.Hits {
rec := i.Record(match.ID)
loaded := rec.Load()
if !loaded {
continue
}
if len(match.Fragments["Body"]) > 0 {
rec.SetBody([]byte(match.Fragments["Body"][0]))
}
records = append(records, rec)
}
return
}
func BlevePIndexQuerySamples() []cbgt.Documentation {
return []cbgt.Documentation{
cbgt.Documentation{
Text: "A simple bleve query POST body:",
JSON: &struct {
*cbgt.QueryCtlParams
*bleve.SearchRequest
}{
nil,
&bleve.SearchRequest{
From: 0,
Size: 10,
Query: bleve.NewQueryStringQuery("a sample query"),
},
},
},
cbgt.Documentation{
Text: `An example POST body using from/size for results paging,
using ctl for a timeout and for "at_plus" consistency level.
On consistency, the index must have incorporated at least mutation
sequence-number 123 for partition (vbucket) 0 and mutation
sequence-number 234 for partition (vbucket) 1 (where vbucket 1
should have a vbucketUUID of a0b1c2):`,
JSON: &struct {
*cbgt.QueryCtlParams
*bleve.SearchRequest
}{
&cbgt.QueryCtlParams{
Ctl: cbgt.QueryCtl{
Timeout: cbgt.QUERY_CTL_DEFAULT_TIMEOUT_MS,
Consistency: &cbgt.ConsistencyParams{
Level: "at_plus",
Vectors: map[string]cbgt.ConsistencyVector{
"customerIndex": cbgt.ConsistencyVector{
"0": 123,
"1/a0b1c2": 234,
},
},
},
},
},
&bleve.SearchRequest{
From: 20,
Size: 10,
Fields: []string{"*"},
Query: bleve.NewQueryStringQuery("alice smith"),
Highlight: bleve.NewHighlight(),
Explain: true,
},
},
},
}
}
func Example() {
INDEX_DIR := "gojieba.bleve"
messages := []struct {
Id string
Body string
}{
{
Id: "1",
Body: "你好",
},
{
Id: "2",
Body: "交代",
},
{
Id: "3",
Body: "长江大桥",
},
}
indexMapping := bleve.NewIndexMapping()
os.RemoveAll(INDEX_DIR)
// clean index when example finished
defer os.RemoveAll(INDEX_DIR)
err := indexMapping.AddCustomTokenizer("gojieba",
map[string]interface{}{
"dictpath": gojieba.DICT_PATH,
"hmmpath": gojieba.HMM_PATH,
"userdictpath": gojieba.USER_DICT_PATH,
"idf": gojieba.IDF_PATH,
"stop_words": gojieba.STOP_WORDS_PATH,
"type": "gojieba",
},
)
if err != nil {
panic(err)
}
err = indexMapping.AddCustomAnalyzer("gojieba",
map[string]interface{}{
"type": "gojieba",
"tokenizer": "gojieba",
},
)
if err != nil {
panic(err)
}
indexMapping.DefaultAnalyzer = "gojieba"
index, err := bleve.New(INDEX_DIR, indexMapping)
if err != nil {
panic(err)
}
for _, msg := range messages {
if err := index.Index(msg.Id, msg); err != nil {
panic(err)
}
}
querys := []string{
"你好世界",
"亲口交代",
"长江",
}
for _, q := range querys {
req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(q))
req.Highlight = bleve.NewHighlight()
res, err := index.Search(req)
if err != nil {
panic(err)
}
fmt.Println(prettify(res))
}
// Output:
// [{"id":"1","score":0.27650412875470115}]
// [{"id":"2","score":0.27650412875470115}]
// [{"id":"3","score":0.7027325540540822}]
}
func resultsHandler(w http.ResponseWriter, r *http.Request) {
var (
pageHTML = "results-search.html"
pageInclude = "results-search.include"
)
urlQuery := r.URL.Query()
err := r.ParseForm()
if err != nil {
responseLogger(r, http.StatusBadRequest, err)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(fmt.Sprintf("error in POST: %s", err)))
return
}
// Collect the submissions fields.
submission := make(map[string]interface{})
// Basic Search results
if r.Method == "GET" {
for k, v := range urlQuery {
if k == "all_ids" {
if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil {
submission[k] = b
}
} else if k == "from" || k == "size" || k == "total" {
if i, err := strconv.Atoi(strings.Join(v, "")); err == nil {
submission[k] = i
}
} else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" {
submission[k] = strings.Join(v, "")
}
}
}
// Advanced Search results
if r.Method == "POST" {
for k, v := range r.Form {
if k == "all_ids" {
if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil {
submission[k] = b
}
} else if k == "from" || k == "size" || k == "total" {
if i, err := strconv.Atoi(strings.Join(v, "")); err == nil {
submission[k] = i
}
} else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" {
submission[k] = strings.Join(v, "")
}
}
}
q, err := mapToSearchQuery(submission)
if err != nil {
responseLogger(r, http.StatusBadRequest, err)
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(fmt.Sprintf("%s", err)))
return
}
//
// Note: Add logic to handle basic and advanced search...
//
// q NewQueryStringQuery
// q_required NewQueryStringQuery with a + prefix for each strings.Fields(q_required) value
// q_exact NewMatchPhraseQuery
// q_excluded NewQueryStringQuery with a - prefix for each strings.Feilds(q_excluded) value
//
var conQry []bleve.Query
if q.Q != "" {
conQry = append(conQry, bleve.NewQueryStringQuery(q.Q))
}
if q.QExact != "" {
conQry = append(conQry, bleve.NewMatchPhraseQuery(q.QExact))
}
var terms []string
for _, s := range strings.Fields(q.QRequired) {
terms = append(terms, fmt.Sprintf("+%s", strings.TrimSpace(s)))
}
for _, s := range strings.Fields(q.QExcluded) {
terms = append(terms, fmt.Sprintf("-%s", strings.TrimSpace(s)))
}
if len(terms) > 0 {
qString := strings.Join(terms, " ")
conQry = append(conQry, bleve.NewQueryStringQuery(qString))
}
qry := bleve.NewConjunctionQuery(conQry)
if q.Size == 0 {
q.Size = 10
}
searchRequest := bleve.NewSearchRequestOptions(qry, q.Size, q.From, q.Explain)
if searchRequest == nil {
responseLogger(r, http.StatusBadRequest, fmt.Errorf("Can't build new search request options %+v, %s", qry, err))
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(fmt.Sprintf("%s", err)))
return
}
searchRequest.Highlight = bleve.NewHighlight()
searchRequest.Highlight.AddField("title")
searchRequest.Highlight.AddField("content_description")
searchRequest.Highlight.AddField("subjects")
searchRequest.Highlight.AddField("subjects_function")
searchRequest.Highlight.AddField("subjects_topical")
searchRequest.Highlight.AddField("extents")
subjectFacet := bleve.NewFacetRequest("subjects", 3)
searchRequest.AddFacet("subjects", subjectFacet)
subjectTopicalFacet := bleve.NewFacetRequest("subjects_topical", 3)
searchRequest.AddFacet("subjects_topical", subjectTopicalFacet)
subjectFunctionFacet := bleve.NewFacetRequest("subjects_function", 3)
searchRequest.AddFacet("subjects_function", subjectFunctionFacet)
// Return all fields
searchRequest.Fields = []string{
"title",
"identifier",
"content_description",
"content_condition",
"resource_type",
"access_restrictions",
"access_restrictions_note",
"use_restrictins",
"use_restrictons_note",
"dates",
"date_expression",
"extents",
"subjects",
"subjects_function",
"subjects_topical",
"linked_agents_creators",
"linked_agents_subjects",
"link_agents_sources",
"digital_objects.title",
"digital_objects.file_uris",
"related_resources",
"deaccessions",
"accession_date",
"created",
}
searchResults, err := index.Search(searchRequest)
if err != nil {
responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Bleve results error %v, %s", qry, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(fmt.Sprintf("%s", err)))
return
}
// q (ciat.SearchQuery) performs double duty as both the structure for query submission as well
// as carring the results to support paging and other types of navigation through
// the query set. Results are a query with the bleve.SearchReults merged
q.AttachSearchResults(searchResults)
pageHTML = "results-search.html"
pageInclude = "results-search.include"
// Load my templates and setup to execute them
tmpl, err := tmplfn.Assemble(tmplFuncs, path.Join(templatesDir, pageHTML), path.Join(templatesDir, pageInclude))
if err != nil {
responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Template Errors: %s, %s, %s\n", pageHTML, pageInclude, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(fmt.Sprintf("Template errors: %s", err)))
return
}
// Render the page
w.Header().Set("Content-Type", "text/html")
var buf bytes.Buffer
err = tmpl.Execute(&buf, q)
//err = tmpl.Execute(w, q)
if err != nil {
responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Can't render %s, %s/%s, %s", templatesDir, pageHTML, pageInclude, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Template error"))
return
}
//NOTE: This bit of ugliness is here because I need to allow <mark> elements and ellipis in the results fragments
w.Write(bytes.Replace(bytes.Replace(bytes.Replace(buf.Bytes(), []byte("<mark>"), []byte("<mark>"), -1), []byte("</mark>"), []byte("</mark>"), -1), []byte(`…`), []byte(`…`), -1))
}
func Example_beleveSearch() {
// open a new index
indexMapping := bleve.NewIndexMapping()
err := indexMapping.AddCustomTokenizer("jieba",
map[string]interface{}{
"file": "../dict.txt",
"type": "jieba",
})
if err != nil {
log.Fatal(err)
}
// create a custom analyzer
err = indexMapping.AddCustomAnalyzer("jieba",
map[string]interface{}{
"type": "custom",
"tokenizer": "jieba",
"token_filters": []string{
"possessive_en",
"to_lower",
"stop_en",
},
})
if err != nil {
log.Fatal(err)
}
indexMapping.DefaultAnalyzer = "jieba"
cacheDir := "jieba.beleve"
os.RemoveAll(cacheDir)
index, err := bleve.New(cacheDir, indexMapping)
if err != nil {
log.Fatal(err)
}
docs := []struct {
Title string
Name string
}{
{
Title: "Doc 1",
Name: "This is the first document we’ve added",
},
{
Title: "Doc 2",
Name: "The second one 你 中文测试中文 is even more interesting! 吃水果",
},
{
Title: "Doc 3",
Name: "买水果然后来世博园。",
},
{
Title: "Doc 4",
Name: "工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作",
},
{
Title: "Doc 5",
Name: "咱俩交换一下吧。",
},
}
// index docs
for _, doc := range docs {
index.Index(doc.Title, doc)
}
// search for some text
for _, keyword := range []string{"水果世博园", "你", "first", "中文", "交换机", "交换"} {
query := bleve.NewQueryStringQuery(keyword)
search := bleve.NewSearchRequest(query)
search.Highlight = bleve.NewHighlight()
searchResults, err := index.Search(search)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Result of \"%s\": %d matches:\n", keyword, searchResults.Total)
for i, hit := range searchResults.Hits {
rv := fmt.Sprintf("%d. %s, (%f)\n", i+searchResults.Request.From+1, hit.ID, hit.Score)
for fragmentField, fragments := range hit.Fragments {
rv += fmt.Sprintf("%s: ", fragmentField)
for _, fragment := range fragments {
rv += fmt.Sprintf("%s", fragment)
}
}
fmt.Printf("%s\n", rv)
}
}
// Output:
// Result of "水果世博园": 2 matches:
// 1. Doc 3, (1.099550)
// Name: 买<span class="highlight">水果</span>然后来<span class="highlight">世博</span>园。
// 2. Doc 2, (0.031941)
// Name: The second one 你 中文测试中文 is even more interesting! 吃<span class="highlight">水果</span>
// Result of "你": 1 matches:
// 1. Doc 2, (0.391161)
// Name: The second one <span class="highlight">你</span> 中文测试中文 is even more interesting! 吃水果
// Result of "first": 1 matches:
// 1. Doc 1, (0.512150)
// Name: This is the <span class="highlight">first</span> document we’ve added
// Result of "中文": 1 matches:
// 1. Doc 2, (0.553186)
// Name: The second one 你 <span class="highlight">中文</span>测试<span class="highlight">中文</span> is even more interesting! 吃水果
// Result of "交换机": 2 matches:
// 1. Doc 4, (0.608495)
// Name: 工信处女干事每月经过下属科室都要亲口交代24口<span class="highlight">交换机</span>等技术性器件的安装工作
// 2. Doc 5, (0.086700)
// Name: 咱俩<span class="highlight">交换</span>一下吧。
// Result of "交换": 2 matches:
// 1. Doc 5, (0.534158)
// Name: 咱俩<span class="highlight">交换</span>一下吧。
// 2. Doc 4, (0.296297)
// Name: 工信处女干事每月经过下属科室都要亲口交代24口<span class="highlight">交换</span>机等技术性器件的安装工作
}
