func (b *Index) Search(queryStr string, offset, limit int, highlights bool) (*bleve.SearchResult, error) {
query := bleve.NewQueryStringQuery(queryStr)
req := bleve.NewSearchRequestOptions(query, limit, offset, false)
if highlights {
req.Highlight = bleve.NewHighlightWithStyle("ansi")
}
return b.index.Search(req)
}
func main() {
flag.Parse()
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
err = pprof.StartCPUProfile(f)
if err != nil {
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
if *indexPath == "" {
log.Fatal("Specify index to query")
}
if flag.NArg() < 1 {
log.Fatal("Specify search query")
}
// open index
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}
defer func() {
cerr := index.Close()
if cerr != nil {
log.Fatalf("error closing index: %v", err)
}
}()
for i := 0; i < *repeat; i++ {
// build a search with the provided parameters
queryString := strings.Join(flag.Args(), " ")
query := bleve.NewQueryStringQuery(queryString)
searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain)
// enable highlights if requested
if *includeHighlights {
searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi")
}
// execute the search
searchResult, err := index.Search(searchRequest)
if err != nil {
log.Fatalf("search error: %v", err)
}
fmt.Println(searchResult)
}
}
func (indexes *BleveIndexes) Query(namespace string, query interface{}, offset int64, size int64) (int64, []string, error) {
var total int64 = 0
var uris []string
indexes.RLock()
defer indexes.RUnlock()
node := indexes.nodes.Node(namespace)
if node == nil {
return total, uris, fmt.Errorf("none node for namespace: %s", namespace)
}
var idx bleve.Index
if node.GetBind() == nil {
index, err := indexes.index(namespace, false)
if err != nil {
return total, uris, err
}
if index == nil {
return total, uris, nil
}
node.SetBind(index)
idx = index
} else {
idx = node.GetBind().(bleve.Index)
}
q, ok := query.(bleve.Query)
if !ok {
return total, uris, fmt.Errorf("query type convert failed")
}
request := bleve.NewSearchRequestOptions(q, IndexSize, IndexOffset, false)
response, err := idx.Search(request)
if err != nil {
return total, uris, err
}
total = int64(response.Total)
for _, doc := range response.Hits {
uris = append(uris, doc.ID)
}
return total, uris, nil
}
func main() {
flag.Parse()
if *indexPath == "" {
log.Fatal("specify index to query")
}
if flag.NArg() < 1 {
log.Fatal("Specify search query")
}
// open index
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}
defer index.Close()
// build a search with the provided parameters
queryString := strings.Join(flag.Args(), " ")
query := bleve.NewQueryStringQuery(queryString)
searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain)
// enable highlights if requested
if *includeHighlights {
searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi")
}
// execute the search
searchResult, err := index.Search(searchRequest)
if err != nil {
log.Fatalf("search error: %v", err)
}
fmt.Println(searchResult)
}
// Periodically removes old entries from index
func (b *Bleve) periodicCleanup(die chan bool) {
var (
bvBatchDelete *bv.Batch
bvRequest *bv.SearchRequest
bvResults *bv.SearchResult
bvCleaningNow bool
bvNbCleaned int
err error
till string
limit int = 20
offset int = 0
sleepDuration time.Duration = 3 * time.Second
)
defer func() {
err = b.index.Close()
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to close Bleve index")
}
}()
for {
select {
case <-die:
return
default:
}
offset = 0
bvNbCleaned = 0
bvCleaningNow = true
till = time.Now().Add(b.intervalCleanup).Format(time.RFC3339)
bvQuery := bv.NewDateRangeQuery(nil, &till)
bvQuery.FieldVal = "timestamp"
for bvCleaningNow != false {
bvRequest = bv.NewSearchRequestOptions(bvQuery, limit, offset, false)
bvResults, err = b.index.Search(bvRequest)
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to get obsolete messages from index")
bvCleaningNow = false
continue
}
if bvResults.Hits.Len() == 0 {
bvCleaningNow = false
continue
}
// List of documents to be deleted
bvBatchDelete = b.index.NewBatch()
for _, hit := range bvResults.Hits {
bvBatchDelete.Delete(hit.ID)
}
// Batch delete them
err = b.index.Batch(bvBatchDelete)
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to delete obsolete messages from index")
bvCleaningNow = false
continue
} else {
bvNbCleaned += bvBatchDelete.Size()
offset += limit
}
}
if bvNbCleaned > 0 {
logger.Instance().
WithField("nb_messages", bvNbCleaned).
Infof("Obsolete messages were deleted from index")
}
time.Sleep(sleepDuration)
}
}
func resultsHandler(w http.ResponseWriter, r *http.Request) {
var (
pageHTML = "results-search.html"
pageInclude = "results-search.include"
)
urlQuery := r.URL.Query()
err := r.ParseForm()
if err != nil {
responseLogger(r, http.StatusBadRequest, err)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(fmt.Sprintf("error in POST: %s", err)))
return
}
// Collect the submissions fields.
submission := make(map[string]interface{})
// Basic Search results
if r.Method == "GET" {
for k, v := range urlQuery {
if k == "all_ids" {
if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil {
submission[k] = b
}
} else if k == "from" || k == "size" || k == "total" {
if i, err := strconv.Atoi(strings.Join(v, "")); err == nil {
submission[k] = i
}
} else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" {
submission[k] = strings.Join(v, "")
}
}
}
// Advanced Search results
if r.Method == "POST" {
for k, v := range r.Form {
if k == "all_ids" {
if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil {
submission[k] = b
}
} else if k == "from" || k == "size" || k == "total" {
if i, err := strconv.Atoi(strings.Join(v, "")); err == nil {
submission[k] = i
}
} else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" {
submission[k] = strings.Join(v, "")
}
}
}
q, err := mapToSearchQuery(submission)
if err != nil {
responseLogger(r, http.StatusBadRequest, err)
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(fmt.Sprintf("%s", err)))
return
}
//
// Note: Add logic to handle basic and advanced search...
//
// q NewQueryStringQuery
// q_required NewQueryStringQuery with a + prefix for each strings.Fields(q_required) value
// q_exact NewMatchPhraseQuery
// q_excluded NewQueryStringQuery with a - prefix for each strings.Feilds(q_excluded) value
//
var conQry []bleve.Query
if q.Q != "" {
conQry = append(conQry, bleve.NewQueryStringQuery(q.Q))
}
if q.QExact != "" {
conQry = append(conQry, bleve.NewMatchPhraseQuery(q.QExact))
}
var terms []string
for _, s := range strings.Fields(q.QRequired) {
terms = append(terms, fmt.Sprintf("+%s", strings.TrimSpace(s)))
}
for _, s := range strings.Fields(q.QExcluded) {
terms = append(terms, fmt.Sprintf("-%s", strings.TrimSpace(s)))
}
if len(terms) > 0 {
qString := strings.Join(terms, " ")
conQry = append(conQry, bleve.NewQueryStringQuery(qString))
}
qry := bleve.NewConjunctionQuery(conQry)
if q.Size == 0 {
q.Size = 10
}
searchRequest := bleve.NewSearchRequestOptions(qry, q.Size, q.From, q.Explain)
if searchRequest == nil {
responseLogger(r, http.StatusBadRequest, fmt.Errorf("Can't build new search request options %+v, %s", qry, err))
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(fmt.Sprintf("%s", err)))
return
}
searchRequest.Highlight = bleve.NewHighlight()
searchRequest.Highlight.AddField("title")
searchRequest.Highlight.AddField("content_description")
searchRequest.Highlight.AddField("subjects")
searchRequest.Highlight.AddField("subjects_function")
searchRequest.Highlight.AddField("subjects_topical")
searchRequest.Highlight.AddField("extents")
subjectFacet := bleve.NewFacetRequest("subjects", 3)
searchRequest.AddFacet("subjects", subjectFacet)
subjectTopicalFacet := bleve.NewFacetRequest("subjects_topical", 3)
searchRequest.AddFacet("subjects_topical", subjectTopicalFacet)
subjectFunctionFacet := bleve.NewFacetRequest("subjects_function", 3)
searchRequest.AddFacet("subjects_function", subjectFunctionFacet)
// Return all fields
searchRequest.Fields = []string{
"title",
"identifier",
"content_description",
"content_condition",
"resource_type",
"access_restrictions",
"access_restrictions_note",
"use_restrictins",
"use_restrictons_note",
"dates",
"date_expression",
"extents",
"subjects",
"subjects_function",
"subjects_topical",
"linked_agents_creators",
"linked_agents_subjects",
"link_agents_sources",
"digital_objects.title",
"digital_objects.file_uris",
"related_resources",
"deaccessions",
"accession_date",
"created",
}
searchResults, err := index.Search(searchRequest)
if err != nil {
responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Bleve results error %v, %s", qry, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(fmt.Sprintf("%s", err)))
return
}
// q (ciat.SearchQuery) performs double duty as both the structure for query submission as well
// as carring the results to support paging and other types of navigation through
// the query set. Results are a query with the bleve.SearchReults merged
q.AttachSearchResults(searchResults)
pageHTML = "results-search.html"
pageInclude = "results-search.include"
// Load my templates and setup to execute them
tmpl, err := tmplfn.Assemble(tmplFuncs, path.Join(templatesDir, pageHTML), path.Join(templatesDir, pageInclude))
if err != nil {
responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Template Errors: %s, %s, %s\n", pageHTML, pageInclude, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(fmt.Sprintf("Template errors: %s", err)))
return
}
// Render the page
w.Header().Set("Content-Type", "text/html")
var buf bytes.Buffer
err = tmpl.Execute(&buf, q)
//err = tmpl.Execute(w, q)
if err != nil {
responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Can't render %s, %s/%s, %s", templatesDir, pageHTML, pageInclude, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Template error"))
return
}
//NOTE: This bit of ugliness is here because I need to allow <mark> elements and ellipis in the results fragments
w.Write(bytes.Replace(bytes.Replace(bytes.Replace(buf.Bytes(), []byte("<mark>"), []byte("<mark>"), -1), []byte("</mark>"), []byte("</mark>"), -1), []byte(`…`), []byte(`…`), -1))
}
var explain, highlight, fields bool
var qtype, qfield string
// queryCmd represents the query command
var queryCmd = &cobra.Command{
Use: "query [index path] [query]",
Short: "queries the index",
Long: `The query command will execute a query against the index.`,
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) < 2 {
return fmt.Errorf("must specify query")
}
query := buildQuery(args)
for i := 0; i < repeat; i++ {
req := bleve.NewSearchRequestOptions(query, limit, skip, explain)
if highlight {
req.Highlight = bleve.NewHighlightWithStyle("ansi")
}
if fields {
req.Fields = []string{"*"}
}
res, err := idx.Search(req)
if err != nil {
return fmt.Errorf("error running query: %v", err)
}
fmt.Println(res)
}
return nil
},
}
func main() {
flag.Parse()
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
err = pprof.StartCPUProfile(f)
if err != nil {
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
if *indexPath == "" {
log.Fatal("Specify index to query")
}
if flag.NArg() < 1 {
log.Fatal("Specify search query")
}
// open index
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}
defer func() {
cerr := index.Close()
if cerr != nil {
log.Fatalf("error closing index: %v", err)
}
}()
for i := 0; i < *repeat; i++ {
var query bleve.Query
switch *qtype {
case "prefix":
pquery := bleve.NewPrefixQuery(strings.Join(flag.Args(), " "))
if *qfield != "" {
pquery.SetField(*qfield)
}
query = pquery
case "term":
pquery := bleve.NewTermQuery(strings.Join(flag.Args(), " "))
if *qfield != "" {
pquery.SetField(*qfield)
}
query = pquery
default:
// build a search with the provided parameters
queryString := strings.Join(flag.Args(), " ")
query = bleve.NewQueryStringQuery(queryString)
}
searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain)
// enable highlights if requested
if *includeHighlights {
searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi")
}
if *includeStoredFields {
searchRequest.Fields = []string{"*"}
}
// execute the search
searchResult, err := index.Search(searchRequest)
if err != nil {
log.Fatalf("search error: %v", err)
}
fmt.Println(searchResult)
}
}
