func (b *Index) Search(queryStr string, offset, limit int, highlights bool) (*bleve.SearchResult, error) { query := bleve.NewQueryStringQuery(queryStr) req := bleve.NewSearchRequestOptions(query, limit, offset, false) if highlights { req.Highlight = bleve.NewHighlightWithStyle("ansi") } return b.index.Search(req) }
func main() { flag.Parse() if *cpuprofile != "" { f, err := os.Create(*cpuprofile) if err != nil { log.Fatal(err) } err = pprof.StartCPUProfile(f) if err != nil { log.Fatal(err) } defer pprof.StopCPUProfile() } if *indexPath == "" { log.Fatal("Specify index to query") } if flag.NArg() < 1 { log.Fatal("Specify search query") } // open index index, err := bleve.Open(*indexPath) if err != nil { log.Fatal(err) } defer func() { cerr := index.Close() if cerr != nil { log.Fatalf("error closing index: %v", err) } }() for i := 0; i < *repeat; i++ { // build a search with the provided parameters queryString := strings.Join(flag.Args(), " ") query := bleve.NewQueryStringQuery(queryString) searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain) // enable highlights if requested if *includeHighlights { searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi") } // execute the search searchResult, err := index.Search(searchRequest) if err != nil { log.Fatalf("search error: %v", err) } fmt.Println(searchResult) } }
func (indexes *BleveIndexes) Query(namespace string, query interface{}, offset int64, size int64) (int64, []string, error) { var total int64 = 0 var uris []string indexes.RLock() defer indexes.RUnlock() node := indexes.nodes.Node(namespace) if node == nil { return total, uris, fmt.Errorf("none node for namespace: %s", namespace) } var idx bleve.Index if node.GetBind() == nil { index, err := indexes.index(namespace, false) if err != nil { return total, uris, err } if index == nil { return total, uris, nil } node.SetBind(index) idx = index } else { idx = node.GetBind().(bleve.Index) } q, ok := query.(bleve.Query) if !ok { return total, uris, fmt.Errorf("query type convert failed") } request := bleve.NewSearchRequestOptions(q, IndexSize, IndexOffset, false) response, err := idx.Search(request) if err != nil { return total, uris, err } total = int64(response.Total) for _, doc := range response.Hits { uris = append(uris, doc.ID) } return total, uris, nil }
func main() { flag.Parse() if *indexPath == "" { log.Fatal("specify index to query") } if flag.NArg() < 1 { log.Fatal("Specify search query") } // open index index, err := bleve.Open(*indexPath) if err != nil { log.Fatal(err) } defer index.Close() // build a search with the provided parameters queryString := strings.Join(flag.Args(), " ") query := bleve.NewQueryStringQuery(queryString) searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain) // enable highlights if requested if *includeHighlights { searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi") } // execute the search searchResult, err := index.Search(searchRequest) if err != nil { log.Fatalf("search error: %v", err) } fmt.Println(searchResult) }
// Periodically removes old entries from index func (b *Bleve) periodicCleanup(die chan bool) { var ( bvBatchDelete *bv.Batch bvRequest *bv.SearchRequest bvResults *bv.SearchResult bvCleaningNow bool bvNbCleaned int err error till string limit int = 20 offset int = 0 sleepDuration time.Duration = 3 * time.Second ) defer func() { err = b.index.Close() if err != nil { logger.Instance(). WithError(err). Warning("Unable to close Bleve index") } }() for { select { case <-die: return default: } offset = 0 bvNbCleaned = 0 bvCleaningNow = true till = time.Now().Add(b.intervalCleanup).Format(time.RFC3339) bvQuery := bv.NewDateRangeQuery(nil, &till) bvQuery.FieldVal = "timestamp" for bvCleaningNow != false { bvRequest = bv.NewSearchRequestOptions(bvQuery, limit, offset, false) bvResults, err = b.index.Search(bvRequest) if err != nil { logger.Instance(). WithError(err). Warning("Unable to get obsolete messages from index") bvCleaningNow = false continue } if bvResults.Hits.Len() == 0 { bvCleaningNow = false continue } // List of documents to be deleted bvBatchDelete = b.index.NewBatch() for _, hit := range bvResults.Hits { bvBatchDelete.Delete(hit.ID) } // Batch delete them err = b.index.Batch(bvBatchDelete) if err != nil { logger.Instance(). WithError(err). Warning("Unable to delete obsolete messages from index") bvCleaningNow = false continue } else { bvNbCleaned += bvBatchDelete.Size() offset += limit } } if bvNbCleaned > 0 { logger.Instance(). WithField("nb_messages", bvNbCleaned). Infof("Obsolete messages were deleted from index") } time.Sleep(sleepDuration) } }
func resultsHandler(w http.ResponseWriter, r *http.Request) { var ( pageHTML = "results-search.html" pageInclude = "results-search.include" ) urlQuery := r.URL.Query() err := r.ParseForm() if err != nil { responseLogger(r, http.StatusBadRequest, err) w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf("error in POST: %s", err))) return } // Collect the submissions fields. submission := make(map[string]interface{}) // Basic Search results if r.Method == "GET" { for k, v := range urlQuery { if k == "all_ids" { if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil { submission[k] = b } } else if k == "from" || k == "size" || k == "total" { if i, err := strconv.Atoi(strings.Join(v, "")); err == nil { submission[k] = i } } else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" { submission[k] = strings.Join(v, "") } } } // Advanced Search results if r.Method == "POST" { for k, v := range r.Form { if k == "all_ids" { if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil { submission[k] = b } } else if k == "from" || k == "size" || k == "total" { if i, err := strconv.Atoi(strings.Join(v, "")); err == nil { submission[k] = i } } else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" { submission[k] = strings.Join(v, "") } } } q, err := mapToSearchQuery(submission) if err != nil { responseLogger(r, http.StatusBadRequest, err) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf("%s", err))) return } // // Note: Add logic to handle basic and advanced search... // // q NewQueryStringQuery // q_required NewQueryStringQuery with a + prefix for each strings.Fields(q_required) value // q_exact NewMatchPhraseQuery // q_excluded NewQueryStringQuery with a - prefix for each strings.Feilds(q_excluded) value // var conQry []bleve.Query if q.Q != "" { conQry = append(conQry, bleve.NewQueryStringQuery(q.Q)) } if q.QExact != "" { conQry = append(conQry, bleve.NewMatchPhraseQuery(q.QExact)) } var terms []string for _, s := range strings.Fields(q.QRequired) { terms = append(terms, fmt.Sprintf("+%s", strings.TrimSpace(s))) } for _, s := range strings.Fields(q.QExcluded) { terms = append(terms, fmt.Sprintf("-%s", strings.TrimSpace(s))) } if len(terms) > 0 { qString := strings.Join(terms, " ") conQry = append(conQry, bleve.NewQueryStringQuery(qString)) } qry := bleve.NewConjunctionQuery(conQry) if q.Size == 0 { q.Size = 10 } searchRequest := bleve.NewSearchRequestOptions(qry, q.Size, q.From, q.Explain) if searchRequest == nil { responseLogger(r, http.StatusBadRequest, fmt.Errorf("Can't build new search request options %+v, %s", qry, err)) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf("%s", err))) return } searchRequest.Highlight = bleve.NewHighlight() searchRequest.Highlight.AddField("title") searchRequest.Highlight.AddField("content_description") searchRequest.Highlight.AddField("subjects") searchRequest.Highlight.AddField("subjects_function") searchRequest.Highlight.AddField("subjects_topical") searchRequest.Highlight.AddField("extents") subjectFacet := bleve.NewFacetRequest("subjects", 3) searchRequest.AddFacet("subjects", subjectFacet) subjectTopicalFacet := bleve.NewFacetRequest("subjects_topical", 3) searchRequest.AddFacet("subjects_topical", subjectTopicalFacet) subjectFunctionFacet := bleve.NewFacetRequest("subjects_function", 3) searchRequest.AddFacet("subjects_function", subjectFunctionFacet) // Return all fields searchRequest.Fields = []string{ "title", "identifier", "content_description", "content_condition", "resource_type", "access_restrictions", "access_restrictions_note", "use_restrictins", "use_restrictons_note", "dates", "date_expression", "extents", "subjects", "subjects_function", "subjects_topical", "linked_agents_creators", "linked_agents_subjects", "link_agents_sources", "digital_objects.title", "digital_objects.file_uris", "related_resources", "deaccessions", "accession_date", "created", } searchResults, err := index.Search(searchRequest) if err != nil { responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Bleve results error %v, %s", qry, err)) w.WriteHeader(http.StatusInternalServerError) w.Write([]byte(fmt.Sprintf("%s", err))) return } // q (ciat.SearchQuery) performs double duty as both the structure for query submission as well // as carring the results to support paging and other types of navigation through // the query set. Results are a query with the bleve.SearchReults merged q.AttachSearchResults(searchResults) pageHTML = "results-search.html" pageInclude = "results-search.include" // Load my templates and setup to execute them tmpl, err := tmplfn.Assemble(tmplFuncs, path.Join(templatesDir, pageHTML), path.Join(templatesDir, pageInclude)) if err != nil { responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Template Errors: %s, %s, %s\n", pageHTML, pageInclude, err)) w.WriteHeader(http.StatusInternalServerError) w.Write([]byte(fmt.Sprintf("Template errors: %s", err))) return } // Render the page w.Header().Set("Content-Type", "text/html") var buf bytes.Buffer err = tmpl.Execute(&buf, q) //err = tmpl.Execute(w, q) if err != nil { responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Can't render %s, %s/%s, %s", templatesDir, pageHTML, pageInclude, err)) w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("Template error")) return } //NOTE: This bit of ugliness is here because I need to allow <mark> elements and ellipis in the results fragments w.Write(bytes.Replace(bytes.Replace(bytes.Replace(buf.Bytes(), []byte("<mark>"), []byte("<mark>"), -1), []byte("</mark>"), []byte("</mark>"), -1), []byte(`…`), []byte(`…`), -1)) }
var explain, highlight, fields bool var qtype, qfield string // queryCmd represents the query command var queryCmd = &cobra.Command{ Use: "query [index path] [query]", Short: "queries the index", Long: `The query command will execute a query against the index.`, RunE: func(cmd *cobra.Command, args []string) error { if len(args) < 2 { return fmt.Errorf("must specify query") } query := buildQuery(args) for i := 0; i < repeat; i++ { req := bleve.NewSearchRequestOptions(query, limit, skip, explain) if highlight { req.Highlight = bleve.NewHighlightWithStyle("ansi") } if fields { req.Fields = []string{"*"} } res, err := idx.Search(req) if err != nil { return fmt.Errorf("error running query: %v", err) } fmt.Println(res) } return nil }, }
func main() { flag.Parse() if *cpuprofile != "" { f, err := os.Create(*cpuprofile) if err != nil { log.Fatal(err) } err = pprof.StartCPUProfile(f) if err != nil { log.Fatal(err) } defer pprof.StopCPUProfile() } if *indexPath == "" { log.Fatal("Specify index to query") } if flag.NArg() < 1 { log.Fatal("Specify search query") } // open index index, err := bleve.Open(*indexPath) if err != nil { log.Fatal(err) } defer func() { cerr := index.Close() if cerr != nil { log.Fatalf("error closing index: %v", err) } }() for i := 0; i < *repeat; i++ { var query bleve.Query switch *qtype { case "prefix": pquery := bleve.NewPrefixQuery(strings.Join(flag.Args(), " ")) if *qfield != "" { pquery.SetField(*qfield) } query = pquery case "term": pquery := bleve.NewTermQuery(strings.Join(flag.Args(), " ")) if *qfield != "" { pquery.SetField(*qfield) } query = pquery default: // build a search with the provided parameters queryString := strings.Join(flag.Args(), " ") query = bleve.NewQueryStringQuery(queryString) } searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain) // enable highlights if requested if *includeHighlights { searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi") } if *includeStoredFields { searchRequest.Fields = []string{"*"} } // execute the search searchResult, err := index.Search(searchRequest) if err != nil { log.Fatalf("search error: %v", err) } fmt.Println(searchResult) } }