func (q *allMatchQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() } analyzerName := m.AnalyzerNameForPath(field) analyzer := m.AnalyzerNamed(analyzerName) tokens := analyzer.Analyze([]byte(q.Match)) if len(tokens) == 0 { noneQuery := bleve.NewMatchNoneQuery() return noneQuery.Searcher(i, m, explain) } tqs := make([]query.Query, len(tokens)) for i, token := range tokens { tq := bleve.NewTermQuery(string(token.Term)) tq.SetField(field) tq.SetBoost(q.BoostVal) tqs[i] = tq } allQuery := bleve.NewConjunctionQuery(tqs...) allQuery.SetBoost(q.BoostVal) return allQuery.Searcher(i, m, explain) }
func (pages *Pages) retrieve() error { if !pages.need { return nil } pages.need = false var pgs []*page.Page if len(pages.queries) > 0 { query := bleve.NewConjunctionQuery(pages.queries) total, uris, err := pages.ctx.app.pageIndexes.Query(pages.uri, query, pages.offset, pages.size) if err != nil { return err } for _, uri := range uris { pg := pages.ctx.get_page(uri) if pg != nil { pgs = append(pgs, pg) } } pages.total = total } else { datas := []interface{}{} for _, sfx := range pages.ctx.markdowns { vals, err := pages.ctx.app.pages.MultiGet(pages.uri, "."+sfx, pages.nested) if err != nil { return err } datas = append(datas, vals...) } for _, data := range datas { pg := page.NewPage("") if err := pg.SetData(data); err != nil { continue } pgs = append(pgs, pg) } pages.total = int64(len(pgs)) } if pages.order != nil { sorter := page.PageSort{ Field: pages.order.Field, Ascend: pages.order.Ascend, Pages: pgs, } sort.Sort(sorter) } pages.results = append(pages.results, pgs...) pages.ctx.Info("pages uri (%s) len(queries) (%d) total (%d) offset:size (%d:%d)", pages.uri, len(pages.queries), pages.total, pages.offset, pages.size) return nil }
// parseExpr2 handles AND expressions // // expr2 = expr3 {"AND" expr3} func (p *Parser) parseExpr2(ctx context) (tokType, bleve.Query, error) { queries := []bleve.Query{} prefixes := []tokType{} for { prefix, q, err := p.parseExpr3(ctx) if err != nil { return tEOF, nil, err } prefixes = append(prefixes, prefix) queries = append(queries, q) tok := p.next() if tok.typ != tAND { p.backup() break } } // let single, non-AND expressions bubble upward, prefix intact if len(queries) == 1 { return prefixes[0], queries[0], nil } // KLUDGINESS - prefixes on terms in AND expressions // we'll ignore "+" and treat "-" as NOT // eg: // `+alice AND -bob AND chuck` => `alice AND (NOT bob) AND chuck` for i, _ := range queries { if prefixes[i] == tMINUS { queries[i] = bleve.NewBooleanQuery( []bleve.Query{}, []bleve.Query{}, []bleve.Query{queries[i]}, // mustNot ) } } return tEOF, bleve.NewConjunctionQuery(queries), nil }
func bleveQueryFromExpr(expr query.Expr) (bleve.Query, error) { if e, ok := expr.(*query.FieldExpr); ok { return bleve.NewPhraseQuery([]string{e.Term}, e.Field), nil } else if e, ok := expr.(*query.BinaryExpr); ok { lhs, err := bleveQueryFromExpr(e.LHS) if err != nil { return nil, err } rhs, err := bleveQueryFromExpr(e.RHS) if err != nil { return nil, err } if e.Op == query.AND { return bleve.NewConjunctionQuery([]bleve.Query{lhs, rhs}), nil } else if e.Op == query.OR { return bleve.NewDisjunctionQuery([]bleve.Query{lhs, rhs}), nil } } return nil, nil }
func query(term, highlight string, index bleve.Index, u content.User, feedIds []data.FeedId, paging ...int) (ua []content.UserArticle, err error) { var query bleve.Query query = bleve.NewQueryStringQuery(term) if len(feedIds) > 0 { queries := make([]bleve.Query, len(feedIds)) conjunct := make([]bleve.Query, 2) for i, id := range feedIds { q := bleve.NewTermQuery(strconv.FormatInt(int64(id), 10)) q.SetField("FeedId") queries[i] = q } disjunct := bleve.NewDisjunctionQuery(queries) conjunct[0] = query conjunct[1] = disjunct query = bleve.NewConjunctionQuery(conjunct) } searchRequest := bleve.NewSearchRequest(query) if highlight != "" { searchRequest.Highlight = bleve.NewHighlightWithStyle(highlight) } limit, offset := pagingLimit(paging) searchRequest.Size = limit searchRequest.From = offset searchResult, err := index.Search(searchRequest) if err != nil { return } if len(searchResult.Hits) == 0 { return } articleIds := []data.ArticleId{} hitMap := map[data.ArticleId]*search.DocumentMatch{} for _, hit := range searchResult.Hits { if articleId, err := strconv.ParseInt(hit.ID, 10, 64); err == nil { id := data.ArticleId(articleId) articleIds = append(articleIds, id) hitMap[id] = hit } } ua = u.ArticlesById(articleIds) if u.HasErr() { return ua, u.Err() } for i := range ua { data := ua[i].Data() hit := hitMap[data.Id] if len(hit.Fragments) > 0 { data.Hit.Fragments = hit.Fragments ua[i].Data(data) } } return }
func TestQuerySyntaxParserValid(t *testing.T) { fivePointOh := 5.0 onePointOh := 1.0 theTruth := true theFalsehood := false jan_01_2015 := numeric_util.Int64ToFloat64(time.Date(2015, time.January, 01, 0, 0, 0, 0, time.UTC).UnixNano()) jan_02_2015 := numeric_util.Int64ToFloat64(time.Date(2015, time.January, 02, 0, 0, 0, 0, time.UTC).UnixNano()) mar_15_2015 := numeric_util.Int64ToFloat64(time.Date(2015, time.March, 15, 0, 0, 0, 0, time.UTC).UnixNano()) mar_16_2015 := numeric_util.Int64ToFloat64(time.Date(2015, time.March, 16, 0, 0, 0, 0, time.UTC).UnixNano()) tests := []struct { input string result bleve.Query mapping *bleve.IndexMapping }{ { input: "test", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("test"), }, { input: `"test phrase 1"`, mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("test phrase 1"), }, { input: "field:test", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("test").SetField("field"), }, // - is allowed inside a term, just not the start { input: "field:t-est", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("t-est").SetField("field"), }, // + is allowed inside a term, just not the start { input: "field:t+est", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("t+est").SetField("field"), }, // > is allowed inside a term, just not the start { input: "field:t>est", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("t>est").SetField("field"), }, // < is allowed inside a term, just not the start { input: "field:t<est", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("t<est").SetField("field"), }, // = is allowed inside a term, just not the start { input: "field:t=est", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("t=est").SetField("field"), }, { input: "+field1:test1", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("test1").SetField("field1"), }, { input: "-field2:test2", mapping: NewIndexMapping(), result: bleve.NewBooleanQuery( nil, nil, []bleve.Query{ bleve.NewMatchPhraseQuery("test2").SetField("field2"), }), }, { input: `field3:"test phrase 2"`, mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("test phrase 2").SetField("field3"), }, { input: `+field4:"test phrase 1"`, mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("test phrase 1").SetField("field4"), }, { input: `-field5:"test phrase 2"`, mapping: NewIndexMapping(), result: bleve.NewBooleanQuery( nil, nil, []bleve.Query{ bleve.NewMatchPhraseQuery("test phrase 2").SetField("field5"), }), }, { input: `+field6:test3 -field7:test4 field8:test5`, mapping: NewIndexMapping(), result: bleve.NewBooleanQuery( []bleve.Query{ bleve.NewMatchPhraseQuery("test3").SetField("field6"), }, []bleve.Query{ bleve.NewMatchPhraseQuery("test5").SetField("field8"), }, []bleve.Query{ bleve.NewMatchPhraseQuery("test4").SetField("field7"), }), }, { input: "test^3", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("test").SetBoost(3.0), }, { input: "test^3 other^6", mapping: NewIndexMapping(), result: bleve.NewBooleanQuery( nil, []bleve.Query{ bleve.NewMatchPhraseQuery("test").SetBoost(3.0), bleve.NewMatchPhraseQuery("other").SetBoost(6.0), }, nil, ), }, { input: "33", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("33"), }, { input: "field:33", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("33").SetField("field"), }, { input: "cat-dog", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("cat-dog"), }, /* // TODO: MatchPhraseQuery doesn't handle fuzziness... { input: "watex~", mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("watex").SetFuzziness(1), }, { input: "watex~2", mapping: NewIndexMapping(), result: bleve.NewBooleanQuery( nil, []bleve.Query{ bleve.NewMatchQuery("watex").SetFuzziness(2), }, nil), }, { input: "watex~ 2", mapping: NewIndexMapping(), result: bleve.NewBooleanQuery( nil, []bleve.Query{ bleve.NewMatchQuery("watex").SetFuzziness(1), bleve.NewMatchQuery("2"), }, nil), }, { input: "field:watex~", mapping: NewIndexMapping(), result: bleve.NewBooleanQuery( nil, []bleve.Query{ bleve.NewMatchQuery("watex").SetFuzziness(1).SetField("field"), }, nil), }, { input: "field:watex~2", mapping: NewIndexMapping(), result: bleve.NewBooleanQuery( nil, []bleve.Query{ bleve.NewMatchQuery("watex").SetFuzziness(2).SetField("field"), }, nil), }, */ { input: `field:555c3bb06f7a127cda000005`, mapping: NewIndexMapping(), result: bleve.NewMatchPhraseQuery("555c3bb06f7a127cda000005").SetField("field"), }, { input: `field:>5`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&fivePointOh, nil, &theFalsehood, nil).SetField("field"), }, { input: `field:>=5`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&fivePointOh, nil, &theTruth, nil).SetField("field"), }, { input: `field:<5`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(nil, &fivePointOh, nil, &theFalsehood).SetField("field"), }, { input: `field:<=5`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(nil, &fivePointOh, nil, &theTruth).SetField("field"), }, { input: `grapefruit AND lemon`, mapping: NewIndexMapping(), result: bleve.NewConjunctionQuery([]bleve.Query{ bleve.NewMatchPhraseQuery("grapefruit"), bleve.NewMatchPhraseQuery("lemon"), }), }, { input: `grapefruit OR lemon`, mapping: NewIndexMapping(), result: bleve.NewDisjunctionQuery([]bleve.Query{ bleve.NewMatchPhraseQuery("grapefruit"), bleve.NewMatchPhraseQuery("lemon"), }), }, { // default operator is OR input: `grapefruit lemon`, mapping: NewIndexMapping(), result: bleve.NewBooleanQuery( nil, []bleve.Query{ bleve.NewMatchPhraseQuery("grapefruit"), bleve.NewMatchPhraseQuery("lemon"), }, nil, ), }, { input: `grapefruit AND NOT lemon`, mapping: NewIndexMapping(), result: bleve.NewConjunctionQuery([]bleve.Query{ bleve.NewMatchPhraseQuery("grapefruit"), bleve.NewBooleanQuery(nil, nil, []bleve.Query{bleve.NewMatchPhraseQuery("lemon")}), }), }, { input: `field:(grapefruit AND lemon)`, mapping: NewIndexMapping(), result: bleve.NewConjunctionQuery([]bleve.Query{ bleve.NewMatchPhraseQuery("grapefruit").SetField("field"), bleve.NewMatchPhraseQuery("lemon").SetField("field"), }), }, { input: `-field:(grapefruit AND lemon)`, mapping: NewIndexMapping(), result: bleve.NewBooleanQuery(nil, nil, []bleve.Query{ bleve.NewConjunctionQuery([]bleve.Query{ bleve.NewMatchPhraseQuery("grapefruit").SetField("field"), bleve.NewMatchPhraseQuery("lemon").SetField("field"), }), }), }, { input: `shoesize:[1 TO 5]`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&onePointOh, &fivePointOh, &theTruth, &theTruth).SetField("shoesize"), }, { input: `shoesize:{1 TO 5}`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&onePointOh, &fivePointOh, &theFalsehood, &theFalsehood).SetField("shoesize"), }, { input: `shoesize:[1 TO 5}`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&onePointOh, &fivePointOh, &theTruth, &theFalsehood).SetField("shoesize"), }, { input: `shoesize:{1 TO 5]`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&onePointOh, &fivePointOh, &theFalsehood, &theTruth).SetField("shoesize"), }, { input: `shoesize:[ TO 5]`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(nil, &fivePointOh, nil, &theTruth).SetField("shoesize"), }, { input: `shoesize:[1 TO ]`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&onePointOh, nil, &theTruth, nil).SetField("shoesize"), }, // date ranges (note that endpoints and inclusivity might be modified by the parser) { input: `when:[2015-01-01 TO 2015-03-15]`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&jan_01_2015, &mar_16_2015, &theTruth, &theFalsehood).SetField("when"), }, { input: `when:{2015-01-01 TO 2015-03-15]`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&jan_02_2015, &mar_16_2015, &theTruth, &theFalsehood).SetField("when"), }, { input: `when:[2015-01-01 TO 2015-03-15}`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&jan_01_2015, &mar_15_2015, &theTruth, &theFalsehood).SetField("when"), }, { input: `when:>2015-03-15`, mapping: NewIndexMapping(), result: bleve.NewNumericRangeInclusiveQuery(&mar_16_2015, nil, &theTruth, nil).SetField("when"), }, } for _, test := range tests { q, err := Parse(test.input) if err != nil { t.Error(err) } if !reflect.DeepEqual(q, test.result) { t.Errorf("Expected %#v, got %#v: for `%s`", test.result, q, test.input) // t.Errorf("Expected %#v, got %#v: for %s", test.result.(*booleanQuery).Should.(*disjunctionQuery).Disjuncts[0], q.(*booleanQuery).Should.(*disjunctionQuery).Disjuncts[0], test.input) } } }
func resultsHandler(w http.ResponseWriter, r *http.Request) { var ( pageHTML = "results-search.html" pageInclude = "results-search.include" ) urlQuery := r.URL.Query() err := r.ParseForm() if err != nil { responseLogger(r, http.StatusBadRequest, err) w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf("error in POST: %s", err))) return } // Collect the submissions fields. submission := make(map[string]interface{}) // Basic Search results if r.Method == "GET" { for k, v := range urlQuery { if k == "all_ids" { if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil { submission[k] = b } } else if k == "from" || k == "size" || k == "total" { if i, err := strconv.Atoi(strings.Join(v, "")); err == nil { submission[k] = i } } else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" { submission[k] = strings.Join(v, "") } } } // Advanced Search results if r.Method == "POST" { for k, v := range r.Form { if k == "all_ids" { if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil { submission[k] = b } } else if k == "from" || k == "size" || k == "total" { if i, err := strconv.Atoi(strings.Join(v, "")); err == nil { submission[k] = i } } else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" { submission[k] = strings.Join(v, "") } } } q, err := mapToSearchQuery(submission) if err != nil { responseLogger(r, http.StatusBadRequest, err) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf("%s", err))) return } // // Note: Add logic to handle basic and advanced search... // // q NewQueryStringQuery // q_required NewQueryStringQuery with a + prefix for each strings.Fields(q_required) value // q_exact NewMatchPhraseQuery // q_excluded NewQueryStringQuery with a - prefix for each strings.Feilds(q_excluded) value // var conQry []bleve.Query if q.Q != "" { conQry = append(conQry, bleve.NewQueryStringQuery(q.Q)) } if q.QExact != "" { conQry = append(conQry, bleve.NewMatchPhraseQuery(q.QExact)) } var terms []string for _, s := range strings.Fields(q.QRequired) { terms = append(terms, fmt.Sprintf("+%s", strings.TrimSpace(s))) } for _, s := range strings.Fields(q.QExcluded) { terms = append(terms, fmt.Sprintf("-%s", strings.TrimSpace(s))) } if len(terms) > 0 { qString := strings.Join(terms, " ") conQry = append(conQry, bleve.NewQueryStringQuery(qString)) } qry := bleve.NewConjunctionQuery(conQry) if q.Size == 0 { q.Size = 10 } searchRequest := bleve.NewSearchRequestOptions(qry, q.Size, q.From, q.Explain) if searchRequest == nil { responseLogger(r, http.StatusBadRequest, fmt.Errorf("Can't build new search request options %+v, %s", qry, err)) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf("%s", err))) return } searchRequest.Highlight = bleve.NewHighlight() searchRequest.Highlight.AddField("title") searchRequest.Highlight.AddField("content_description") searchRequest.Highlight.AddField("subjects") searchRequest.Highlight.AddField("subjects_function") searchRequest.Highlight.AddField("subjects_topical") searchRequest.Highlight.AddField("extents") subjectFacet := bleve.NewFacetRequest("subjects", 3) searchRequest.AddFacet("subjects", subjectFacet) subjectTopicalFacet := bleve.NewFacetRequest("subjects_topical", 3) searchRequest.AddFacet("subjects_topical", subjectTopicalFacet) subjectFunctionFacet := bleve.NewFacetRequest("subjects_function", 3) searchRequest.AddFacet("subjects_function", subjectFunctionFacet) // Return all fields searchRequest.Fields = []string{ "title", "identifier", "content_description", "content_condition", "resource_type", "access_restrictions", "access_restrictions_note", "use_restrictins", "use_restrictons_note", "dates", "date_expression", "extents", "subjects", "subjects_function", "subjects_topical", "linked_agents_creators", "linked_agents_subjects", "link_agents_sources", "digital_objects.title", "digital_objects.file_uris", "related_resources", "deaccessions", "accession_date", "created", } searchResults, err := index.Search(searchRequest) if err != nil { responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Bleve results error %v, %s", qry, err)) w.WriteHeader(http.StatusInternalServerError) w.Write([]byte(fmt.Sprintf("%s", err))) return } // q (ciat.SearchQuery) performs double duty as both the structure for query submission as well // as carring the results to support paging and other types of navigation through // the query set. Results are a query with the bleve.SearchReults merged q.AttachSearchResults(searchResults) pageHTML = "results-search.html" pageInclude = "results-search.include" // Load my templates and setup to execute them tmpl, err := tmplfn.Assemble(tmplFuncs, path.Join(templatesDir, pageHTML), path.Join(templatesDir, pageInclude)) if err != nil { responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Template Errors: %s, %s, %s\n", pageHTML, pageInclude, err)) w.WriteHeader(http.StatusInternalServerError) w.Write([]byte(fmt.Sprintf("Template errors: %s", err))) return } // Render the page w.Header().Set("Content-Type", "text/html") var buf bytes.Buffer err = tmpl.Execute(&buf, q) //err = tmpl.Execute(w, q) if err != nil { responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Can't render %s, %s/%s, %s", templatesDir, pageHTML, pageInclude, err)) w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("Template error")) return } //NOTE: This bit of ugliness is here because I need to allow <mark> elements and ellipis in the results fragments w.Write(bytes.Replace(bytes.Replace(bytes.Replace(buf.Bytes(), []byte("<mark>"), []byte("<mark>"), -1), []byte("</mark>"), []byte("</mark>"), -1), []byte(`…`), []byte(`…`), -1)) }
func makeSearchQuery(queryString string, ids []string) (query.Query, error) { nodes, err := blevext.Parse(queryString) if err != nil { return nil, err } addIdsFilter := func(q query.Query) query.Query { if len(ids) == 0 { return q } return bleve.NewConjunctionQuery( query.NewDocIDQuery(ids), q, ) } var makeQuery func(*blevext.Node) (query.Query, error) makeQuery = func(n *blevext.Node) (query.Query, error) { if n == nil { return bleve.NewMatchAllQuery(), nil } switch n.Kind { case blevext.NodeAnd, blevext.NodeOr: left, err := makeQuery(n.Children[0]) if err != nil { return nil, err } right, err := makeQuery(n.Children[1]) if err != nil { return nil, err } if n.Kind == blevext.NodeOr { q := query.NewDisjunctionQuery([]query.Query{left, right}) q.Min = 1 return q, nil } return query.NewConjunctionQuery([]query.Query{left, right}), nil case blevext.NodeString, blevext.NodePhrase: fn := func(s string) query.FieldableQuery { return bleve.NewMatchQuery(s) } if n.Kind == blevext.NodePhrase { fn = func(s string) query.FieldableQuery { return blevext.NewAllMatchQuery(s) } } htmlQuery := fn(n.Value) htmlQuery.SetField("html") titleQuery := fn(n.Value) titleQuery.SetField("title") q := query.NewDisjunctionQuery([]query.Query{ addIdsFilter(htmlQuery), addIdsFilter(titleQuery), }) q.Min = 1 return q, nil } return nil, fmt.Errorf("unknown query node type: %d", n.Kind) } return makeQuery(nodes) }