Ejemplo n.º 1
0
func BlevePIndexQuerySamples() []cbgt.Documentation {
	return []cbgt.Documentation{
		cbgt.Documentation{
			Text: "A simple bleve query POST body:",
			JSON: &struct {
				*cbgt.QueryCtlParams
				*bleve.SearchRequest
			}{
				nil,
				&bleve.SearchRequest{
					From:  0,
					Size:  10,
					Query: bleve.NewQueryStringQuery("a sample query"),
				},
			},
		},
		cbgt.Documentation{
			Text: `An example POST body using from/size for results paging,
using ctl for a timeout and for "at_plus" consistency level.
On consistency, the index must have incorporated at least mutation
sequence-number 123 for partition (vbucket) 0 and mutation
sequence-number 234 for partition (vbucket) 1 (where vbucket 1
should have a vbucketUUID of a0b1c2):`,
			JSON: &struct {
				*cbgt.QueryCtlParams
				*bleve.SearchRequest
			}{
				&cbgt.QueryCtlParams{
					Ctl: cbgt.QueryCtl{
						Timeout: cbgt.QUERY_CTL_DEFAULT_TIMEOUT_MS,
						Consistency: &cbgt.ConsistencyParams{
							Level: "at_plus",
							Vectors: map[string]cbgt.ConsistencyVector{
								"customerIndex": cbgt.ConsistencyVector{
									"0":        123,
									"1/a0b1c2": 234,
								},
							},
						},
					},
				},
				&bleve.SearchRequest{
					From:      20,
					Size:      10,
					Fields:    []string{"*"},
					Query:     bleve.NewQueryStringQuery("alice smith"),
					Highlight: bleve.NewHighlight(),
					Explain:   true,
				},
			},
		},
	}
}
Ejemplo n.º 2
0
func (pages *Pages) QueryString(field string, str string) *Pages {
	q := bleve.NewQueryStringQuery(str)
	q.SetField(field)
	pages.queries = append(pages.queries, q)
	pages.clear()
	return pages
}
Ejemplo n.º 3
0
// Search is an http.HandlerFunc that accepts a Parse Cloud Code Webhook request.
//
// The expected query parameter is 'q'
func (i *Indexer) Search(w http.ResponseWriter, r *http.Request) {
	req, err := webhookRequest(r, i.webhookKey)
	if err != nil {
		writeErr(w, err)
		return
	}
	rawq := req.Params["q"]
	if rawq == nil {
		writeErr(w, fmt.Errorf("no term provided"))
		return
	}
	q, ok := rawq.(string)
	if q == "" || !ok {
		writeErr(w, fmt.Errorf("no term provided"))
		return
	}
	query := bleve.NewQueryStringQuery(q)
	search := bleve.NewSearchRequest(query)
	searchResults, err := i.index.Search(search)
	if err != nil {
		writeErr(w, err)
		return
	}
	spew.Dump(searchResults)
	ids := []string{}
	for _, h := range searchResults.Hits {
		ids = append(ids, h.ID)
	}
	err = json.NewEncoder(w).Encode(Response{
		Success: ids,
	})
	if err != nil {
		log.Println("error encoding response:", err)
	}
}
Ejemplo n.º 4
0
func (docstore *DocStoreExt) Search(collection, queryString string) ([]byte, error) {
	js := []byte{}
	query := bleve.NewQueryStringQuery(queryString)
	searchRequest := bleve.NewSearchRequest(query)
	searchResult, err := docstore.index.Search(searchRequest)
	if err != nil {
		return nil, err
	}
	for index, sr := range searchResult.Hits {
		jsPart := []byte{}
		_, err := docstore.fetchDoc(collection, sr.ID, &jsPart)
		if err != nil {
			return nil, err
		}
		js = append(js, addID(jsPart, sr.ID)...)
		if index != len(searchResult.Hits)-1 {
			js = append(js, []byte(",")...)
		}
	}
	if len(searchResult.Hits) > 0 {
		js = js[0 : len(js)-1]
	}
	js = append(js, []byte("]")...)
	// "_meta": searchResult,
	// "data":  docs,
	// TODO(tsileo) returns meta along with argument
	return js, nil
}
Ejemplo n.º 5
0
// Search method lookup for records using a query
func (i *bleveIndexer) Search(q string) (records []indexer.Record) {
	query := bleve.NewQueryStringQuery(q)
	request := bleve.NewSearchRequest(query)
	request.Highlight = bleve.NewHighlight()
	result, err := i.bleve.Search(request)
	if err != nil { // an empty query would cause this
		return
	}

	for _, match := range result.Hits {
		rec := i.Record(match.ID)
		loaded := rec.Load()

		if !loaded {
			continue
		}

		if len(match.Fragments["Body"]) > 0 {
			rec.SetBody([]byte(match.Fragments["Body"][0]))
		}

		records = append(records, rec)
	}

	return
}
Ejemplo n.º 6
0
func (b *Index) Search(queryStr string, offset, limit int, highlights bool) (*bleve.SearchResult, error) {
	query := bleve.NewQueryStringQuery(queryStr)
	req := bleve.NewSearchRequestOptions(query, limit, offset, false)
	if highlights {
		req.Highlight = bleve.NewHighlightWithStyle("ansi")
	}
	return b.index.Search(req)
}
Ejemplo n.º 7
0
func main() {

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		err = pprof.StartCPUProfile(f)
		if err != nil {
			log.Fatal(err)
		}
		defer pprof.StopCPUProfile()
	}

	if *indexPath == "" {
		log.Fatal("Specify index to query")
	}

	if flag.NArg() < 1 {
		log.Fatal("Specify search query")
	}

	// open index
	index, err := bleve.Open(*indexPath)
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		cerr := index.Close()
		if cerr != nil {
			log.Fatalf("error closing index: %v", err)
		}
	}()

	for i := 0; i < *repeat; i++ {
		// build a search with the provided parameters
		queryString := strings.Join(flag.Args(), " ")
		query := bleve.NewQueryStringQuery(queryString)
		searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain)

		// enable highlights if requested
		if *includeHighlights {
			searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi")
		}

		// execute the search
		searchResult, err := index.Search(searchRequest)
		if err != nil {
			log.Fatalf("search error: %v", err)
		}
		fmt.Println(searchResult)
	}
}
Ejemplo n.º 8
0
func (be *BleveEngine) Search(query string) (interface{}, error) {
	index, _ := bleve.Open(INDEX)
	bleveQuery := bleve.NewQueryStringQuery(query)
	searchRequest := bleve.NewSearchRequest(bleveQuery)
	searchResults, err := index.Search(searchRequest)
	if err != nil {
		return nil, err
	}

	return searchResults, nil
}
Ejemplo n.º 9
0
// Search performs a search of the index using the given query. Returns IDs of documents
// which satisfy all queries. Returns Doc IDs in sorted order, ascending.
func (i *Index) Search(q string) (DocIDs, error) {
	query := bleve.NewQueryStringQuery(q)
	searchRequest := bleve.NewSearchRequest(query)
	searchRequest.Size = maxSearchHitSize
	searchResults, err := i.Alias.Search(searchRequest)
	if err != nil {
		return nil, err
	}

	docIDs := make(DocIDs, 0, len(searchResults.Hits))
	for _, d := range searchResults.Hits {
		docIDs = append(docIDs, DocID(d.ID))
	}
	sort.Sort(docIDs)
	return docIDs, nil
}
Ejemplo n.º 10
0
// Find all comments matching text
// @return result.Result<[]*Comment, error>
func CommentsWithContent(repoPath, content string) result.Result {
	return openIndex(repoPath, func(repo *git.Repository, index bleve.Index) result.Result {
		query := bleve.NewQueryStringQuery(content)
		request := bleve.NewSearchRequest(query)
		return result.NewResult(index.Search(request)).FlatMap(func(match interface{}) result.Result {
			hits := match.(*bleve.SearchResult).Hits
			comments := make([]*gc.Comment, len(hits))
			for idx, hit := range hits {
				gc.CommentByID(repo, hit.ID).FlatMap(func(comment interface{}) result.Result {
					comments[idx] = comment.(*gc.Comment)
					return result.Result{}
				})
			}
			return result.NewSuccess(comments)
		})
	})
}
Ejemplo n.º 11
0
func (cache *BleveCache) Search(ctx context.Context, term string, options SearchOptions) ([]*RecipeLink, error) {
	q := bleve.NewQueryStringQuery(term)
	req := bleve.NewSearchRequest(q)
	res, err := cache.index.Search(req)
	if err != nil {
		return nil, err
	}
	links := make([]*RecipeLink, len(res.Hits))
	for i, hit := range res.Hits {
		filePath := cache.dbPath + "/" + hit.ID + ".json"
		links[i], err = loadRecipeLink(filePath)
		if err != nil {
			return links, err
		}
	}
	return links, err
}
Ejemplo n.º 12
0
// Search the swish index for a given query.
func searchBleve(queryString string) (Results, error) {
	index, err := openIndex()
	if err != nil {
		LogError(err)
		return Results{}, err
	}
	defer index.Close()

	newQueryString := ""
	for _, tmp := range strings.Split(queryString, " ") {
		word := strings.TrimSpace(tmp)
		if word[0] == '-' || word[0] == '+' {
			newQueryString += " " + word
		} else if word[0] == '~' {
			// Remove prefix to make term optional
			newQueryString += " " + word[1:]
		} else {
			newQueryString += " +" + word
		}
	}

	query := bleve.NewQueryStringQuery(newQueryString[1:]) // Remove leading space
	search := bleve.NewSearchRequest(query)
	search.Size = Config.MaxResults
	searchResults, err := index.Search(search)
	if err != nil {
		println("Invalid query string: '" + newQueryString[1:] + "'")
		LogError(err)
		return Results{}, err
	}

	var ids []Scroll
	for _, match := range searchResults.Hits {
		id := Id(match.ID)
		content, err := readScroll(id)
		TryLogError(err)
		scroll := Parse(string(id), content)
		ids = append(ids, scroll)
	}

	return Results{ids[:len(searchResults.Hits)], int(searchResults.Total)}, nil
}
Ejemplo n.º 13
0
Archivo: main.go Proyecto: d4l3k/campus
// search executes a search for rooms or buildings.
func (s *Server) search(w http.ResponseWriter, r *http.Request) {
	query := r.URL.Query()

	q := query.Get("q")
	typeFilter := query.Get("type")

	results := []*models.Index{}
	if idx, ok := s.idIndex[q]; ok {
		results = append(results, idx)
	} else {
		query := bleve.NewBooleanQuery()
		if len(q) > 0 {
			/*fuzzy_query := bleve.NewFuzzyQuery(q)
			fuzzy_query.FuzzinessVal = 3
			queryShould = append(queryShould, fuzzy_query)
			queryShould = append(queryShould, bleve.NewRegexpQuery("[a-zA-Z0-9_]*"+q+"[a-zA-Z0-9_]*"))
			queryShould = append(queryShould, bleve.NewQueryStringQuery(q))*/
			query.AddShould(bleve.NewQueryStringQuery(q))
		}

		if typeFilter != "all" {
			termQuery := bleve.NewTermQuery(typeFilter)
			query.AddMust(termQuery)
		}

		searchRequest := bleve.NewSearchRequest(query)
		searchRequest.Size = 25
		searchResult, err := s.index.Search(searchRequest)
		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}

		for _, result := range []*search.DocumentMatch(searchResult.Hits) {
			results = append(results, s.idIndex[result.ID])
		}
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(results)
}
Ejemplo n.º 14
0
// START OMIT
func main() {

	index, err := bleve.Open("gophercon.bleve")
	if err != nil {
		log.Fatal(err)
	}

	qString := `+content:debugger ` // HLQUERY
	qString += `title:"delve" `     // HLQUERY
	qString += `title:go~2 `        // HLQUERY
	qString += `-content:rust `     // HLQUERY
	qString += `word_count:>30`     // HLQUERY
	q := bleve.NewQueryStringQuery(qString)
	req := bleve.NewSearchRequest(q)
	req.Highlight = bleve.NewHighlightWithStyle("html")
	req.Fields = []string{"title", "author", "content", "word_count"}
	res, err := index.Search(req)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(res)
}
Ejemplo n.º 15
0
func buildQuery(args []string) query.Query {
	var q query.Query
	switch qtype {
	case "prefix":
		pquery := bleve.NewPrefixQuery(strings.Join(args[1:], " "))
		if qfield != "" {
			pquery.SetField(qfield)
		}
		q = pquery
	case "term":
		pquery := bleve.NewTermQuery(strings.Join(args[1:], " "))
		if qfield != "" {
			pquery.SetField(qfield)
		}
		q = pquery
	default:
		// build a search with the provided parameters
		queryString := strings.Join(args[1:], " ")
		q = bleve.NewQueryStringQuery(queryString)
	}
	return q
}
Ejemplo n.º 16
0
Archivo: main.go Proyecto: bozzcq/bleve
func main() {

	flag.Parse()

	if *indexPath == "" {
		log.Fatal("specify index to query")
	}

	if flag.NArg() < 1 {
		log.Fatal("Specify search query")
	}

	// open index
	index, err := bleve.Open(*indexPath)
	if err != nil {
		log.Fatal(err)
	}
	defer index.Close()

	// build a search with the provided parameters
	queryString := strings.Join(flag.Args(), " ")
	query := bleve.NewQueryStringQuery(queryString)
	searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain)

	// enable highlights if requested
	if *includeHighlights {
		searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi")
	}

	// execute the search
	searchResult, err := index.Search(searchRequest)
	if err != nil {
		log.Fatalf("search error: %v", err)
	}
	fmt.Println(searchResult)
}
Ejemplo n.º 17
0
func main() {

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		err = pprof.StartCPUProfile(f)
		if err != nil {
			log.Fatal(err)
		}
		defer pprof.StopCPUProfile()
	}

	if *indexPath == "" {
		log.Fatal("Specify index to query")
	}

	if flag.NArg() < 1 {
		log.Fatal("Specify search query")
	}

	// open index
	index, err := bleve.Open(*indexPath)
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		cerr := index.Close()
		if cerr != nil {
			log.Fatalf("error closing index: %v", err)
		}
	}()

	for i := 0; i < *repeat; i++ {
		var query bleve.Query

		switch *qtype {
		case "prefix":
			pquery := bleve.NewPrefixQuery(strings.Join(flag.Args(), " "))
			if *qfield != "" {
				pquery.SetField(*qfield)
			}
			query = pquery
		case "term":
			pquery := bleve.NewTermQuery(strings.Join(flag.Args(), " "))
			if *qfield != "" {
				pquery.SetField(*qfield)
			}
			query = pquery
		default:
			// build a search with the provided parameters
			queryString := strings.Join(flag.Args(), " ")
			query = bleve.NewQueryStringQuery(queryString)
		}

		searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain)

		// enable highlights if requested
		if *includeHighlights {
			searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi")
		}

		if *includeStoredFields {
			searchRequest.Fields = []string{"*"}
		}

		// execute the search
		searchResult, err := index.Search(searchRequest)
		if err != nil {
			log.Fatalf("search error: %v", err)
		}
		fmt.Println(searchResult)
	}
}
Ejemplo n.º 18
0
func query(term, highlight string, index bleve.Index, u content.User, feedIds []data.FeedId, paging ...int) (ua []content.UserArticle, err error) {
	var query bleve.Query

	query = bleve.NewQueryStringQuery(term)

	if len(feedIds) > 0 {
		queries := make([]bleve.Query, len(feedIds))
		conjunct := make([]bleve.Query, 2)

		for i, id := range feedIds {
			q := bleve.NewTermQuery(strconv.FormatInt(int64(id), 10))
			q.SetField("FeedId")

			queries[i] = q
		}

		disjunct := bleve.NewDisjunctionQuery(queries)

		conjunct[0] = query
		conjunct[1] = disjunct

		query = bleve.NewConjunctionQuery(conjunct)
	}

	searchRequest := bleve.NewSearchRequest(query)

	if highlight != "" {
		searchRequest.Highlight = bleve.NewHighlightWithStyle(highlight)
	}

	limit, offset := pagingLimit(paging)
	searchRequest.Size = limit
	searchRequest.From = offset

	searchResult, err := index.Search(searchRequest)

	if err != nil {
		return
	}

	if len(searchResult.Hits) == 0 {
		return
	}

	articleIds := []data.ArticleId{}
	hitMap := map[data.ArticleId]*search.DocumentMatch{}

	for _, hit := range searchResult.Hits {
		if articleId, err := strconv.ParseInt(hit.ID, 10, 64); err == nil {
			id := data.ArticleId(articleId)
			articleIds = append(articleIds, id)
			hitMap[id] = hit
		}
	}

	ua = u.ArticlesById(articleIds)
	if u.HasErr() {
		return ua, u.Err()
	}

	for i := range ua {
		data := ua[i].Data()

		hit := hitMap[data.Id]

		if len(hit.Fragments) > 0 {
			data.Hit.Fragments = hit.Fragments
			ua[i].Data(data)
		}
	}
	return
}
Ejemplo n.º 19
0
func Search(q string) (*bleve.SearchResult, error) {
	query := bleve.NewQueryStringQuery(q)
	req := bleve.NewSearchRequest(query)
	req.Highlight = bleve.NewHighlightWithStyle("html_ex")
	return index.Search(req)
}
Ejemplo n.º 20
0
func Example() {
	INDEX_DIR := "gojieba.bleve"
	messages := []struct {
		Id   string
		Body string
	}{
		{
			Id:   "1",
			Body: "你好",
		},
		{
			Id:   "2",
			Body: "交代",
		},
		{
			Id:   "3",
			Body: "长江大桥",
		},
	}

	indexMapping := bleve.NewIndexMapping()
	os.RemoveAll(INDEX_DIR)
	// clean index when example finished
	defer os.RemoveAll(INDEX_DIR)

	err := indexMapping.AddCustomTokenizer("gojieba",
		map[string]interface{}{
			"dictpath":     gojieba.DICT_PATH,
			"hmmpath":      gojieba.HMM_PATH,
			"userdictpath": gojieba.USER_DICT_PATH,
			"idf":          gojieba.IDF_PATH,
			"stop_words":   gojieba.STOP_WORDS_PATH,
			"type":         "gojieba",
		},
	)
	if err != nil {
		panic(err)
	}
	err = indexMapping.AddCustomAnalyzer("gojieba",
		map[string]interface{}{
			"type":      "gojieba",
			"tokenizer": "gojieba",
		},
	)
	if err != nil {
		panic(err)
	}
	indexMapping.DefaultAnalyzer = "gojieba"

	index, err := bleve.New(INDEX_DIR, indexMapping)
	if err != nil {
		panic(err)
	}
	for _, msg := range messages {
		if err := index.Index(msg.Id, msg); err != nil {
			panic(err)
		}
	}

	querys := []string{
		"你好世界",
		"亲口交代",
		"长江",
	}

	for _, q := range querys {
		req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(q))
		req.Highlight = bleve.NewHighlight()
		res, err := index.Search(req)
		if err != nil {
			panic(err)
		}
		fmt.Println(prettify(res))
	}

	// Output:
	// [{"id":"1","score":0.27650412875470115}]
	// [{"id":"2","score":0.27650412875470115}]
	// [{"id":"3","score":0.7027325540540822}]
}
Ejemplo n.º 21
0
func searchHandler(w http.ResponseWriter, r *http.Request) {
	d := map[string]interface{}{
		"Path": path.Base(*zimPath),
	}

	if !idx {
		templates["searchNoIdx"].Execute(w, d)
		return
	}

	if r.Method == "GET" {
		templates["search"].Execute(w, d)
		return
	}

	q := r.FormValue("search_data")
	if q == "" {
		templates["search"].Execute(w, d)
		return
	}

	query := bleve.NewQueryStringQuery(q)
	search := bleve.NewSearchRequest(query)
	search.Fields = []string{"Title"}

	sr, err := index.Search(search)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	if sr.Total > 0 {
		d["Info"] = fmt.Sprintf("%d matches for query [%s], took %s", sr.Total, q, sr.Took)

		// Constructs a list of Hits
		var l []map[string]string

		for _, h := range sr.Hits {
			idx, err := strconv.Atoi(h.ID)
			if err != nil {
				log.Println(err.Error())
				continue
			}
			a, err := Z.ArticleAtURLIdx(uint32(idx))
			if err != nil {
				continue
			}
			l = append(l, map[string]string{
				"Score": strconv.FormatFloat(h.Score, 'f', 1, 64),
				"Title": a.Title,
				"URL":   "/zim/" + a.FullURL()})

		}
		d["Hits"] = l

	} else {
		d["Info"] = fmt.Sprintf("No match for [%s], took %s", q, sr.Took)
		d["Hits"] = 0
	}

	templates["searchResult"].Execute(w, d)
}
Ejemplo n.º 22
0
func TestBeerSearchAll(t *testing.T) {
	defer os.RemoveAll("beer-search-test.bleve")

	mapping, err := buildIndexMapping()
	if err != nil {
		t.Fatal(err)
	}
	index, err := bleve.New("beer-search-test.bleve", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	// open the directory
	dirEntries, err := ioutil.ReadDir("data/")
	if err != nil {
		t.Fatal(err)
	}

	indexBatchSize := 100
	batch := index.NewBatch()
	batchCount := 0
	for _, dirEntry := range dirEntries {
		filename := dirEntry.Name()
		// read the bytes
		jsonBytes, err := ioutil.ReadFile("data/" + filename)
		if err != nil {
			t.Fatal(err)
		}
		// // shred them into a document
		ext := filepath.Ext(filename)
		docId := filename[:(len(filename) - len(ext))]
		batch.Index(docId, jsonBytes)
		batchCount++

		if batchCount >= indexBatchSize {
			err = index.Batch(batch)
			if err != nil {
				t.Fatal(err)
			}
			batch = index.NewBatch()
			batchCount = 0
		}
	}
	// flush the last batch
	if batchCount > 0 {
		err = index.Batch(batch)
		if err != nil {
			t.Fatal(err)
		}
	}

	expectedCount := uint64(7303)
	actualCount, err := index.DocCount()
	if err != nil {
		t.Error(err)
	}
	if actualCount != expectedCount {
		t.Errorf("expected %d documents, got %d", expectedCount, actualCount)
	}

	// run a term search
	termQuery := bleve.NewTermQuery("shock").SetField("name")
	termSearchRequest := bleve.NewSearchRequest(termQuery)
	termSearchResult, err := index.Search(termSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount := uint64(1)
	if termSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total)
	} else {
		expectedResultId := "anheuser_busch-shock_top"
		if termSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID)
		}
	}

	// run a match phrase search
	matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food")
	matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery)
	matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if matchPhraseSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total)
	} else {
		expectedResultId := "great_divide_brewing-wild_raspberry_ale"
		if matchPhraseSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID)
		}
	}

	// run a syntax query
	syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley")
	syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery)
	syntaxSearchResult, err := index.Search(syntaxSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if syntaxSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total)
	} else {
		expectedResultId := "iron_city_brewing_co-ic_light"
		if syntaxSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID)
		}
	}

	// run a numeric range search
	queryMin := 50.0
	numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv")
	numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery)
	numericSearchResult, err := index.Search(numericSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if numericSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total)
	} else {
		expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale"
		if numericSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID)
		}
	}

	// run a date range search
	queryStartDate := "2011-10-04"
	dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated")
	dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery)
	dateSearchResult, err := index.Search(dateSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(2)
	if dateSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total)
	} else {
		expectedResultId := "brasserie_du_bouffay-ambr"
		if dateSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID)
		}
	}

	// run a prefix search
	prefixQuery := bleve.NewPrefixQuery("adir").SetField("name")
	prefixSearchRequest := bleve.NewSearchRequest(prefixQuery)
	prefixSearchResult, err := index.Search(prefixSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if prefixSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total)
	} else {
		expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager"
		if prefixSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID)
		}
	}
}
Ejemplo n.º 23
0
func resultsHandler(w http.ResponseWriter, r *http.Request) {
	var (
		pageHTML    = "results-search.html"
		pageInclude = "results-search.include"
	)

	urlQuery := r.URL.Query()
	err := r.ParseForm()
	if err != nil {
		responseLogger(r, http.StatusBadRequest, err)
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(fmt.Sprintf("error in POST: %s", err)))
		return
	}

	// Collect the submissions fields.
	submission := make(map[string]interface{})
	// Basic Search results
	if r.Method == "GET" {
		for k, v := range urlQuery {
			if k == "all_ids" {
				if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil {
					submission[k] = b
				}
			} else if k == "from" || k == "size" || k == "total" {
				if i, err := strconv.Atoi(strings.Join(v, "")); err == nil {
					submission[k] = i
				}
			} else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" {
				submission[k] = strings.Join(v, "")
			}
		}
	}

	// Advanced Search results
	if r.Method == "POST" {
		for k, v := range r.Form {
			if k == "all_ids" {
				if b, err := strconv.ParseBool(strings.Join(v, "")); err == nil {
					submission[k] = b
				}
			} else if k == "from" || k == "size" || k == "total" {
				if i, err := strconv.Atoi(strings.Join(v, "")); err == nil {
					submission[k] = i
				}
			} else if k == "q" || k == "q_exact" || k == "q_excluded" || k == "q_required" {
				submission[k] = strings.Join(v, "")
			}
		}
	}

	q, err := mapToSearchQuery(submission)
	if err != nil {
		responseLogger(r, http.StatusBadRequest, err)
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(fmt.Sprintf("%s", err)))
		return
	}

	//
	// Note: Add logic to handle basic and advanced search...
	//
	// q           NewQueryStringQuery
	// q_required  NewQueryStringQuery with a + prefix for each strings.Fields(q_required) value
	// q_exact     NewMatchPhraseQuery
	// q_excluded NewQueryStringQuery with a - prefix for each strings.Feilds(q_excluded) value
	//
	var conQry []bleve.Query

	if q.Q != "" {
		conQry = append(conQry, bleve.NewQueryStringQuery(q.Q))
	}
	if q.QExact != "" {
		conQry = append(conQry, bleve.NewMatchPhraseQuery(q.QExact))
	}
	var terms []string
	for _, s := range strings.Fields(q.QRequired) {
		terms = append(terms, fmt.Sprintf("+%s", strings.TrimSpace(s)))
	}
	for _, s := range strings.Fields(q.QExcluded) {
		terms = append(terms, fmt.Sprintf("-%s", strings.TrimSpace(s)))
	}
	if len(terms) > 0 {
		qString := strings.Join(terms, " ")
		conQry = append(conQry, bleve.NewQueryStringQuery(qString))
	}

	qry := bleve.NewConjunctionQuery(conQry)
	if q.Size == 0 {
		q.Size = 10
	}
	searchRequest := bleve.NewSearchRequestOptions(qry, q.Size, q.From, q.Explain)
	if searchRequest == nil {
		responseLogger(r, http.StatusBadRequest, fmt.Errorf("Can't build new search request options %+v, %s", qry, err))
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(fmt.Sprintf("%s", err)))
		return
	}

	searchRequest.Highlight = bleve.NewHighlight()
	searchRequest.Highlight.AddField("title")
	searchRequest.Highlight.AddField("content_description")
	searchRequest.Highlight.AddField("subjects")
	searchRequest.Highlight.AddField("subjects_function")
	searchRequest.Highlight.AddField("subjects_topical")
	searchRequest.Highlight.AddField("extents")

	subjectFacet := bleve.NewFacetRequest("subjects", 3)
	searchRequest.AddFacet("subjects", subjectFacet)

	subjectTopicalFacet := bleve.NewFacetRequest("subjects_topical", 3)
	searchRequest.AddFacet("subjects_topical", subjectTopicalFacet)

	subjectFunctionFacet := bleve.NewFacetRequest("subjects_function", 3)
	searchRequest.AddFacet("subjects_function", subjectFunctionFacet)

	// Return all fields
	searchRequest.Fields = []string{
		"title",
		"identifier",
		"content_description",
		"content_condition",
		"resource_type",
		"access_restrictions",
		"access_restrictions_note",
		"use_restrictins",
		"use_restrictons_note",
		"dates",
		"date_expression",
		"extents",
		"subjects",
		"subjects_function",
		"subjects_topical",
		"linked_agents_creators",
		"linked_agents_subjects",
		"link_agents_sources",
		"digital_objects.title",
		"digital_objects.file_uris",
		"related_resources",
		"deaccessions",
		"accession_date",
		"created",
	}

	searchResults, err := index.Search(searchRequest)
	if err != nil {
		responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Bleve results error %v, %s", qry, err))
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(fmt.Sprintf("%s", err)))
		return
	}

	// q (ciat.SearchQuery) performs double duty as both the structure for query submission as well
	// as carring the results to support paging and other types of navigation through
	// the query set. Results are a query with the bleve.SearchReults merged
	q.AttachSearchResults(searchResults)
	pageHTML = "results-search.html"
	pageInclude = "results-search.include"

	// Load my templates and setup to execute them
	tmpl, err := tmplfn.Assemble(tmplFuncs, path.Join(templatesDir, pageHTML), path.Join(templatesDir, pageInclude))
	if err != nil {
		responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Template Errors: %s, %s, %s\n", pageHTML, pageInclude, err))
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(fmt.Sprintf("Template errors: %s", err)))
		return
	}
	// Render the page
	w.Header().Set("Content-Type", "text/html")
	var buf bytes.Buffer
	err = tmpl.Execute(&buf, q)
	//err = tmpl.Execute(w, q)
	if err != nil {
		responseLogger(r, http.StatusInternalServerError, fmt.Errorf("Can't render %s, %s/%s, %s", templatesDir, pageHTML, pageInclude, err))
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("Template error"))
		return
	}
	//NOTE: This bit of ugliness is here because I need to allow <mark> elements and ellipis in the results fragments
	w.Write(bytes.Replace(bytes.Replace(bytes.Replace(buf.Bytes(), []byte("&lt;mark&gt;"), []byte("<mark>"), -1), []byte("&lt;/mark&gt;"), []byte("</mark>"), -1), []byte(`…`), []byte(`&hellip;`), -1))
}
Ejemplo n.º 24
0
func TestBeerSearchAll(t *testing.T) {
	defer os.RemoveAll("beer-search-test.bleve")

	mapping, err := buildIndexMapping()
	if err != nil {
		t.Fatal(err)
	}
	index, err := bleve.New("beer-search-test.bleve", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	for jf := range walkDirectory("data/", t) {
		docId := jf.filename[0:strings.LastIndex(jf.filename, ".")]
		err = index.Index(docId, jf.contents)
		if err != nil {
			t.Error(err)
		}
	}

	expectedCount := uint64(7303)
	actualCount := index.DocCount()
	if actualCount != expectedCount {
		t.Errorf("expected %d documents, got %d", expectedCount, actualCount)
	}

	// run a term search
	termQuery := bleve.NewTermQuery("shock").SetField("name")
	termSearchRequest := bleve.NewSearchRequest(termQuery)
	termSearchResult, err := index.Search(termSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount := uint64(1)
	if termSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total)
	} else {
		expectedResultId := "anheuser_busch-shock_top"
		if termSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID)
		}
	}

	// run a match phrase search
	matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food")
	matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery)
	matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if matchPhraseSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total)
	} else {
		expectedResultId := "great_divide_brewing-wild_raspberry_ale"
		if matchPhraseSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID)
		}
	}

	// run a syntax query
	syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley")
	syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery)
	syntaxSearchResult, err := index.Search(syntaxSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if syntaxSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total)
	} else {
		expectedResultId := "iron_city_brewing_co-ic_light"
		if syntaxSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID)
		}
	}

	// run a numeric range search
	queryMin := 50.0
	numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv")
	numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery)
	numericSearchResult, err := index.Search(numericSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if numericSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total)
	} else {
		expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale"
		if numericSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID)
		}
	}

	// run a date range search
	queryStartDate := "2011-10-04"
	dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated")
	dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery)
	dateSearchResult, err := index.Search(dateSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(2)
	if dateSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total)
	} else {
		expectedResultId := "brasserie_du_bouffay-ambr"
		if dateSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID)
		}
	}

	// run a prefix search
	prefixQuery := bleve.NewPrefixQuery("adir").SetField("name")
	prefixSearchRequest := bleve.NewSearchRequest(prefixQuery)
	prefixSearchResult, err := index.Search(prefixSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if prefixSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total)
	} else {
		expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager"
		if prefixSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID)
		}
	}
}
Ejemplo n.º 25
0
func Example_beleveSearch() {
	// open a new index
	indexMapping := bleve.NewIndexMapping()

	err := indexMapping.AddCustomTokenizer("jieba",
		map[string]interface{}{
			"file": "../dict.txt",
			"type": "jieba",
		})
	if err != nil {
		log.Fatal(err)
	}

	// create a custom analyzer
	err = indexMapping.AddCustomAnalyzer("jieba",
		map[string]interface{}{
			"type":      "custom",
			"tokenizer": "jieba",
			"token_filters": []string{
				"possessive_en",
				"to_lower",
				"stop_en",
			},
		})

	if err != nil {
		log.Fatal(err)
	}

	indexMapping.DefaultAnalyzer = "jieba"
	cacheDir := "jieba.beleve"
	os.RemoveAll(cacheDir)
	index, err := bleve.New(cacheDir, indexMapping)

	if err != nil {
		log.Fatal(err)
	}

	docs := []struct {
		Title string
		Name  string
	}{
		{
			Title: "Doc 1",
			Name:  "This is the first document we’ve added",
		},
		{
			Title: "Doc 2",
			Name:  "The second one 你 中文测试中文 is even more interesting! 吃水果",
		},
		{
			Title: "Doc 3",
			Name:  "买水果然后来世博园。",
		},
		{
			Title: "Doc 4",
			Name:  "工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作",
		},
		{
			Title: "Doc 5",
			Name:  "咱俩交换一下吧。",
		},
	}
	// index docs
	for _, doc := range docs {
		index.Index(doc.Title, doc)
	}

	// search for some text
	for _, keyword := range []string{"水果世博园", "你", "first", "中文", "交换机", "交换"} {
		query := bleve.NewQueryStringQuery(keyword)
		search := bleve.NewSearchRequest(query)
		search.Highlight = bleve.NewHighlight()
		searchResults, err := index.Search(search)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Result of \"%s\": %d matches:\n", keyword, searchResults.Total)
		for i, hit := range searchResults.Hits {
			rv := fmt.Sprintf("%d. %s, (%f)\n", i+searchResults.Request.From+1, hit.ID, hit.Score)
			for fragmentField, fragments := range hit.Fragments {
				rv += fmt.Sprintf("%s: ", fragmentField)
				for _, fragment := range fragments {
					rv += fmt.Sprintf("%s", fragment)
				}
			}
			fmt.Printf("%s\n", rv)
		}
	}
	// Output:
	// Result of "水果世博园": 2 matches:
	// 1. Doc 3, (1.099550)
	// Name: 买<span class="highlight">水果</span>然后来<span class="highlight">世博</span>园。
	// 2. Doc 2, (0.031941)
	// Name: The second one 你 中文测试中文 is even more interesting! 吃<span class="highlight">水果</span>
	// Result of "你": 1 matches:
	// 1. Doc 2, (0.391161)
	// Name: The second one <span class="highlight">你</span> 中文测试中文 is even more interesting! 吃水果
	// Result of "first": 1 matches:
	// 1. Doc 1, (0.512150)
	// Name: This is the <span class="highlight">first</span> document we’ve added
	// Result of "中文": 1 matches:
	// 1. Doc 2, (0.553186)
	// Name: The second one 你 <span class="highlight">中文</span>测试<span class="highlight">中文</span> is even more interesting! 吃水果
	// Result of "交换机": 2 matches:
	// 1. Doc 4, (0.608495)
	// Name: 工信处女干事每月经过下属科室都要亲口交代24口<span class="highlight">交换机</span>等技术性器件的安装工作
	// 2. Doc 5, (0.086700)
	// Name: 咱俩<span class="highlight">交换</span>一下吧。
	// Result of "交换": 2 matches:
	// 1. Doc 5, (0.534158)
	// Name: 咱俩<span class="highlight">交换</span>一下吧。
	// 2. Doc 4, (0.296297)
	// Name: 工信处女干事每月经过下属科室都要亲口交代24口<span class="highlight">交换</span>机等技术性器件的安装工作
}