예제 #1
1
파일: web.go 프로젝트: pmezard/apec
func findOffersFromText(index bleve.Index, query string, ids []string) (
	[]datedOffer, error) {

	if query == "" {
		return nil, nil
	}
	datedOffers := []datedOffer{}
	q, err := makeSearchQuery(query, ids)
	if err != nil {
		return nil, err
	}
	rq := bleve.NewSearchRequest(q)
	rq.Size = 20000
	rq.Fields = []string{"date"}
	res, err := index.Search(rq)
	if err != nil {
		return nil, err
	}
	for _, doc := range res.Hits {
		date, ok := doc.Fields["date"].(string)
		if !ok {
			return nil, fmt.Errorf("could not retrieve date for %s", doc.ID)
		}
		datedOffers = append(datedOffers, datedOffer{
			Date: date,
			Id:   doc.ID,
		})
	}
	return datedOffers, nil
}
예제 #2
0
파일: section.go 프로젝트: romanoff/gowiki
func (self *Wiki) Search(queryString string) ([]*SearchResult, error) {
	query := bleve.NewMatchQuery(queryString)
	search := bleve.NewSearchRequest(query)
	search.Highlight = bleve.NewHighlight()
	searchResult, err := self.Index.Search(search)
	if err != nil {
		return nil, err
	}
	results := []*SearchResult{}
	for _, result := range searchResult.Hits {
		section, article := self.Find(result.ID)
		if article == nil && section == nil {
			return nil, fmt.Errorf("%v section or article not found", result.ID)
		}
		text := ""
		for _, values := range result.Fragments {
			for _, value := range values {
				text += value
			}
		}
		name := ""
		if section != nil {
			name = section.Name
		}
		if article != nil {
			name = article.Name
		}
		results = append(results, &SearchResult{Path: result.ID, Name: name, Text: text})
	}
	return results, nil
}
예제 #3
0
파일: search.go 프로젝트: pmezard/apec
func search(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return err
	}
	index, err := bleve.Open(cfg.Index())
	if err != nil {
		return err
	}
	defer index.Close()
	q, err := makeSearchQuery(*searchQuery, nil)
	if err != nil {
		return err
	}
	rq := bleve.NewSearchRequest(q)
	rq.Size = 100
	ids := []string{}
	for {
		res, err := index.Search(rq)
		if err != nil {
			return err
		}
		for _, doc := range res.Hits {
			ids = append(ids, doc.ID)
		}
		if len(res.Hits) < rq.Size {
			break
		}
		rq.From += rq.Size
	}
	return printOffers(store, ids)
}
예제 #4
0
// Search method lookup for records using a query
func (i *bleveIndexer) Search(q string) (records []indexer.Record) {
	query := bleve.NewQueryStringQuery(q)
	request := bleve.NewSearchRequest(query)
	request.Highlight = bleve.NewHighlight()
	result, err := i.bleve.Search(request)
	if err != nil { // an empty query would cause this
		return
	}

	for _, match := range result.Hits {
		rec := i.Record(match.ID)
		loaded := rec.Load()

		if !loaded {
			continue
		}

		if len(match.Fragments["Body"]) > 0 {
			rec.SetBody([]byte(match.Fragments["Body"][0]))
		}

		records = append(records, rec)
	}

	return
}
예제 #5
0
func main() {
	// open a new index
	mapping := bleve.NewIndexMapping()
	index, err := bleve.New("example.bleve", mapping)
	if err != nil {
		fmt.Println(err)
		return
	}

	data := struct {
		Name string
	}{
		Name: "text",
	}

	// index some data
	index.Index("id", data)

	// search for some text
	query := bleve.NewMatchQuery("text")
	search := bleve.NewSearchRequest(query)
	searchResults, err := index.Search(search)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(searchResults)
}
예제 #6
0
func (docstore *DocStoreExt) Search(collection, queryString string) ([]byte, error) {
	js := []byte{}
	query := bleve.NewQueryStringQuery(queryString)
	searchRequest := bleve.NewSearchRequest(query)
	searchResult, err := docstore.index.Search(searchRequest)
	if err != nil {
		return nil, err
	}
	for index, sr := range searchResult.Hits {
		jsPart := []byte{}
		_, err := docstore.fetchDoc(collection, sr.ID, &jsPart)
		if err != nil {
			return nil, err
		}
		js = append(js, addID(jsPart, sr.ID)...)
		if index != len(searchResult.Hits)-1 {
			js = append(js, []byte(",")...)
		}
	}
	if len(searchResult.Hits) > 0 {
		js = js[0 : len(js)-1]
	}
	js = append(js, []byte("]")...)
	// "_meta": searchResult,
	// "data":  docs,
	// TODO(tsileo) returns meta along with argument
	return js, nil
}
예제 #7
0
파일: easysearch.go 프로젝트: DexterB/Go
func main() {
	// Open an index if it already exists.
	index, err := bleve.Open("example.bleve")
	if err != nil {
		mapping := bleve.NewIndexMapping()
		index, err = bleve.New("example.bleve", mapping)
		if err != nil {
			fmt.Println(err)
			return
		}
	}

	data := struct {
		Name string
	}{
		Name: "text",
	}

	// Index some data.
	index.Index("id", data)

	// Search for some text
	query := bleve.NewMatchQuery("text")
	search := bleve.NewSearchRequest(query)
	searchResults, err := index.Search(search)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println(searchResults)
}
예제 #8
0
파일: indexer.go 프로젝트: tmc/parsesearch
// Search is an http.HandlerFunc that accepts a Parse Cloud Code Webhook request.
//
// The expected query parameter is 'q'
func (i *Indexer) Search(w http.ResponseWriter, r *http.Request) {
	req, err := webhookRequest(r, i.webhookKey)
	if err != nil {
		writeErr(w, err)
		return
	}
	rawq := req.Params["q"]
	if rawq == nil {
		writeErr(w, fmt.Errorf("no term provided"))
		return
	}
	q, ok := rawq.(string)
	if q == "" || !ok {
		writeErr(w, fmt.Errorf("no term provided"))
		return
	}
	query := bleve.NewQueryStringQuery(q)
	search := bleve.NewSearchRequest(query)
	searchResults, err := i.index.Search(search)
	if err != nil {
		writeErr(w, err)
		return
	}
	spew.Dump(searchResults)
	ids := []string{}
	for _, h := range searchResults.Hits {
		ids = append(ids, h.ID)
	}
	err = json.NewEncoder(w).Encode(Response{
		Success: ids,
	})
	if err != nil {
		log.Println("error encoding response:", err)
	}
}
예제 #9
0
func TestFindByAnything(t *testing.T) {
	db, eventList := dbCreate()
	idx := idxCreate()
	indexEvents(idx, eventList)

	// We are looking to an Event with some string which match with dotGo
	query := bleve.NewMatchQuery("dotGo")
	searchRequest := bleve.NewSearchRequest(query)
	searchResult, err := idx.Search(searchRequest)
	if err != nil {
		t.Error("Something wrong happen with the search", err, ballotX)
	} else {
		t.Log("Should search the query", checkMark)
	}

	if searchResult.Total != 1 {
		t.Error("Only 1 result are expected, got ", searchResult.Total, ballotX)
	} else {
		t.Log("Should return only one result", checkMark)
	}

	event := &Event{}
	db.First(&event, &searchResult.Hits[0].ID)
	if event.Name != "dotGo 2015" {
		t.Error("Expected \"dotGo 2015\", Receive: ", event.Name)
	} else {
		t.Log("Should return an event with the name equal a", event.Name, checkMark)
	}

	idxDestroy()
	dbDestroy()
}
예제 #10
0
파일: main.go 프로젝트: fairlance/backend
func jobTags(w http.ResponseWriter, r *http.Request) {

	query := bleve.NewMatchAllQuery()

	searchRequest := bleve.NewSearchRequest(query)
	searchRequest.Fields = []string{"tags"}

	tagsFacet := bleve.NewFacetRequest("tags", 99999)
	searchRequest.AddFacet("tags", tagsFacet)

	jobsSearchResults, err := doRequest("jobs", searchRequest)
	if err != nil {
		respond.With(w, r, http.StatusInternalServerError, err)
		return
	}

	tags := []string{}
	for _, t := range jobsSearchResults.Facets["tags"].Terms {
		tags = append(tags, fmt.Sprintf("%s", t.Term))
	}

	respond.With(w, r, http.StatusOK, struct {
		Total int      `json:"total"`
		Tags  []string `json:"tags"`
	}{
		Total: len(tags),
		Tags:  tags,
	})
}
예제 #11
0
파일: density.go 프로젝트: pmezard/apec
// listPoints returns the location of offers satisfying specified full-text
// query. If query is empty, it returns all locations. If not nil, spatial is
// exploited as a cache to fetch indexed offers and their locations, which
// avoid store lookups.
func listPoints(store *Store, index bleve.Index, spatial *SpatialIndex,
	query string) ([]Point, error) {

	var ids []string
	if query == "" {
		if spatial != nil {
			ids = spatial.List()
		} else {
			list, err := store.List()
			if err != nil {
				return nil, err
			}
			ids = list
		}
	} else {
		q, err := makeSearchQuery(query, nil)
		if err != nil {
			return nil, err
		}
		rq := bleve.NewSearchRequest(q)
		rq.Size = 20000
		res, err := index.Search(rq)
		if err != nil {
			return nil, err
		}
		for _, doc := range res.Hits {
			ids = append(ids, doc.ID)
		}
	}
	points := make([]Point, 0, len(ids))
	for _, id := range ids {
		var p *Point
		if spatial != nil {
			offer := spatial.Get(id)
			if offer != nil {
				p = &offer.Point
			}
		}
		if p == nil {
			loc, _, err := store.GetLocation(id)
			if err != nil {
				return nil, err
			}
			if loc == nil {
				continue
			}
			p = &Point{
				Lat: loc.Lat,
				Lon: loc.Lon,
			}
		}
		points = append(points, *p)
	}
	return points, nil
}
예제 #12
0
func (be *BleveEngine) Search(query string) (interface{}, error) {
	index, _ := bleve.Open(INDEX)
	bleveQuery := bleve.NewQueryStringQuery(query)
	searchRequest := bleve.NewSearchRequest(bleveQuery)
	searchResults, err := index.Search(searchRequest)
	if err != nil {
		return nil, err
	}

	return searchResults, nil
}
예제 #13
0
파일: search.go 프로젝트: misakik/f3
//See https://godoc.org/github.com/blevesearch/bleve#SearchResult
func Search(keyword string) (*bleve.SearchResult, error) {
	fmt.Println(keyword)

	query := bleve.NewMatchQuery(keyword)
	request := bleve.NewSearchRequest(query)
	request.Size = 20
	//request.Fields = []string{"Name", "Text", "Size", "IsDir", "ModTime"}

	result, err := LimeIndex.Search(request)
	if err != nil {
		return nil, err
	}
	return result, nil
}
예제 #14
0
// START OMIT
func main() {
	index, err := bleve.Open("people.bleve") // HLOPEN
	if err != nil {
		log.Fatal(err)
	}

	query := bleve.NewTermQuery("marty")     // HLQUERY
	request := bleve.NewSearchRequest(query) // HLREQ
	result, err := index.Search(request)     // HLSEARCH
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(result)
}
예제 #15
0
// this test reproduces bug #87
// https://github.com/blevesearch/bleve/issues/87
// because of which, it will deadlock
func TestBeerSearchBug87(t *testing.T) {
	defer os.RemoveAll("beer-search-test.bleve")

	mapping, err := buildIndexMapping()
	if err != nil {
		t.Fatal(err)
	}
	index, err := bleve.New("beer-search-test.bleve", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	// start indexing documents in the background
	go func() {
		// open the directory
		dirEntries, err := ioutil.ReadDir("data/")
		if err != nil {
			t.Fatal(err)
		}

		for _, dirEntry := range dirEntries {
			filename := dirEntry.Name()
			jsonBytes, err := ioutil.ReadFile("data/" + filename)
			if err != nil {
				t.Fatal(err)
			}
			ext := filepath.Ext(filename)
			docId := filename[:(len(filename) - len(ext))]
			index.Index(docId, jsonBytes)
		}
	}()

	// give indexing a head start
	time.Sleep(1 * time.Second)

	// start querying
	for i := 0; i < 1000; i++ {
		time.Sleep(1 * time.Millisecond)
		termQuery := bleve.NewTermQuery("shock").SetField("name")
		termSearchRequest := bleve.NewSearchRequest(termQuery)
		// termSearchRequest.AddFacet("styles", bleve.NewFacetRequest("style", 3))
		termSearchRequest.Fields = []string{"abv"}
		_, err := index.Search(termSearchRequest)
		if err != nil {
			t.Error(err)
		}
	}
}
예제 #16
0
파일: index.go 프로젝트: alisheikh/ekanite
// Search performs a search of the index using the given query. Returns IDs of documents
// which satisfy all queries. Returns Doc IDs in sorted order, ascending.
func (i *Index) Search(q string) (DocIDs, error) {
	query := bleve.NewQueryStringQuery(q)
	searchRequest := bleve.NewSearchRequest(query)
	searchRequest.Size = maxSearchHitSize
	searchResults, err := i.Alias.Search(searchRequest)
	if err != nil {
		return nil, err
	}

	docIDs := make(DocIDs, 0, len(searchResults.Hits))
	for _, d := range searchResults.Hits {
		docIDs = append(docIDs, DocID(d.ID))
	}
	sort.Sort(docIDs)
	return docIDs, nil
}
예제 #17
0
// Find all comments matching text
// @return result.Result<[]*Comment, error>
func CommentsWithContent(repoPath, content string) result.Result {
	return openIndex(repoPath, func(repo *git.Repository, index bleve.Index) result.Result {
		query := bleve.NewQueryStringQuery(content)
		request := bleve.NewSearchRequest(query)
		return result.NewResult(index.Search(request)).FlatMap(func(match interface{}) result.Result {
			hits := match.(*bleve.SearchResult).Hits
			comments := make([]*gc.Comment, len(hits))
			for idx, hit := range hits {
				gc.CommentByID(repo, hit.ID).FlatMap(func(comment interface{}) result.Result {
					comments[idx] = comment.(*gc.Comment)
					return result.Result{}
				})
			}
			return result.NewSuccess(comments)
		})
	})
}
예제 #18
0
func (cache *BleveCache) Search(ctx context.Context, term string, options SearchOptions) ([]*RecipeLink, error) {
	q := bleve.NewQueryStringQuery(term)
	req := bleve.NewSearchRequest(q)
	res, err := cache.index.Search(req)
	if err != nil {
		return nil, err
	}
	links := make([]*RecipeLink, len(res.Hits))
	for i, hit := range res.Hits {
		filePath := cache.dbPath + "/" + hit.ID + ".json"
		links[i], err = loadRecipeLink(filePath)
		if err != nil {
			return links, err
		}
	}
	return links, err
}
예제 #19
0
// START OMIT
func main() {

	index, err := bleve.Open("gophercon.bleve")
	if err != nil {
		log.Fatal(err)
	}

	q := bleve.NewMatchAllQuery()
	req := bleve.NewSearchRequest(q)
	req.Size = 0
	req.AddFacet("sections", // HL
		bleve.NewFacetRequest("section", 50)) // HL
	res, err := index.Search(req)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(res)
}
예제 #20
0
파일: Bleve.go 프로젝트: yzhs/alexandria
// Search the swish index for a given query.
func searchBleve(queryString string) (Results, error) {
	index, err := openIndex()
	if err != nil {
		LogError(err)
		return Results{}, err
	}
	defer index.Close()

	newQueryString := ""
	for _, tmp := range strings.Split(queryString, " ") {
		word := strings.TrimSpace(tmp)
		if word[0] == '-' || word[0] == '+' {
			newQueryString += " " + word
		} else if word[0] == '~' {
			// Remove prefix to make term optional
			newQueryString += " " + word[1:]
		} else {
			newQueryString += " +" + word
		}
	}

	query := bleve.NewQueryStringQuery(newQueryString[1:]) // Remove leading space
	search := bleve.NewSearchRequest(query)
	search.Size = Config.MaxResults
	searchResults, err := index.Search(search)
	if err != nil {
		println("Invalid query string: '" + newQueryString[1:] + "'")
		LogError(err)
		return Results{}, err
	}

	var ids []Scroll
	for _, match := range searchResults.Hits {
		id := Id(match.ID)
		content, err := readScroll(id)
		TryLogError(err)
		scroll := Parse(string(id), content)
		ids = append(ids, scroll)
	}

	return Results{ids[:len(searchResults.Hits)], int(searchResults.Total)}, nil
}
예제 #21
0
func (brs *BleveRecipeStore) All() ([]model.Recipe, error) {
	recipes := []model.Recipe{}

	q := bleve.NewMatchAllQuery()
	src := bleve.NewSearchRequest(q)
	res, err := brs.idx.Search(src)

	for _, hit := range res.Hits {
		data, err := brs.idx.GetInternal([]byte(hit.ID))
		if err != nil {
			panic(err)
		}

		recipe := model.Recipe{}
		json.Unmarshal(data, &recipe)
		recipes = append(recipes, recipe)
	}

	return recipes, err
}
예제 #22
0
파일: main.go 프로젝트: d4l3k/campus
// search executes a search for rooms or buildings.
func (s *Server) search(w http.ResponseWriter, r *http.Request) {
	query := r.URL.Query()

	q := query.Get("q")
	typeFilter := query.Get("type")

	results := []*models.Index{}
	if idx, ok := s.idIndex[q]; ok {
		results = append(results, idx)
	} else {
		query := bleve.NewBooleanQuery()
		if len(q) > 0 {
			/*fuzzy_query := bleve.NewFuzzyQuery(q)
			fuzzy_query.FuzzinessVal = 3
			queryShould = append(queryShould, fuzzy_query)
			queryShould = append(queryShould, bleve.NewRegexpQuery("[a-zA-Z0-9_]*"+q+"[a-zA-Z0-9_]*"))
			queryShould = append(queryShould, bleve.NewQueryStringQuery(q))*/
			query.AddShould(bleve.NewQueryStringQuery(q))
		}

		if typeFilter != "all" {
			termQuery := bleve.NewTermQuery(typeFilter)
			query.AddMust(termQuery)
		}

		searchRequest := bleve.NewSearchRequest(query)
		searchRequest.Size = 25
		searchResult, err := s.index.Search(searchRequest)
		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}

		for _, result := range []*search.DocumentMatch(searchResult.Hits) {
			results = append(results, s.idIndex[result.ID])
		}
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(results)
}
예제 #23
0
// this test reproduces bug #87
// https://github.com/blevesearch/bleve/issues/87
// because of which, it will deadlock
func TestBeerSearchBug87(t *testing.T) {
	defer os.RemoveAll("beer-search-test.bleve")

	mapping, err := buildIndexMapping()
	if err != nil {
		t.Fatal(err)
	}
	index, err := bleve.New("beer-search-test.bleve", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	var wg sync.WaitGroup
	wg.Add(1)
	// start indexing documents in the background
	go func() {
		for jf := range walkDirectory("data/", t) {
			docId := jf.filename[0:strings.LastIndex(jf.filename, ".")]
			err = index.Index(docId, jf.contents)
			if err != nil {
				t.Error(err)
			}
		}
		wg.Done()
	}()

	for i := 0; i < 50; i++ {
		time.Sleep(1 * time.Second)
		termQuery := bleve.NewTermQuery("shock").SetField("name")
		termSearchRequest := bleve.NewSearchRequest(termQuery)
		// termSearchRequest.AddFacet("styles", bleve.NewFacetRequest("style", 3))
		termSearchRequest.Fields = []string{"abv"}
		_, err := index.Search(termSearchRequest)
		if err != nil {
			t.Error(err)
		}
	}

	wg.Wait()
}
예제 #24
0
// START OMIT
func main() {

	index, err := bleve.Open("gophercon.bleve")
	if err != nil {
		log.Fatal(err)
	}

	qString := `+content:debugger ` // HLQUERY
	qString += `title:"delve" `     // HLQUERY
	qString += `title:go~2 `        // HLQUERY
	qString += `-content:rust `     // HLQUERY
	qString += `word_count:>30`     // HLQUERY
	q := bleve.NewQueryStringQuery(qString)
	req := bleve.NewSearchRequest(q)
	req.Highlight = bleve.NewHighlightWithStyle("html")
	req.Fields = []string{"title", "author", "content", "word_count"}
	res, err := index.Search(req)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(res)
}
예제 #25
0
파일: main.go 프로젝트: fairlance/backend
func freelancers(w http.ResponseWriter, r *http.Request) {

	searchRequest := bleve.NewSearchRequest(bleve.NewMatchAllQuery())
	searchRequest.Fields = []string{"*"}

	freelnacersSearchResults, err := doRequest("freelancers", searchRequest)
	if err != nil {
		respond.With(w, r, http.StatusInternalServerError, err)
		return
	}

	freelancers := []interface{}{}
	for _, hit := range freelnacersSearchResults.Hits {
		freelancers = append(freelancers, hit.Fields)
	}
	respond.With(w, r, http.StatusOK, struct {
		Total int           `json:"total"`
		Items []interface{} `json:"items"`
	}{
		Total: len(freelancers),
		Items: freelancers,
	})
}
예제 #26
0
파일: article.go 프로젝트: RavenB/readeef
func query(term, highlight string, index bleve.Index, u content.User, feedIds []data.FeedId, paging ...int) (ua []content.UserArticle, err error) {
	var query bleve.Query

	query = bleve.NewQueryStringQuery(term)

	if len(feedIds) > 0 {
		queries := make([]bleve.Query, len(feedIds))
		conjunct := make([]bleve.Query, 2)

		for i, id := range feedIds {
			q := bleve.NewTermQuery(strconv.FormatInt(int64(id), 10))
			q.SetField("FeedId")

			queries[i] = q
		}

		disjunct := bleve.NewDisjunctionQuery(queries)

		conjunct[0] = query
		conjunct[1] = disjunct

		query = bleve.NewConjunctionQuery(conjunct)
	}

	searchRequest := bleve.NewSearchRequest(query)

	if highlight != "" {
		searchRequest.Highlight = bleve.NewHighlightWithStyle(highlight)
	}

	limit, offset := pagingLimit(paging)
	searchRequest.Size = limit
	searchRequest.From = offset

	searchResult, err := index.Search(searchRequest)

	if err != nil {
		return
	}

	if len(searchResult.Hits) == 0 {
		return
	}

	articleIds := []data.ArticleId{}
	hitMap := map[data.ArticleId]*search.DocumentMatch{}

	for _, hit := range searchResult.Hits {
		if articleId, err := strconv.ParseInt(hit.ID, 10, 64); err == nil {
			id := data.ArticleId(articleId)
			articleIds = append(articleIds, id)
			hitMap[id] = hit
		}
	}

	ua = u.ArticlesById(articleIds)
	if u.HasErr() {
		return ua, u.Err()
	}

	for i := range ua {
		data := ua[i].Data()

		hit := hitMap[data.Id]

		if len(hit.Fragments) > 0 {
			data.Hit.Fragments = hit.Fragments
			ua[i].Data(data)
		}
	}
	return
}
예제 #27
0
파일: index.go 프로젝트: oblank/knowledgedb
func Search(q string) (*bleve.SearchResult, error) {
	query := bleve.NewQueryStringQuery(q)
	req := bleve.NewSearchRequest(query)
	req.Highlight = bleve.NewHighlightWithStyle("html_ex")
	return index.Search(req)
}
예제 #28
0
func Example() {
	INDEX_DIR := "gojieba.bleve"
	messages := []struct {
		Id   string
		Body string
	}{
		{
			Id:   "1",
			Body: "你好",
		},
		{
			Id:   "2",
			Body: "交代",
		},
		{
			Id:   "3",
			Body: "长江大桥",
		},
	}

	indexMapping := bleve.NewIndexMapping()
	os.RemoveAll(INDEX_DIR)
	// clean index when example finished
	defer os.RemoveAll(INDEX_DIR)

	err := indexMapping.AddCustomTokenizer("gojieba",
		map[string]interface{}{
			"dictpath":     gojieba.DICT_PATH,
			"hmmpath":      gojieba.HMM_PATH,
			"userdictpath": gojieba.USER_DICT_PATH,
			"idf":          gojieba.IDF_PATH,
			"stop_words":   gojieba.STOP_WORDS_PATH,
			"type":         "gojieba",
		},
	)
	if err != nil {
		panic(err)
	}
	err = indexMapping.AddCustomAnalyzer("gojieba",
		map[string]interface{}{
			"type":      "gojieba",
			"tokenizer": "gojieba",
		},
	)
	if err != nil {
		panic(err)
	}
	indexMapping.DefaultAnalyzer = "gojieba"

	index, err := bleve.New(INDEX_DIR, indexMapping)
	if err != nil {
		panic(err)
	}
	for _, msg := range messages {
		if err := index.Index(msg.Id, msg); err != nil {
			panic(err)
		}
	}

	querys := []string{
		"你好世界",
		"亲口交代",
		"长江",
	}

	for _, q := range querys {
		req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(q))
		req.Highlight = bleve.NewHighlight()
		res, err := index.Search(req)
		if err != nil {
			panic(err)
		}
		fmt.Println(prettify(res))
	}

	// Output:
	// [{"id":"1","score":0.27650412875470115}]
	// [{"id":"2","score":0.27650412875470115}]
	// [{"id":"3","score":0.7027325540540822}]
}
예제 #29
0
func runConfig(conf string, tar string, cpu string) []*Line {
	if cpu != "" {
		f, err := os.Create(cpu)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	start := time.Now()

	wikiReader, err := blevebench.NewWikiReader(*source)
	if err != nil {
		log.Fatal(err)
	}
	defer wikiReader.Close()

	mapping := blevebench.BuildArticleMapping()
	benchConfig := blevebench.LoadConfigFile(conf)

	fmt.Printf("Using Index Type: %s\n", benchConfig.IndexType)
	fmt.Printf("Using KV store: %s\n", benchConfig.KVStore)
	fmt.Printf("Using KV config: %#v\n", benchConfig.KVConfig)
	index, err := bleve.NewUsing(tar, mapping, benchConfig.IndexType, benchConfig.KVStore, benchConfig.KVConfig)
	if err != nil {
		log.Fatal(err)
	}
	_, store, err := index.Advanced()
	if err != nil {
		log.Fatal(err)
	}

	itr := *count / (*level)
	lines := NewLines(itr, len(typename), conf, typename)
	tot := 0
	// print header
	fmt.Printf("elapsed,docs,avg_single_doc_ms,avg_batched_doc_ms,query_water_matches,first_query_water_ms,avg_repeated%d_query_water_ms", *qrepeat)
	printOtherHeader(store)
	fmt.Printf("\n")

	singleCount := 0
	var singleTime time.Duration
	batchCount := 0
	var batchTime time.Duration
	batch := index.NewBatch()
	for i := 1; i < (*count)+1; i++ {

		leveli := i % *level

		a, err := wikiReader.Next()
		if err != nil {
			log.Fatal(err)
		}
		if leveli < *batchSize {
			// index single
			singleStart := time.Now()
			err = index.Index(a.Title, a)
			if err != nil {
				log.Fatalf("error indexing: %v", err)
			}
			duration := time.Since(singleStart)
			singleCount++
			singleTime += duration
		} else {
			// add to batch
			batch.Index(a.Title, a)
			// if batch is full index it
			if batch.Size() == *batchSize {
				batchStart := time.Now()
				err := index.Batch(batch)
				if err != nil {
					log.Fatalf("error executing batch: %v", err)
				}
				duration := time.Since(batchStart)
				batchCount++
				batchTime += duration
				// reset batch
				batch = index.NewBatch()
			}
		}

		if leveli == 0 {

			// run some queries
			termQueryCount := 0
			termQueryStart := time.Now()
			termQuery := bleve.NewTermQuery("water")
			termQuery.SetField("text")
			termSearch := bleve.NewSearchRequest(termQuery)
			searchResults, err := index.Search(termSearch)
			if err != nil {
				log.Fatalf("error searching: %v", err)
			}
			termQueryCount++
			termQueryTime := time.Since(termQueryStart)

			firstQueryTime := float64(termQueryTime)

			for termQueryCount < *qrepeat {
				termQueryStart = time.Now()
				searchResults, err = index.Search(termSearch)
				if err != nil {
					log.Fatal(err)
				}
				termQueryCount++
				termQueryTime += time.Since(termQueryStart)
			}

			// print stats
			avgSingleDocTime := float64(singleTime) / float64(singleCount)
			avgBatchTime := float64(batchTime) / float64(batchCount)
			avgBatchDocTime := float64(avgBatchTime) / float64(*batchSize)
			avgQueryTime := float64(termQueryTime) / float64(termQueryCount)
			elapsedTime := time.Since(start) / time.Millisecond
			fmt.Printf("%d,%d,%f,%f,%d,%f,%f", elapsedTime, i, avgSingleDocTime/float64(time.Millisecond), avgBatchDocTime/float64(time.Millisecond), searchResults.Total, firstQueryTime/float64(time.Millisecond), avgQueryTime/float64(time.Millisecond))
			printOther(store)
			lines[0].Pt[tot].Y = avgSingleDocTime / float64(time.Millisecond)
			lines[0].Pt[tot].X = float64(i)
			lines[1].Pt[tot].Y = avgBatchDocTime / float64(time.Millisecond)
			lines[1].Pt[tot].X = float64(i)
			lines[2].Pt[tot].Y = firstQueryTime / float64(time.Millisecond)
			lines[2].Pt[tot].X = float64(i)
			lines[3].Pt[tot].Y = avgQueryTime / float64(time.Millisecond)
			lines[3].Pt[tot].X = float64(i)
			tot++

			fmt.Printf("\n")

			// reset stats
			singleCount = 0
			singleTime = 0
			batchCount = 0
			batchTime = 0

			// dump mem stats if requested
			if *memprofile != "" {
				f, err := os.Create(strconv.Itoa(i) + "-" + *memprofile)
				if err != nil {
					log.Fatal(err)
				}
				pprof.WriteHeapProfile(f)
			}
		}

	}
	return lines
}
예제 #30
0
func TestBeerSearchAll(t *testing.T) {
	defer os.RemoveAll("beer-search-test.bleve")

	mapping, err := buildIndexMapping()
	if err != nil {
		t.Fatal(err)
	}
	index, err := bleve.New("beer-search-test.bleve", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	for jf := range walkDirectory("data/", t) {
		docId := jf.filename[0:strings.LastIndex(jf.filename, ".")]
		err = index.Index(docId, jf.contents)
		if err != nil {
			t.Error(err)
		}
	}

	expectedCount := uint64(7303)
	actualCount := index.DocCount()
	if actualCount != expectedCount {
		t.Errorf("expected %d documents, got %d", expectedCount, actualCount)
	}

	// run a term search
	termQuery := bleve.NewTermQuery("shock").SetField("name")
	termSearchRequest := bleve.NewSearchRequest(termQuery)
	termSearchResult, err := index.Search(termSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount := uint64(1)
	if termSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total)
	} else {
		expectedResultId := "anheuser_busch-shock_top"
		if termSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID)
		}
	}

	// run a match phrase search
	matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food")
	matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery)
	matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if matchPhraseSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total)
	} else {
		expectedResultId := "great_divide_brewing-wild_raspberry_ale"
		if matchPhraseSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID)
		}
	}

	// run a syntax query
	syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley")
	syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery)
	syntaxSearchResult, err := index.Search(syntaxSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if syntaxSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total)
	} else {
		expectedResultId := "iron_city_brewing_co-ic_light"
		if syntaxSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID)
		}
	}

	// run a numeric range search
	queryMin := 50.0
	numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv")
	numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery)
	numericSearchResult, err := index.Search(numericSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if numericSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total)
	} else {
		expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale"
		if numericSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID)
		}
	}

	// run a date range search
	queryStartDate := "2011-10-04"
	dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated")
	dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery)
	dateSearchResult, err := index.Search(dateSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(2)
	if dateSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total)
	} else {
		expectedResultId := "brasserie_du_bouffay-ambr"
		if dateSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID)
		}
	}

	// run a prefix search
	prefixQuery := bleve.NewPrefixQuery("adir").SetField("name")
	prefixSearchRequest := bleve.NewSearchRequest(prefixQuery)
	prefixSearchResult, err := index.Search(prefixSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if prefixSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total)
	} else {
		expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager"
		if prefixSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID)
		}
	}
}