Exemple #1
0
func openIndex(path string) (bleve.Index, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		mapping := bleve.NewIndexMapping()
		return bleve.New(path, mapping)
	}
	return bleve.Open(path)
}
Exemple #2
0
func main() {

	flag.Parse()

	if *indexPath == "" {
		log.Fatal("must specify index path")
	}

	// create a new default mapping
	mapping := bleve.NewIndexMapping()
	if *mappingFile != "" {
		mappingBytes, err := ioutil.ReadFile(*mappingFile)
		if err != nil {
			log.Fatal(err)
		}
		err = json.Unmarshal(mappingBytes, &mapping)
		if err != nil {
			log.Fatal(err)
		}
	}

	// create the index
	index, err := bleve.New(*indexPath, mapping)
	if err != nil {
		log.Fatal(err)
	}
	defer index.Close()

	log.Printf("Created bleve index at: %s", *indexPath)
}
Exemple #3
0
func main() {
	// Open an index if it already exists.
	index, err := bleve.Open("example.bleve")
	if err != nil {
		mapping := bleve.NewIndexMapping()
		index, err = bleve.New("example.bleve", mapping)
		if err != nil {
			fmt.Println(err)
			return
		}
	}

	data := struct {
		Name string
	}{
		Name: "text",
	}

	// Index some data.
	index.Index("id", data)

	// Search for some text
	query := bleve.NewMatchQuery("text")
	search := bleve.NewSearchRequest(query)
	searchResults, err := index.Search(search)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println(searchResults)
}
Exemple #4
0
func main() {
	// open a new index
	mapping := bleve.NewIndexMapping()
	index, err := bleve.New("example.bleve", mapping)
	if err != nil {
		fmt.Println(err)
		return
	}

	data := struct {
		Name string
	}{
		Name: "text",
	}

	// index some data
	index.Index("id", data)

	// search for some text
	query := bleve.NewMatchQuery("text")
	search := bleve.NewSearchRequest(query)
	searchResults, err := index.Search(search)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(searchResults)
}
Exemple #5
0
func NewBleveRecipeStore(path string) RecipeStore {
	idx, err := bleve.Open(path)
	if err == bleve.ErrorIndexPathDoesNotExist {
		mapping := bleve.NewIndexMapping()
		idx, err = bleve.New(path, mapping)
	}
	if err != nil {
		panic(fmt.Errorf("could not open/create bleve idx at %s: %v", path, err))
	}

	return &BleveRecipeStore{idx: idx}
}
Exemple #6
0
func Init(allDocs GetAllDocsFn) {
	var err error

	idx_path := path.Join(config.Current.DataDir, index_name)

	index, err = bleve.Open(idx_path)
	if err == bleve.ErrorIndexPathDoesNotExist {
		log.Info("Creating new Index")
		indexMapping := bleve.NewIndexMapping()
		indexMapping.DefaultAnalyzer = config.Current.DefaultAnalyzer

		entryMapping := bleve.NewDocumentMapping()

		textField := bleve.NewTextFieldMapping()

		entryMapping.AddFieldMappingsAt("Body", textField)
		entryMapping.AddFieldMappingsAt("Title", textField)

		tagField := bleve.NewTextFieldMapping()
		tagField.Analyzer = tag_analyzer

		entryMapping.AddFieldMappingsAt("Tags", tagField)

		indexMapping.AddDocumentMapping("entry", entryMapping)

		index, err = bleve.New(idx_path, indexMapping)
		if err != nil {
			log.Fatal(err)
		}

		// reindex existing documents
		indexRebuildingLogged := false
		for itm := range allDocs() {
			if !indexRebuildingLogged {
				indexRebuildingLogged = true
				log.Info("Start rebuilding Search-Index")
			}

			index.Index(itm.Id(), itm.Content())
		}
		if indexRebuildingLogged {
			log.Info("Finished rebuilding Search-Index")
		}

	} else if err == nil {
		log.Info("Opening existing Index")
	} else {
		log.Fatal(err)
	}
}
Exemple #7
0
func main() {
	mapping := bleve.NewIndexMapping()
	index, err := bleve.New("people.bleve", mapping)
	if err != nil {
		log.Fatal(err)
	}

	person := Person{"Marty Schoch"}
	err = index.Index("m1", person)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Indexed Document")
}
Exemple #8
0
func main() {
	mapping := bleve.NewIndexMapping()               // HLMAPPING
	index, err := bleve.New("people.bleve", mapping) // HLNEW
	if err != nil {
		log.Fatal(err)
	}

	person := Person{"Marty Schoch"} // HLINDEX
	err = index.Index("m1", person)  // HLINDEX
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Indexed Document")
}
Exemple #9
0
func New(path, language string) (*Index, error) {
	index, err := bleve.Open(path)
	if err == bleve.ErrorIndexPathDoesNotExist {
		indexMapping := buildIndexMapping(language)
		index, err = bleve.New(path, indexMapping)
		if err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}

	return &Index{index: index}, nil
}
Exemple #10
0
func main() {

	flag.Parse()

	// open the index
	beerIndex, err := bleve.Open(*indexPath)
	if err == bleve.ErrorIndexPathDoesNotExist {
		log.Printf("Creating new index...")
		// create a mapping
		indexMapping, err := buildIndexMapping()
		if err != nil {
			log.Fatal(err)
		}
		beerIndex, err = bleve.New(*indexPath, indexMapping)
		if err != nil {
			log.Fatal(err)
		}

		// index data in the background
		go func() {
			err = indexBeer(beerIndex)
			if err != nil {
				log.Fatal(err)
			}
		}()
	} else if err != nil {
		log.Fatal(err)
	} else {
		log.Printf("Opening existing index...")
	}

	// create a router to serve static files
	router := staticFileRouter()

	// add the API
	bleveHttp.RegisterIndexName("beer", beerIndex)
	searchHandler := bleveHttp.NewSearchHandler("beer")
	router.Handle("/api/search", searchHandler).Methods("POST")
	listFieldsHandler := bleveHttp.NewListFieldsHandler("beer")
	router.Handle("/api/fields", listFieldsHandler).Methods("GET")
	debugHandler := bleveHttp.NewDebugDocumentHandler("beer")
	router.Handle("/api/debug/{docID}", debugHandler).Methods("GET")

	// start the HTTP server
	http.Handle("/", router)
	log.Printf("Listening on %v", *bindAddr)
	log.Fatal(http.ListenAndServe(*bindAddr, nil))

}
Exemple #11
0
// this test reproduces bug #87
// https://github.com/blevesearch/bleve/issues/87
// because of which, it will deadlock
func TestBeerSearchBug87(t *testing.T) {
	defer os.RemoveAll("beer-search-test.bleve")

	mapping, err := buildIndexMapping()
	if err != nil {
		t.Fatal(err)
	}
	index, err := bleve.New("beer-search-test.bleve", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	// start indexing documents in the background
	go func() {
		// open the directory
		dirEntries, err := ioutil.ReadDir("data/")
		if err != nil {
			t.Fatal(err)
		}

		for _, dirEntry := range dirEntries {
			filename := dirEntry.Name()
			jsonBytes, err := ioutil.ReadFile("data/" + filename)
			if err != nil {
				t.Fatal(err)
			}
			ext := filepath.Ext(filename)
			docId := filename[:(len(filename) - len(ext))]
			index.Index(docId, jsonBytes)
		}
	}()

	// give indexing a head start
	time.Sleep(1 * time.Second)

	// start querying
	for i := 0; i < 1000; i++ {
		time.Sleep(1 * time.Millisecond)
		termQuery := bleve.NewTermQuery("shock").SetField("name")
		termSearchRequest := bleve.NewSearchRequest(termQuery)
		// termSearchRequest.AddFacet("styles", bleve.NewFacetRequest("style", 3))
		termSearchRequest.Fields = []string{"abv"}
		_, err := index.Search(termSearchRequest)
		if err != nil {
			t.Error(err)
		}
	}
}
Exemple #12
0
// Open or create a search index
// @return result.Result<bleve.Index, error>
func openIndex(repoPath string, ifSuccess func(*git.Repository, bleve.Index) result.Result) result.Result {
	storage := filepath.Join(repoPath, gc.CommentStorageDir)
	indexPath := filepath.Join(storage, indexFilePath)
	return gg.WithRepository(repoPath, func(repo *git.Repository) result.Result {
		os.Mkdir(storage, 0700)
		success := func(index interface{}) result.Result {
			return ifSuccess(repo, index.(bleve.Index))
		}
		return result.NewResult(bleve.Open(indexPath)).Analysis(success, func(err error) result.Result {
			mapping := bleve.NewIndexMapping()
			index := result.NewResult(bleve.New(indexPath, mapping))
			return index.FlatMap(success)
		})
	})
}
Exemple #13
0
func initIndex(name string) error {
	indexPath := *dataDir + string(os.PathSeparator) + name
	if _, err := os.Stat(indexPath); os.IsNotExist(err) {
		log.Printf("%s does not exist. Creating...", name)
		index, err := bleve.New(indexPath, bleve.NewIndexMapping())
		if err != nil {
			return err
		}
		// close so we can open later
		index.Close()
		log.Printf("%s created.", name)
	}

	return nil
}
Exemple #14
0
func (be *BleveEngine) Index(document *Document) (int64, error) {
	start := time.Now().UnixNano() / int64(time.Millisecond)

	var index bleve.Index
	mapping := bleve.NewIndexMapping()
	index, err := bleve.New(INDEX, mapping)
	if err != nil {
		index, _ = bleve.Open(INDEX)
	}
	id := (*document).Id
	data := (*document).Data
	index.Index(id, data)
	index.Close()

	return time.Now().UnixNano()/int64(time.Millisecond) - start, nil
}
Exemple #15
0
// NewIndexer prepares a new Indexer given the necessary Parse App credentials.
func NewIndexer(webhookKey, masterKey, appID string) (*Indexer, error) {
	path := "contents.bleve"
	i, err := bleve.Open(path)
	if err == bleve.ErrorIndexPathDoesNotExist {
		im := bleve.NewIndexMapping()
		i, err = bleve.New(path, im)
	} else if err != nil {
		return nil, err
	}
	return &Indexer{
		index:      i,
		webhookKey: webhookKey,
		masterKey:  masterKey,
		appID:      appID,
		status:     &indexStatusList{},
	}, nil
}
Exemple #16
0
func OpenBleveCache(dbPath string) (LocalCache, error) {
	index, err := bleve.Open(dbPath)
	if err == bleve.ErrorIndexPathDoesNotExist {
		indexMapping := buildIndexMapping()
		index, err = bleve.New(dbPath, indexMapping)
		if err != nil {
			return nil, err
		}

	} else if err != nil {
		return nil, err
	}
	return &BleveCache{
		dbPath: dbPath,
		index:  index,
	}, nil
}
func openIndex(path string) bleve.Index {
	index, err := bleve.Open(path)
	if err == bleve.ErrorIndexPathDoesNotExist {
		log.Printf("Creating new index...")
		// create a mapping
		indexMapping := buildIndexMapping()
		index, err = bleve.New(path, indexMapping)
		if err != nil {
			log.Fatal(err)
		}
	} else if err == nil {
		log.Printf("Opening existing index...")
	} else {
		log.Fatal(err)
	}
	return index
}
Exemple #18
0
func NewSearchIndex(repo content.Repo, config Config, logger webfw.Logger) (SearchIndex, error) {
	var err error
	var index bleve.Index

	si := SearchIndex{}

	_, err = os.Stat(config.SearchIndex.BlevePath)
	if err == nil {
		logger.Infoln("Opening search index " + config.SearchIndex.BlevePath)
		index, err = bleve.Open(config.SearchIndex.BlevePath)

		if err != nil {
			return EmptySearchIndex, errors.New(fmt.Sprintf("Error opening search index: %v\n", err))
		}
	} else if os.IsNotExist(err) {
		mapping := bleve.NewIndexMapping()
		docMapping := bleve.NewDocumentMapping()

		idfieldmapping := bleve.NewTextFieldMapping()
		idfieldmapping.IncludeInAll = false
		docMapping.AddFieldMappingsAt("FeedId", idfieldmapping)
		docMapping.AddFieldMappingsAt("ArticleId", idfieldmapping)

		mapping.AddDocumentMapping(mapping.DefaultType, docMapping)

		logger.Infoln("Creating search index " + config.SearchIndex.BlevePath)
		index, err = bleve.New(config.SearchIndex.BlevePath, mapping)

		if err != nil {
			return EmptySearchIndex, errors.New(fmt.Sprintf("Error creating search index: %v\n", err))
		}

		si.newIndex = true
	} else {
		return EmptySearchIndex, errors.New(
			fmt.Sprintf("Error getting stat of '%s': %v\n", config.SearchIndex.BlevePath, err))
	}

	si.logger = logger
	si.repo = repo
	si.Index = index
	si.batchSize = config.SearchIndex.BatchSize

	return si, nil
}
Exemple #19
0
func (h *CreateIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// find the name of the index to create
	var indexName string
	if h.IndexNameLookup != nil {
		indexName = h.IndexNameLookup(req)
	}
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	indexMapping := bleve.NewIndexMapping()

	// read the request body
	requestBody, err := ioutil.ReadAll(req.Body)
	if err != nil {
		showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400)
		return
	}

	// interpret request body as index mapping
	if len(requestBody) > 0 {
		err := json.Unmarshal(requestBody, &indexMapping)
		if err != nil {
			showError(w, req, fmt.Sprintf("error parsing index mapping: %v", err), 400)
			return
		}
	}

	newIndex, err := bleve.New(h.indexPath(indexName), indexMapping)
	if err != nil {
		showError(w, req, fmt.Sprintf("error creating index: %v", err), 500)
		return
	}
	newIndex.SetName(indexName)
	RegisterIndexName(indexName, newIndex)
	rv := struct {
		Status string `json:"status"`
	}{
		Status: "ok",
	}
	mustEncode(w, rv)
}
Exemple #20
0
// indexBuildings builds the search index as well as a way to look items up by SIS/room number.
func (s *Server) indexBuildings() {
	s.idIndex = make(map[string]*models.Index)
	dir, err := ioutil.TempDir("", "campus")
	if err != nil {
		log.Fatal(err)
	}
	file := dir + "/index.bleve"
	log.Printf("Index file %s", file)
	mapping := bleve.NewIndexMapping()
	index, err := bleve.New(file, mapping)
	if err != nil {
		log.Fatal(err)
	}
	s.index = index
	for _, b := range s.buildings {
		idx := &models.Index{
			Id:          b.SIS,
			Name:        b.Name,
			Type:        "building",
			Description: b.Description,
		}
		index.Index(b.SIS, idx)
		idx.Item = b.Meta()
		idx.Image = b.Image
		s.idIndex[b.SIS] = idx
		for _, f := range b.Floors {
			for _, r := range f.Rooms {
				id := b.SIS + " " + r.Id
				idx := &models.Index{
					Id:   id,
					Name: r.Name,
					Type: r.Type,
				}
				index.Index(id, idx)
				idx.Item = r
				r.Floor = f.Name
				r.SIS = b.SIS
				s.idIndex[id] = idx
			}
		}
	}
}
Exemple #21
0
func (be *BleveEngine) BatchIndex(documents []*Document) (int64, error) {
	start := time.Now().UnixNano() / int64(time.Millisecond)
	var index bleve.Index
	mapping := bleve.NewIndexMapping()
	index, err := bleve.New(INDEX, mapping)
	if err != nil {
		index, _ = bleve.Open(INDEX)
	}

	batch := index.NewBatch()

	for _, document := range documents {
		batch.Index(document.Id, document.Data)
	}

	index.Batch(batch)
	index.Close()

	return time.Now().UnixNano()/int64(time.Millisecond) - start, nil
}
Exemple #22
0
// this test reproduces bug #87
// https://github.com/blevesearch/bleve/issues/87
// because of which, it will deadlock
func TestBeerSearchBug87(t *testing.T) {
	defer os.RemoveAll("beer-search-test.bleve")

	mapping, err := buildIndexMapping()
	if err != nil {
		t.Fatal(err)
	}
	index, err := bleve.New("beer-search-test.bleve", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	var wg sync.WaitGroup
	wg.Add(1)
	// start indexing documents in the background
	go func() {
		for jf := range walkDirectory("data/", t) {
			docId := jf.filename[0:strings.LastIndex(jf.filename, ".")]
			err = index.Index(docId, jf.contents)
			if err != nil {
				t.Error(err)
			}
		}
		wg.Done()
	}()

	for i := 0; i < 50; i++ {
		time.Sleep(1 * time.Second)
		termQuery := bleve.NewTermQuery("shock").SetField("name")
		termSearchRequest := bleve.NewSearchRequest(termQuery)
		// termSearchRequest.AddFacet("styles", bleve.NewFacetRequest("style", 3))
		termSearchRequest.Fields = []string{"abv"}
		_, err := index.Search(termSearchRequest)
		if err != nil {
			t.Error(err)
		}
	}

	wg.Wait()
}
Exemple #23
0
func main() {

	flag.Parse()

	if *indexPath == "" {
		log.Fatal("must specify index path")
	}

	// create a new default mapping
	mapping := bleve.NewIndexMapping()
	if *mappingFile != "" {
		mappingBytes, err := ioutil.ReadFile(*mappingFile)
		if err != nil {
			log.Fatal(err)
		}
		err = json.Unmarshal(mappingBytes, &mapping)
		if err != nil {
			log.Fatal(err)
		}
	}

	// create the index
	var index bleve.Index
	var err error
	if *storeType != "" {
		index, err = bleve.NewUsing(*indexPath, mapping, *storeType, nil)
	} else {
		index, err = bleve.New(*indexPath, mapping)
	}
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		cerr := index.Close()
		if cerr != nil {
			log.Fatalf("error closing index: %v", err)
		}
	}()

	log.Printf("Created bleve index at: %s", *indexPath)
}
Exemple #24
0
// Opens the shard. If no data exists at the shard's path, an empty shard
// will be created.
func (s *Shard) Open() error {
	_, err := os.Stat(s.path)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to check existence of shard")
	} else if os.IsNotExist(err) {
		mapping, err := buildIndexMapping()
		if err != nil {
			return err
		}
		s.b, err = bleve.New(s.path, mapping)
		if err != nil {
			return fmt.Errorf("bleve new: %s", err.Error())
		}
	} else {
		s.b, err = bleve.Open(s.path)
		if err != nil {
			return fmt.Errorf("bleve open: %s", err.Error())
		}
	}
	return nil
}
Exemple #25
0
// Bleve connect or create the index persistence
func Bleve(indexPath string) (bleve.Index, error) {

	// with bleveIdx isn't set...
	if bleveIdx == nil {
		var err error
		// try to open de persistence file...
		bleveIdx, err = bleve.Open(indexPath)
		// if doesn't exists or something goes wrong...
		if err != nil {
			// create a new mapping file and create a new index
			mapping := bleve.NewIndexMapping()
			bleveIdx, err = bleve.New(indexPath, mapping)
			if err != nil {
				return nil, err
			}
		}
	}

	// return the index
	return bleveIdx, nil
}
Exemple #26
0
func openIndex(name string) (bleve.Index, error) {
	textFieldMapping := bleve.NewTextFieldMapping()

	doc := bleve.NewDocumentMapping()
	doc.AddFieldMappingsAt("name", textFieldMapping)
	doc.AddFieldMappingsAt("body", textFieldMapping)
	doc.AddFieldMappingsAt("modied", textFieldMapping)

	indexMap := bleve.NewIndexMapping()
	indexMap.AddDocumentMapping("document", doc)

	blv, err := bleve.New(name, indexMap)

	if err != nil {
		blv, err = bleve.Open(name)
		if err != nil {
			return nil, err
		}
	}

	return blv, nil
}
Exemple #27
0
func (indexes *BleveIndexes) index(namespace string, createIfNotExist bool) (bleve.Index, error) {
	dir := path.Join(indexes.root, namespace)
	fn := path.Join(indexes.root, namespace, indexes.name)

	if _, err := os.Stat(fn); err == nil {
		if idx, err := bleve.Open(fn); err == nil {
			return idx, nil
		}
	}

	if !createIfNotExist {
		return nil, nil
	}

	os.RemoveAll(fn)
	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
		return nil, err
	}

	mapping := bleve.NewIndexMapping()
	idx, err := bleve.New(fn, mapping)
	return idx, err
}
func runTestDir(t *testing.T, dir, datasetName string) {
	// read the mapping
	mappingBytes, err := ioutil.ReadFile(dir + string(filepath.Separator) + "mapping.json")
	if err != nil {
		t.Errorf("error reading mapping: %v", err)
		return
	}
	var mapping bleve.IndexMapping
	err = json.Unmarshal(mappingBytes, &mapping)
	if err != nil {
		t.Errorf("error unmarshalling mapping: %v", err)
		return
	}

	// open new index
	if !*keepIndex {
		defer func() {
			err := os.RemoveAll("test.bleve")
			if err != nil {
				t.Fatal(err)
			}
		}()
	}
	index, err := bleve.New("test.bleve", &mapping)
	if err != nil {
		t.Errorf("error creating new index: %v", err)
		return
	}
	// set a custom index name
	index.SetName(datasetName)
	defer func() {
		err := index.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	// index data
	fis, err := ioutil.ReadDir(dir + string(filepath.Separator) + "data")
	if err != nil {
		t.Errorf("error reading data dir: %v", err)
		return
	}
	for _, fi := range fis {
		fileBytes, err := ioutil.ReadFile(dir + string(filepath.Separator) + "data" + string(filepath.Separator) + fi.Name())
		if err != nil {
			t.Errorf("error reading data file: %v", err)
			return
		}
		filename := fi.Name()
		ext := filepath.Ext(filename)
		id := filename[0 : len(filename)-len(ext)]
		err = index.Index(id, fileBytes)
		if err != nil {
			t.Errorf("error indexing data: %v", err)
			return
		}
	}

	indexInternal, _, err := index.Advanced()
	if err != nil {
		t.Fatal(err)
	}
	if indexInternal, ok := indexInternal.(bleveIndex.AsyncIndex); ok {
		start := time.Now()
		err = indexInternal.Wait(5 * time.Second)
		if err != nil {
			t.Fatal(err)
		}
		t.Logf("we had to wait for %v", time.Since(start))
	}

	// read the searches
	searchBytes, err := ioutil.ReadFile(dir + string(filepath.Separator) + "searches.json")
	if err != nil {
		t.Errorf("error reading searches: %v", err)
		return
	}
	var searches SearchTests
	err = json.Unmarshal(searchBytes, &searches)
	if err != nil {
		t.Errorf("error unmarshalling searches: %v", err)
		return
	}

	// run the searches
	for testNum, search := range searches {
		res, err := index.Search(search.Search)
		if err != nil {
			t.Errorf("error running search: %v", err)
		}
		if res.Total != search.Result.Total {
			t.Errorf("test %d - expected total: %d got %d", testNum, search.Result.Total, res.Total)
			continue
		}
		if len(res.Hits) != len(search.Result.Hits) {
			t.Errorf("test %d - expected hits len: %d got %d", testNum, len(search.Result.Hits), len(res.Hits))
			continue
		}
		for hi, hit := range search.Result.Hits {
			if hit.ID != res.Hits[hi].ID {
				t.Errorf("test %d - expected hit %d to have ID %s got %s", testNum, hi, hit.ID, res.Hits[hi].ID)
			}
			if hit.Fields != nil {
				if !reflect.DeepEqual(hit.Fields, res.Hits[hi].Fields) {
					t.Errorf("test  %d - expected hit %d to have fields %#v got %#v", testNum, hi, hit.Fields, res.Hits[hi].Fields)
				}
			}
			if hit.Fragments != nil {
				if !reflect.DeepEqual(hit.Fragments, res.Hits[hi].Fragments) {
					t.Errorf("test %d - expected hit %d to have fragments %#v got %#v", testNum, hi, hit.Fragments, res.Hits[hi].Fragments)
				}
			}
			if hit.Locations != nil {
				if !reflect.DeepEqual(hit.Locations, res.Hits[hi].Locations) {
					t.Errorf("test %d - expected hit %d to have locations %v got %v", testNum, hi, hit.Locations, res.Hits[hi].Locations)
				}
			}
		}
		if search.Result.Facets != nil {
			if !reflect.DeepEqual(search.Result.Facets, res.Facets) {
				t.Errorf("test %d - expected facets: %#v got %#v", testNum, search.Result.Facets, res.Facets)
			}
		}
		// check that custom index name is in results
		for _, hit := range res.Hits {
			if hit.Index != datasetName {
				t.Fatalf("expected name: %s, got: %s", datasetName, hit.Index)

			}
		}
	}
}
Exemple #29
0
func main() {
	flag.Parse()

	expvars.Set("indexes", bleveHttp.IndexStats())

	// open the index
	tpbIndex, err := bleve.Open(*indexPath)
	if err == bleve.ErrorIndexPathDoesNotExist {
		log.Printf("Creating new index...")

		// create a mapping
		indexMapping := buildIndexMapping()

		// use the mapping
		tpbIndex, err = bleve.New(*indexPath, indexMapping)
		if err != nil {
			log.Fatal(err)
		}

		// index data in the background
		go func() {
			err = indexTPB(tpbIndex)
			if err != nil {
				log.Fatal(err)
			}
		}()
	} else if err != nil {
		log.Fatal(err)
	} else {
		log.Printf("Opening existing index...")
	}

	// create a router to serve static files
	router := staticFileRouter()

	// add the API
	bleveHttp.RegisterIndexName("tpb", tpbIndex)

	listIndexesHandler := bleveHttp.NewListIndexesHandler()
	router.Handle("/api", listIndexesHandler).Methods("GET")

	getIndexHandler := bleveHttp.NewGetIndexHandler()
	getIndexHandler.IndexNameLookup = indexNameLookup
	router.Handle("/api/tpb", getIndexHandler).Methods("GET")

	searchHandler := bleveHttp.NewSearchHandler("tpb")
	router.Handle("/api/search", searchHandler).Methods("POST")

	listFieldsHandler := bleveHttp.NewListFieldsHandler("tpb")
	router.Handle("/api/fields", listFieldsHandler).Methods("GET")

	docCountHandler := bleveHttp.NewDocCountHandler("tpb")
	router.Handle("/api/count", docCountHandler).Methods("GET")

	docGetHandler := bleveHttp.NewDocGetHandler("tpb")
	docGetHandler.DocIDLookup = docIDLookup
	router.Handle("/api/tpb/{docID}", docGetHandler).Methods("GET")

	debugHandler := bleveHttp.NewDebugDocumentHandler("tpb")
	debugHandler.DocIDLookup = docIDLookup
	router.Handle("/api/tpb/{docID}/_debug", debugHandler).Methods("GET")

	aliasHandler := bleveHttp.NewAliasHandler()
	router.Handle("/api/_aliases", aliasHandler).Methods("POST")

	// start the HTTP server
	http.Handle("/", router)

	bind = fmt.Sprintf("%s:%d", *host, *port)

	log.Printf("Listening on http://%v", bind)
	log.Fatal(http.ListenAndServe(bind, nil))
}
Exemple #30
0
func Example() {
	INDEX_DIR := "gojieba.bleve"
	messages := []struct {
		Id   string
		Body string
	}{
		{
			Id:   "1",
			Body: "你好",
		},
		{
			Id:   "2",
			Body: "交代",
		},
		{
			Id:   "3",
			Body: "长江大桥",
		},
	}

	indexMapping := bleve.NewIndexMapping()
	os.RemoveAll(INDEX_DIR)
	// clean index when example finished
	defer os.RemoveAll(INDEX_DIR)

	err := indexMapping.AddCustomTokenizer("gojieba",
		map[string]interface{}{
			"dictpath":     gojieba.DICT_PATH,
			"hmmpath":      gojieba.HMM_PATH,
			"userdictpath": gojieba.USER_DICT_PATH,
			"idf":          gojieba.IDF_PATH,
			"stop_words":   gojieba.STOP_WORDS_PATH,
			"type":         "gojieba",
		},
	)
	if err != nil {
		panic(err)
	}
	err = indexMapping.AddCustomAnalyzer("gojieba",
		map[string]interface{}{
			"type":      "gojieba",
			"tokenizer": "gojieba",
		},
	)
	if err != nil {
		panic(err)
	}
	indexMapping.DefaultAnalyzer = "gojieba"

	index, err := bleve.New(INDEX_DIR, indexMapping)
	if err != nil {
		panic(err)
	}
	for _, msg := range messages {
		if err := index.Index(msg.Id, msg); err != nil {
			panic(err)
		}
	}

	querys := []string{
		"你好世界",
		"亲口交代",
		"长江",
	}

	for _, q := range querys {
		req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(q))
		req.Highlight = bleve.NewHighlight()
		res, err := index.Search(req)
		if err != nil {
			panic(err)
		}
		fmt.Println(prettify(res))
	}

	// Output:
	// [{"id":"1","score":0.27650412875470115}]
	// [{"id":"2","score":0.27650412875470115}]
	// [{"id":"3","score":0.7027325540540822}]
}