func search(cfg *Config) error {
store, err := OpenStore(cfg.Store())
if err != nil {
return err
}
index, err := bleve.Open(cfg.Index())
if err != nil {
return err
}
defer index.Close()
q, err := makeSearchQuery(*searchQuery, nil)
if err != nil {
return err
}
rq := bleve.NewSearchRequest(q)
rq.Size = 100
ids := []string{}
for {
res, err := index.Search(rq)
if err != nil {
return err
}
for _, doc := range res.Hits {
ids = append(ids, doc.ID)
}
if len(res.Hits) < rq.Size {
break
}
rq.From += rq.Size
}
return printOffers(store, ids)
}
func OpenBlevePIndexImpl(indexType, path string,
restart func()) (cbgt.PIndexImpl, cbgt.Dest, error) {
buf, err := ioutil.ReadFile(path +
string(os.PathSeparator) + "PINDEX_BLEVE_META")
if err != nil {
return nil, nil, err
}
bleveParams := NewBleveParams()
err = json.Unmarshal(buf, bleveParams)
if err != nil {
return nil, nil, fmt.Errorf("bleve: parse params: %v", err)
}
// TODO: boltdb sometimes locks on Open(), so need to investigate,
// where perhaps there was a previous missing or race-y Close().
bindex, err := bleve.Open(path)
if err != nil {
return nil, nil, err
}
return bindex, &cbgt.DestForwarder{
DestProvider: NewBleveDest(path, bindex, restart),
}, nil
}
func openIndex(path string) (bleve.Index, error) {
if _, err := os.Stat(path); os.IsNotExist(err) {
mapping := bleve.NewIndexMapping()
return bleve.New(path, mapping)
}
return bleve.Open(path)
}
func main() {
// Open an index if it already exists.
index, err := bleve.Open("example.bleve")
if err != nil {
mapping := bleve.NewIndexMapping()
index, err = bleve.New("example.bleve", mapping)
if err != nil {
fmt.Println(err)
return
}
}
data := struct {
Name string
}{
Name: "text",
}
// Index some data.
index.Index("id", data)
// Search for some text
query := bleve.NewMatchQuery("text")
search := bleve.NewSearchRequest(query)
searchResults, err := index.Search(search)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(searchResults)
}
func OpenIndex() {
index, err := bleve.Open(IndexDBDir)
LimeIndex = index
if err != nil {
fmt.Println(err)
return
}
}
func (be *BleveEngine) Delete() error {
index, err := bleve.Open(INDEX)
if err != nil {
return err
}
index.Close()
os.RemoveAll(INDEX)
return nil
}
func main() {
flag.Parse()
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
err = pprof.StartCPUProfile(f)
if err != nil {
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
if *indexPath == "" {
log.Fatal("Specify index to query")
}
if flag.NArg() < 1 {
log.Fatal("Specify search query")
}
// open index
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}
defer func() {
cerr := index.Close()
if cerr != nil {
log.Fatalf("error closing index: %v", err)
}
}()
for i := 0; i < *repeat; i++ {
// build a search with the provided parameters
queryString := strings.Join(flag.Args(), " ")
query := bleve.NewQueryStringQuery(queryString)
searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain)
// enable highlights if requested
if *includeHighlights {
searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi")
}
// execute the search
searchResult, err := index.Search(searchRequest)
if err != nil {
log.Fatalf("search error: %v", err)
}
fmt.Println(searchResult)
}
}
func (be *BleveEngine) Search(query string) (interface{}, error) {
index, _ := bleve.Open(INDEX)
bleveQuery := bleve.NewQueryStringQuery(query)
searchRequest := bleve.NewSearchRequest(bleveQuery)
searchResults, err := index.Search(searchRequest)
if err != nil {
return nil, err
}
return searchResults, nil
}
func NewBleveRecipeStore(path string) RecipeStore {
idx, err := bleve.Open(path)
if err == bleve.ErrorIndexPathDoesNotExist {
mapping := bleve.NewIndexMapping()
idx, err = bleve.New(path, mapping)
}
if err != nil {
panic(fmt.Errorf("could not open/create bleve idx at %s: %v", path, err))
}
return &BleveRecipeStore{idx: idx}
}
func New(path, language string) (*Index, error) {
index, err := bleve.Open(path)
if err == bleve.ErrorIndexPathDoesNotExist {
indexMapping := buildIndexMapping(language)
index, err = bleve.New(path, indexMapping)
if err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return &Index{index: index}, nil
}
func Init(allDocs GetAllDocsFn) {
var err error
idx_path := path.Join(config.Current.DataDir, index_name)
index, err = bleve.Open(idx_path)
if err == bleve.ErrorIndexPathDoesNotExist {
log.Info("Creating new Index")
indexMapping := bleve.NewIndexMapping()
indexMapping.DefaultAnalyzer = config.Current.DefaultAnalyzer
entryMapping := bleve.NewDocumentMapping()
textField := bleve.NewTextFieldMapping()
entryMapping.AddFieldMappingsAt("Body", textField)
entryMapping.AddFieldMappingsAt("Title", textField)
tagField := bleve.NewTextFieldMapping()
tagField.Analyzer = tag_analyzer
entryMapping.AddFieldMappingsAt("Tags", tagField)
indexMapping.AddDocumentMapping("entry", entryMapping)
index, err = bleve.New(idx_path, indexMapping)
if err != nil {
log.Fatal(err)
}
// reindex existing documents
indexRebuildingLogged := false
for itm := range allDocs() {
if !indexRebuildingLogged {
indexRebuildingLogged = true
log.Info("Start rebuilding Search-Index")
}
index.Index(itm.Id(), itm.Content())
}
if indexRebuildingLogged {
log.Info("Finished rebuilding Search-Index")
}
} else if err == nil {
log.Info("Opening existing Index")
} else {
log.Fatal(err)
}
}
// START OMIT
func main() {
index, err := bleve.Open("people.bleve") // HLOPEN
if err != nil {
log.Fatal(err)
}
query := bleve.NewTermQuery("marty") // HLQUERY
request := bleve.NewSearchRequest(query) // HLREQ
result, err := index.Search(request) // HLSEARCH
if err != nil {
log.Fatal(err)
}
fmt.Println(result)
}
func debugQueryFn(cfg *Config) error {
index, err := bleve.Open(cfg.Index())
if err != nil {
return err
}
defer index.Close()
q, err := makeSearchQuery(*debugQueryQuery, nil)
if err != nil {
return err
}
s, err := query.DumpQuery(index.Mapping(), q)
fmt.Println(s)
return err
}
func main() {
flag.Parse()
// open the index
beerIndex, err := bleve.Open(*indexPath)
if err == bleve.ErrorIndexPathDoesNotExist {
log.Printf("Creating new index...")
// create a mapping
indexMapping, err := buildIndexMapping()
if err != nil {
log.Fatal(err)
}
beerIndex, err = bleve.New(*indexPath, indexMapping)
if err != nil {
log.Fatal(err)
}
// index data in the background
go func() {
err = indexBeer(beerIndex)
if err != nil {
log.Fatal(err)
}
}()
} else if err != nil {
log.Fatal(err)
} else {
log.Printf("Opening existing index...")
}
// create a router to serve static files
router := staticFileRouter()
// add the API
bleveHttp.RegisterIndexName("beer", beerIndex)
searchHandler := bleveHttp.NewSearchHandler("beer")
router.Handle("/api/search", searchHandler).Methods("POST")
listFieldsHandler := bleveHttp.NewListFieldsHandler("beer")
router.Handle("/api/fields", listFieldsHandler).Methods("GET")
debugHandler := bleveHttp.NewDebugDocumentHandler("beer")
router.Handle("/api/debug/{docID}", debugHandler).Methods("GET")
// start the HTTP server
http.Handle("/", router)
log.Printf("Listening on %v", *bindAddr)
log.Fatal(http.ListenAndServe(*bindAddr, nil))
}
// Open or create a search index
// @return result.Result<bleve.Index, error>
func openIndex(repoPath string, ifSuccess func(*git.Repository, bleve.Index) result.Result) result.Result {
storage := filepath.Join(repoPath, gc.CommentStorageDir)
indexPath := filepath.Join(storage, indexFilePath)
return gg.WithRepository(repoPath, func(repo *git.Repository) result.Result {
os.Mkdir(storage, 0700)
success := func(index interface{}) result.Result {
return ifSuccess(repo, index.(bleve.Index))
}
return result.NewResult(bleve.Open(indexPath)).Analysis(success, func(err error) result.Result {
mapping := bleve.NewIndexMapping()
index := result.NewResult(bleve.New(indexPath, mapping))
return index.FlatMap(success)
})
})
}
func (be *BleveEngine) Index(document *Document) (int64, error) {
start := time.Now().UnixNano() / int64(time.Millisecond)
var index bleve.Index
mapping := bleve.NewIndexMapping()
index, err := bleve.New(INDEX, mapping)
if err != nil {
index, _ = bleve.Open(INDEX)
}
id := (*document).Id
data := (*document).Data
index.Index(id, data)
index.Close()
return time.Now().UnixNano()/int64(time.Millisecond) - start, nil
}
// NewIndexer prepares a new Indexer given the necessary Parse App credentials.
func NewIndexer(webhookKey, masterKey, appID string) (*Indexer, error) {
path := "contents.bleve"
i, err := bleve.Open(path)
if err == bleve.ErrorIndexPathDoesNotExist {
im := bleve.NewIndexMapping()
i, err = bleve.New(path, im)
} else if err != nil {
return nil, err
}
return &Indexer{
index: i,
webhookKey: webhookKey,
masterKey: masterKey,
appID: appID,
status: &indexStatusList{},
}, nil
}
func OpenBleveCache(dbPath string) (LocalCache, error) {
index, err := bleve.Open(dbPath)
if err == bleve.ErrorIndexPathDoesNotExist {
indexMapping := buildIndexMapping()
index, err = bleve.New(dbPath, indexMapping)
if err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return &BleveCache{
dbPath: dbPath,
index: index,
}, nil
}
func main() {
flag.Parse()
if *indexPath == "" {
log.Fatal("specify index to dump")
}
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}
defer func() {
cerr := index.Close()
if cerr != nil {
log.Fatalf("error closing index: %v", err)
}
}()
if *mappingOnly {
mapping := index.Mapping()
jsonBytes, err := json.MarshalIndent(mapping, "", " ")
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", jsonBytes)
return
}
var dumpChan chan interface{}
if *docID != "" {
dumpChan = index.DumpDoc(*docID)
} else if *fieldsOnly {
dumpChan = index.DumpFields()
} else {
dumpChan = index.DumpAll()
}
for rowOrErr := range dumpChan {
switch rowOrErr := rowOrErr.(type) {
case error:
log.Printf("error dumping: %v", rowOrErr)
case upside_down.UpsideDownCouchRow:
fmt.Printf("%v\n", rowOrErr)
fmt.Printf("Key: % -100x\nValue: % -100x\n\n", rowOrErr.Key(), rowOrErr.Value())
}
}
}
func openIndex(path string) bleve.Index {
index, err := bleve.Open(path)
if err == bleve.ErrorIndexPathDoesNotExist {
log.Printf("Creating new index...")
// create a mapping
indexMapping := buildIndexMapping()
index, err = bleve.New(path, indexMapping)
if err != nil {
log.Fatal(err)
}
} else if err == nil {
log.Printf("Opening existing index...")
} else {
log.Fatal(err)
}
return index
}
func NewSearchIndex(repo content.Repo, config Config, logger webfw.Logger) (SearchIndex, error) {
var err error
var index bleve.Index
si := SearchIndex{}
_, err = os.Stat(config.SearchIndex.BlevePath)
if err == nil {
logger.Infoln("Opening search index " + config.SearchIndex.BlevePath)
index, err = bleve.Open(config.SearchIndex.BlevePath)
if err != nil {
return EmptySearchIndex, errors.New(fmt.Sprintf("Error opening search index: %v\n", err))
}
} else if os.IsNotExist(err) {
mapping := bleve.NewIndexMapping()
docMapping := bleve.NewDocumentMapping()
idfieldmapping := bleve.NewTextFieldMapping()
idfieldmapping.IncludeInAll = false
docMapping.AddFieldMappingsAt("FeedId", idfieldmapping)
docMapping.AddFieldMappingsAt("ArticleId", idfieldmapping)
mapping.AddDocumentMapping(mapping.DefaultType, docMapping)
logger.Infoln("Creating search index " + config.SearchIndex.BlevePath)
index, err = bleve.New(config.SearchIndex.BlevePath, mapping)
if err != nil {
return EmptySearchIndex, errors.New(fmt.Sprintf("Error creating search index: %v\n", err))
}
si.newIndex = true
} else {
return EmptySearchIndex, errors.New(
fmt.Sprintf("Error getting stat of '%s': %v\n", config.SearchIndex.BlevePath, err))
}
si.logger = logger
si.repo = repo
si.Index = index
si.batchSize = config.SearchIndex.BatchSize
return si, nil
}
// START OMIT
func main() {
index, err := bleve.Open("gophercon.bleve")
if err != nil {
log.Fatal(err)
}
q := bleve.NewMatchAllQuery()
req := bleve.NewSearchRequest(q)
req.Size = 0
req.AddFacet("sections", // HL
bleve.NewFacetRequest("section", 50)) // HL
res, err := index.Search(req)
if err != nil {
log.Fatal(err)
}
fmt.Println(res)
}
func createOpenIndex(path string) (bleve.Index, error) {
index, err := bleve.Open(path)
if err == bleve.ErrorIndexPathDoesNotExist {
jww.INFO.Println("Creating Index: ", path)
indexMapping, err := buildIndexMapping()
if err != nil {
return nil, err
}
index, err = bleve.NewUsing(path, indexMapping, bleve.Config.DefaultIndexType, goleveldb.Name, nil)
if err != nil {
return nil, err
}
} else if err != nil {
return nil, err
} else {
jww.INFO.Println("Opening Index: ", path)
}
return index, nil
}
func (be *BleveEngine) BatchIndex(documents []*Document) (int64, error) {
start := time.Now().UnixNano() / int64(time.Millisecond)
var index bleve.Index
mapping := bleve.NewIndexMapping()
index, err := bleve.New(INDEX, mapping)
if err != nil {
index, _ = bleve.Open(INDEX)
}
batch := index.NewBatch()
for _, document := range documents {
batch.Index(document.Id, document.Data)
}
index.Batch(batch)
index.Close()
return time.Now().UnixNano()/int64(time.Millisecond) - start, nil
}
// switchIndex returns the error if a problem happens swaping the index
func switchIndex() error {
var (
curName string
nextName string
)
curName = index.Name()
if len(curName) == 0 {
return fmt.Errorf("No index defined")
}
indexList := strings.Split(bleveNames, ":")
if len(indexList) > 1 {
// Find the name of the next index
for i, iName := range indexList {
if strings.Compare(iName, curName) == 0 {
i++
// Wrap to the beginning if we go off end of list
if i >= len(indexList) {
i = 0
}
nextName = indexList[i]
}
}
log.Printf("Opening index %q", nextName)
indexNext, err := bleve.Open(nextName)
if err != nil {
fmt.Printf("Can't open Bleve index %q, %s, aborting swap", nextName, err)
} else {
log.Printf("Switching from %q to %q", curName, nextName)
indexAlias.Swap([]bleve.Index{indexNext}, []bleve.Index{index})
log.Printf("Removing %q", index.Name())
indexAlias.Remove(index)
log.Printf("Closing %q", curName)
index.Close()
// Point index at indexNext
index = indexNext
log.Printf("Swap complete, index now %q", index.Name())
}
return nil
}
return fmt.Errorf("Only %q index defined, no swap possible", curName)
}
func main() {
flag.Parse()
if *indexPath == "" {
log.Fatal("must specify index path")
}
// open the index
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}
defer func() {
cerr := index.Close()
if cerr != nil {
log.Fatalf("error closing index: %v", err)
}
}()
if flag.NArg() < 1 {
log.Fatal("must specify at least one path to index")
}
for file := range handleArgs(flag.Args()) {
// index the files
docID := file.filename
if !*keepDir {
_, docID = filepath.Split(docID)
}
if !*keepExt {
ext := filepath.Ext(docID)
docID = docID[0 : len(docID)-len(ext)]
}
log.Printf("Indexing: %s", docID)
err = index.Index(docID, file.contents)
if err != nil {
log.Fatal(err)
}
}
}
// Opens the shard. If no data exists at the shard's path, an empty shard
// will be created.
func (s *Shard) Open() error {
_, err := os.Stat(s.path)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to check existence of shard")
} else if os.IsNotExist(err) {
mapping, err := buildIndexMapping()
if err != nil {
return err
}
s.b, err = bleve.New(s.path, mapping)
if err != nil {
return fmt.Errorf("bleve new: %s", err.Error())
}
} else {
s.b, err = bleve.Open(s.path)
if err != nil {
return fmt.Errorf("bleve open: %s", err.Error())
}
}
return nil
}
// Bleve connect or create the index persistence
func Bleve(indexPath string) (bleve.Index, error) {
// with bleveIdx isn't set...
if bleveIdx == nil {
var err error
// try to open de persistence file...
bleveIdx, err = bleve.Open(indexPath)
// if doesn't exists or something goes wrong...
if err != nil {
// create a new mapping file and create a new index
mapping := bleve.NewIndexMapping()
bleveIdx, err = bleve.New(indexPath, mapping)
if err != nil {
return nil, err
}
}
}
// return the index
return bleveIdx, nil
}
func openIndex(name string) (bleve.Index, error) {
textFieldMapping := bleve.NewTextFieldMapping()
doc := bleve.NewDocumentMapping()
doc.AddFieldMappingsAt("name", textFieldMapping)
doc.AddFieldMappingsAt("body", textFieldMapping)
doc.AddFieldMappingsAt("modied", textFieldMapping)
indexMap := bleve.NewIndexMapping()
indexMap.AddDocumentMapping("document", doc)
blv, err := bleve.New(name, indexMap)
if err != nil {
blv, err = bleve.Open(name)
if err != nil {
return nil, err
}
}
return blv, nil
}
// START OMIT
func main() {
index, err := bleve.Open("gophercon.bleve")
if err != nil {
log.Fatal(err)
}
qString := `+content:debugger ` // HLQUERY
qString += `title:"delve" ` // HLQUERY
qString += `title:go~2 ` // HLQUERY
qString += `-content:rust ` // HLQUERY
qString += `word_count:>30` // HLQUERY
q := bleve.NewQueryStringQuery(qString)
req := bleve.NewSearchRequest(q)
req.Highlight = bleve.NewHighlightWithStyle("html")
req.Fields = []string{"title", "author", "content", "word_count"}
res, err := index.Search(req)
if err != nil {
log.Fatal(err)
}
fmt.Println(res)
}
