// Returns data model for index func getIndexMapping() *bv.IndexMapping { indexMapping := bv.NewIndexMapping() messageMapping := bv.NewDocumentStaticMapping() // Will search exact string, e.g.: «hostname.example.org» will search for «hostname.example.org» mappingKeyword := getTextFieldMapping() mappingKeyword.Analyzer = bvKeywordAnalyzer.Name // Tokenized query, e.g.: «hostname example org» will search for «hostname», «example» or «org» mappingText := getTextFieldMapping() mappingText.Analyzer = bvStandardAnalyzer.Name messageMapping.AddFieldMappingsAt("version", mappingKeyword) messageMapping.AddFieldMappingsAt("host", mappingKeyword) messageMapping.AddFieldMappingsAt("short_message", mappingText) messageMapping.AddFieldMappingsAt("full_message", mappingText) messageMapping.AddFieldMappingsAt("timestamp", bv.NewDateTimeFieldMapping()) messageMapping.AddFieldMappingsAt("level", bv.NewNumericFieldMapping()) messageMapping.AddFieldMappingsAt("facility", mappingKeyword) messageMapping.AddSubDocumentMapping("extra", bv.NewDocumentMapping()) indexMapping.AddDocumentMapping(DOC_TYPE, messageMapping) return indexMapping }
// BuildArticleMapping returns a mapping for indexing wikipedia articles // in a manner similar to that done by lucene nightly benchmarks func BuildArticleMapping() *bleve.IndexMapping { // a generic reusable mapping for english text standardJustIndexed := bleve.NewTextFieldMapping() standardJustIndexed.Store = false standardJustIndexed.IncludeInAll = false standardJustIndexed.IncludeTermVectors = false standardJustIndexed.Analyzer = "standard" keywordJustIndexed := bleve.NewTextFieldMapping() keywordJustIndexed.Store = false keywordJustIndexed.IncludeInAll = false keywordJustIndexed.IncludeTermVectors = false keywordJustIndexed.Analyzer = "keyword" articleMapping := bleve.NewDocumentMapping() // title articleMapping.AddFieldMappingsAt("title", keywordJustIndexed) // text articleMapping.AddFieldMappingsAt("text", standardJustIndexed) // _all (disabled) disabledSection := bleve.NewDocumentDisabledMapping() articleMapping.AddSubDocumentMapping("_all", disabledSection) indexMapping := bleve.NewIndexMapping() indexMapping.DefaultMapping = articleMapping indexMapping.DefaultAnalyzer = "standard" return indexMapping }
func main() { // open a new index mapping := bleve.NewIndexMapping() index, err := bleve.New("example.bleve", mapping) if err != nil { fmt.Println(err) return } data := struct { Name string }{ Name: "text", } // index some data index.Index("id", data) // search for some text query := bleve.NewMatchQuery("text") search := bleve.NewSearchRequest(query) searchResults, err := index.Search(search) if err != nil { fmt.Println(err) return } fmt.Println(searchResults) }
func openIndex(path string) (bleve.Index, error) { if _, err := os.Stat(path); os.IsNotExist(err) { mapping := bleve.NewIndexMapping() return bleve.New(path, mapping) } return bleve.Open(path) }
func ValidateMapping(w http.ResponseWriter, req *http.Request) { indexMapping := bleve.NewIndexMapping() // read the request body requestBody, err := ioutil.ReadAll(req.Body) if err != nil { showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400) return } // interpret request body as index mapping if len(requestBody) > 0 { requestBody, err = CleanseJSON(requestBody) if err != nil { showError(w, req, fmt.Sprintf("error preparing index mapping: %v", err), 400) return } err = json.Unmarshal(requestBody, &indexMapping) if err != nil { showError(w, req, fmt.Sprintf("error parsing index mapping: %v", err), 400) return } } rv := struct { Status string `json:"status"` }{ Status: "ok", } mustEncode(w, rv) }
func main() { // Open an index if it already exists. index, err := bleve.Open("example.bleve") if err != nil { mapping := bleve.NewIndexMapping() index, err = bleve.New("example.bleve", mapping) if err != nil { fmt.Println(err) return } } data := struct { Name string }{ Name: "text", } // Index some data. index.Index("id", data) // Search for some text query := bleve.NewMatchQuery("text") search := bleve.NewSearchRequest(query) searchResults, err := index.Search(search) if err != nil { fmt.Println(err) return } fmt.Println(searchResults) }
func init() { // 定义bleve分词 -> gojieba(中文分词) indexMapping = bleve.NewIndexMapping() err := indexMapping.AddCustomTokenizer("gojieba", map[string]interface{}{ "dictpath": gojieba.DICT_PATH, "hmmpath": gojieba.HMM_PATH, "userdictpath": gojieba.USER_DICT_PATH, "type": "gojieba", }, ) if err != nil { fmt.Println(err) panic(err) } err = indexMapping.AddCustomAnalyzer("gojieba", map[string]interface{}{ "type": "gojieba", "tokenizer": "gojieba", }, ) if err != nil { fmt.Println(err) panic(err) } indexMapping.DefaultAnalyzer = "gojieba" }
func buildIndexMapping() *bleve.IndexMapping { enTextFieldMapping := bleve.NewTextFieldMapping() enTextFieldMapping.Analyzer = textFieldAnalyzer storeFieldOnlyMapping := bleve.NewTextFieldMapping() storeFieldOnlyMapping.Index = false storeFieldOnlyMapping.IncludeTermVectors = false storeFieldOnlyMapping.IncludeInAll = false dateTimeMapping := bleve.NewDateTimeFieldMapping() wikiMapping := bleve.NewDocumentMapping() wikiMapping.AddFieldMappingsAt("name", enTextFieldMapping) wikiMapping.AddFieldMappingsAt("body", enTextFieldMapping) wikiMapping.AddFieldMappingsAt("modified_by", enTextFieldMapping) wikiMapping.AddFieldMappingsAt("modified_by_name", enTextFieldMapping) wikiMapping.AddFieldMappingsAt("modified_by_email", enTextFieldMapping) wikiMapping.AddFieldMappingsAt("modified_by_avatar", storeFieldOnlyMapping) wikiMapping.AddFieldMappingsAt("modified", dateTimeMapping) indexMapping := bleve.NewIndexMapping() indexMapping.AddDocumentMapping("wiki", wikiMapping) indexMapping.DefaultAnalyzer = textFieldAnalyzer return indexMapping }
func main() { flag.Parse() if *indexPath == "" { log.Fatal("must specify index path") } // create a new default mapping mapping := bleve.NewIndexMapping() if *mappingFile != "" { mappingBytes, err := ioutil.ReadFile(*mappingFile) if err != nil { log.Fatal(err) } err = json.Unmarshal(mappingBytes, &mapping) if err != nil { log.Fatal(err) } } // create the index index, err := bleve.New(*indexPath, mapping) if err != nil { log.Fatal(err) } defer index.Close() log.Printf("Created bleve index at: %s", *indexPath) }
func NewBleveParams() *BleveParams { return &BleveParams{ Mapping: *bleve.NewIndexMapping(), Store: map[string]interface{}{ "kvStoreName": bleve.Config.DefaultKVStore, }, } }
func init() { bleve.Config.DefaultKVStore = gtreap.Name bleveHttp.SetLog(log.New(os.Stderr, "bleve.http ", log.LstdFlags)) router := mux.NewRouter() router.StrictSlash(true) listIndexesHandler := bleveHttp.NewListIndexesHandler() router.Handle("/api", listIndexesHandler).Methods("GET") docCountHandler := bleveHttp.NewDocCountHandler("") docCountHandler.IndexNameLookup = indexNameLookup router.Handle("/api/{indexName}/_count", docCountHandler).Methods("GET") searchHandler := bleveHttp.NewSearchHandler("") searchHandler.IndexNameLookup = indexNameLookup router.Handle("/api/{indexName}/_search", searchHandler).Methods("POST") http.Handle("/", &CORSWrapper{router}) log.Printf("opening indexes") // walk the data dir and register index names dirEntries, err := ioutil.ReadDir(indexDir) if err != nil { log.Printf("error reading data dir: %v", err) return } for _, dirInfo := range dirEntries { indexPath := indexDir + string(os.PathSeparator) + dirInfo.Name() if dirInfo.IsDir() { log.Printf("see directory %s, this is not supported in the appengine environment", dirInfo.Name()) } else { log.Printf("preloading index export %s", dirInfo.Name()) // assume file in this dir is actually a bleve export i, err := bleve.NewUsing( "", bleve.NewIndexMapping(), bleve.Config.DefaultIndexType, preload.Name, map[string]interface{}{ "kvStoreName_actual": gtreap.Name, "preloadpath": indexPath, }) if err != nil { log.Printf("error preloading index %s: %v", indexPath, err) } else { i.SetName(dirInfo.Name()) log.Printf("registered index: %s", dirInfo.Name()) bleveHttp.RegisterIndexName(dirInfo.Name(), i) } } } }
func NewBleveParams() *BleveParams { rv := &BleveParams{ Mapping: *bleve.NewIndexMapping(), Store: map[string]interface{}{ "kvStoreName": bleve.Config.DefaultKVStore, }, } rv.Mapping.TypeField = "type" return rv }
func NewBleveRecipeStore(path string) RecipeStore { idx, err := bleve.Open(path) if err == bleve.ErrorIndexPathDoesNotExist { mapping := bleve.NewIndexMapping() idx, err = bleve.New(path, mapping) } if err != nil { panic(fmt.Errorf("could not open/create bleve idx at %s: %v", path, err)) } return &BleveRecipeStore{idx: idx} }
func buildIndexMapping() (*bleve.IndexMapping, error) { var err error // Create the index mapping, configure the analyzer, and set as default. indexMapping := bleve.NewIndexMapping() err = indexMapping.AddCustomTokenizer("ekanite_tk", map[string]interface{}{ "regexp": `[^\W_]+`, "type": regexp_tokenizer.Name, }) if err != nil { return nil, err } err = indexMapping.AddCustomAnalyzer("ekanite", map[string]interface{}{ "type": custom_analyzer.Name, "char_filters": []interface{}{}, "tokenizer": `ekanite_tk`, "token_filters": []interface{}{`to_lower`}, }) if err != nil { return nil, err } indexMapping.DefaultAnalyzer = "ekanite" // Create field-specific mappings. simpleJustIndexed := bleve.NewTextFieldMapping() simpleJustIndexed.Store = false simpleJustIndexed.IncludeInAll = true // XXX Move to false when using AST simpleJustIndexed.IncludeTermVectors = false timeJustIndexed := bleve.NewDateTimeFieldMapping() timeJustIndexed.Store = false timeJustIndexed.IncludeInAll = false timeJustIndexed.IncludeTermVectors = false articleMapping := bleve.NewDocumentMapping() // Connect field mappings to fields. articleMapping.AddFieldMappingsAt("Message", simpleJustIndexed) articleMapping.AddFieldMappingsAt("ReferenceTime", timeJustIndexed) articleMapping.AddFieldMappingsAt("ReceptionTime", timeJustIndexed) // Tell the index about field mappings. indexMapping.DefaultMapping = articleMapping return indexMapping, nil }
func main() { mapping := bleve.NewIndexMapping() // HLMAPPING index, err := bleve.New("people.bleve", mapping) // HLNEW if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} // HLINDEX err = index.Index("m1", person) // HLINDEX if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") }
func buildMapping() (*bleve.IndexMapping, error) { mapping := bleve.NewIndexMapping() if mappingPath != "" { mappingBytes, err := ioutil.ReadFile(mappingPath) if err != nil { return nil, err } err = json.Unmarshal(mappingBytes, &mapping) if err != nil { return nil, err } } return mapping, nil }
func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") }
func Init(allDocs GetAllDocsFn) { var err error idx_path := path.Join(config.Current.DataDir, index_name) index, err = bleve.Open(idx_path) if err == bleve.ErrorIndexPathDoesNotExist { log.Info("Creating new Index") indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = config.Current.DefaultAnalyzer entryMapping := bleve.NewDocumentMapping() textField := bleve.NewTextFieldMapping() entryMapping.AddFieldMappingsAt("Body", textField) entryMapping.AddFieldMappingsAt("Title", textField) tagField := bleve.NewTextFieldMapping() tagField.Analyzer = tag_analyzer entryMapping.AddFieldMappingsAt("Tags", tagField) indexMapping.AddDocumentMapping("entry", entryMapping) index, err = bleve.New(idx_path, indexMapping) if err != nil { log.Fatal(err) } // reindex existing documents indexRebuildingLogged := false for itm := range allDocs() { if !indexRebuildingLogged { indexRebuildingLogged = true log.Info("Start rebuilding Search-Index") } index.Index(itm.Id(), itm.Content()) } if indexRebuildingLogged { log.Info("Finished rebuilding Search-Index") } } else if err == nil { log.Info("Opening existing Index") } else { log.Fatal(err) } }
// Open or create a search index // @return result.Result<bleve.Index, error> func openIndex(repoPath string, ifSuccess func(*git.Repository, bleve.Index) result.Result) result.Result { storage := filepath.Join(repoPath, gc.CommentStorageDir) indexPath := filepath.Join(storage, indexFilePath) return gg.WithRepository(repoPath, func(repo *git.Repository) result.Result { os.Mkdir(storage, 0700) success := func(index interface{}) result.Result { return ifSuccess(repo, index.(bleve.Index)) } return result.NewResult(bleve.Open(indexPath)).Analysis(success, func(err error) result.Result { mapping := bleve.NewIndexMapping() index := result.NewResult(bleve.New(indexPath, mapping)) return index.FlatMap(success) }) }) }
func initIndex(name string) error { indexPath := *dataDir + string(os.PathSeparator) + name if _, err := os.Stat(indexPath); os.IsNotExist(err) { log.Printf("%s does not exist. Creating...", name) index, err := bleve.New(indexPath, bleve.NewIndexMapping()) if err != nil { return err } // close so we can open later index.Close() log.Printf("%s created.", name) } return nil }
func (be *BleveEngine) Index(document *Document) (int64, error) { start := time.Now().UnixNano() / int64(time.Millisecond) var index bleve.Index mapping := bleve.NewIndexMapping() index, err := bleve.New(INDEX, mapping) if err != nil { index, _ = bleve.Open(INDEX) } id := (*document).Id data := (*document).Data index.Index(id, data) index.Close() return time.Now().UnixNano()/int64(time.Millisecond) - start, nil }
func AnalyzerText(w http.ResponseWriter, req *http.Request) { // read the request body requestBody, err := ioutil.ReadAll(req.Body) if err != nil { showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400) return } mapping := bleve.NewIndexMapping() var analyzeRequest = struct { Analyzer string `json:"analyzer"` Text string `json:"text"` Mapping *bleve.IndexMapping `json:"mapping"` }{} requestBody, err = CleanseJSON(requestBody) if err != nil { showError(w, req, fmt.Sprintf("error preparing index mapping: %v", err), 400) return } err = json.Unmarshal(requestBody, &analyzeRequest) if err != nil { showError(w, req, fmt.Sprintf("error parsing index mapping: %v", err), 400) return } if analyzeRequest.Mapping != nil { mapping = analyzeRequest.Mapping } ts, err := mapping.AnalyzeText(analyzeRequest.Analyzer, []byte(analyzeRequest.Text)) if err != nil { showError(w, req, fmt.Sprintf("error analyzing text: %v", err), 400) return } rv := struct { Status string `json:"status"` Text string `json:"text"` TokenStream analysis.TokenStream `json:"token_stream"` }{ Status: "ok", Text: analyzeRequest.Text, TokenStream: ts, } mustEncode(w, rv) }
// NewIndexer prepares a new Indexer given the necessary Parse App credentials. func NewIndexer(webhookKey, masterKey, appID string) (*Indexer, error) { path := "contents.bleve" i, err := bleve.Open(path) if err == bleve.ErrorIndexPathDoesNotExist { im := bleve.NewIndexMapping() i, err = bleve.New(path, im) } else if err != nil { return nil, err } return &Indexer{ index: i, webhookKey: webhookKey, masterKey: masterKey, appID: appID, status: &indexStatusList{}, }, nil }
func buildIndexMapping() (*bleve.IndexMapping, error) { // a generic reusable mapping for english text englishTextFieldMapping := bleve.NewTextFieldMapping() englishTextFieldMapping.Analyzer = "en" // a generic reusable mapping for keyword text keywordFieldMapping := bleve.NewTextFieldMapping() keywordFieldMapping.Analyzer = "keyword" // a specific mapping to index the description fields // detected language descriptionLangFieldMapping := bleve.NewTextFieldMapping() descriptionLangFieldMapping.Name = "descriptionLang" descriptionLangFieldMapping.Analyzer = "detect_lang" descriptionLangFieldMapping.Store = false descriptionLangFieldMapping.IncludeTermVectors = false descriptionLangFieldMapping.IncludeInAll = false beerMapping := bleve.NewDocumentMapping() // name beerMapping.AddFieldMappingsAt("name", englishTextFieldMapping) // description beerMapping.AddFieldMappingsAt("description", englishTextFieldMapping, descriptionLangFieldMapping) beerMapping.AddFieldMappingsAt("type", keywordFieldMapping) beerMapping.AddFieldMappingsAt("style", keywordFieldMapping) beerMapping.AddFieldMappingsAt("category", keywordFieldMapping) breweryMapping := bleve.NewDocumentMapping() breweryMapping.AddFieldMappingsAt("name", englishTextFieldMapping) breweryMapping.AddFieldMappingsAt("description", englishTextFieldMapping) indexMapping := bleve.NewIndexMapping() indexMapping.AddDocumentMapping("beer", beerMapping) indexMapping.AddDocumentMapping("brewery", breweryMapping) indexMapping.TypeField = "type" indexMapping.DefaultAnalyzer = "en" return indexMapping, nil }
func NewSearchIndex(repo content.Repo, config Config, logger webfw.Logger) (SearchIndex, error) { var err error var index bleve.Index si := SearchIndex{} _, err = os.Stat(config.SearchIndex.BlevePath) if err == nil { logger.Infoln("Opening search index " + config.SearchIndex.BlevePath) index, err = bleve.Open(config.SearchIndex.BlevePath) if err != nil { return EmptySearchIndex, errors.New(fmt.Sprintf("Error opening search index: %v\n", err)) } } else if os.IsNotExist(err) { mapping := bleve.NewIndexMapping() docMapping := bleve.NewDocumentMapping() idfieldmapping := bleve.NewTextFieldMapping() idfieldmapping.IncludeInAll = false docMapping.AddFieldMappingsAt("FeedId", idfieldmapping) docMapping.AddFieldMappingsAt("ArticleId", idfieldmapping) mapping.AddDocumentMapping(mapping.DefaultType, docMapping) logger.Infoln("Creating search index " + config.SearchIndex.BlevePath) index, err = bleve.New(config.SearchIndex.BlevePath, mapping) if err != nil { return EmptySearchIndex, errors.New(fmt.Sprintf("Error creating search index: %v\n", err)) } si.newIndex = true } else { return EmptySearchIndex, errors.New( fmt.Sprintf("Error getting stat of '%s': %v\n", config.SearchIndex.BlevePath, err)) } si.logger = logger si.repo = repo si.Index = index si.batchSize = config.SearchIndex.BatchSize return si, nil }
func buildIndexMapping() *bleve.IndexMapping { // a generic reusable mapping for keyword text keywordFieldMapping := bleve.NewTextFieldMapping() keywordFieldMapping.Analyzer = "keyword" torrentMapping := bleve.NewDocumentMapping() torrentMapping.AddFieldMappingsAt("category", keywordFieldMapping) im := bleve.NewIndexMapping() im.AddDocumentMapping("torrent", torrentMapping) im.TypeField = "type" im.DefaultAnalyzer = "standard" im.DefaultType = "torrent" im.DefaultField = "name" return im }
func (h *CreateIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { // find the name of the index to create var indexName string if h.IndexNameLookup != nil { indexName = h.IndexNameLookup(req) } if indexName == "" { showError(w, req, "index name is required", 400) return } indexMapping := bleve.NewIndexMapping() // read the request body requestBody, err := ioutil.ReadAll(req.Body) if err != nil { showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400) return } // interpret request body as index mapping if len(requestBody) > 0 { err := json.Unmarshal(requestBody, &indexMapping) if err != nil { showError(w, req, fmt.Sprintf("error parsing index mapping: %v", err), 400) return } } newIndex, err := bleve.New(h.indexPath(indexName), indexMapping) if err != nil { showError(w, req, fmt.Sprintf("error creating index: %v", err), 500) return } newIndex.SetName(indexName) RegisterIndexName(indexName, newIndex) rv := struct { Status string `json:"status"` }{ Status: "ok", } mustEncode(w, rv) }
func ListAnalyzerNames(w http.ResponseWriter, req *http.Request) { indexMapping := bleve.NewIndexMapping() // read the request body requestBody, err := ioutil.ReadAll(req.Body) if err != nil { showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400) return } // interpret request body as index mapping if len(requestBody) > 0 { requestBody, err = CleanseJSON(requestBody) if err != nil { showError(w, req, fmt.Sprintf("error preparing index mapping: %v", err), 400) return } err = json.Unmarshal(requestBody, &indexMapping) if err != nil { showError(w, req, fmt.Sprintf("error parsing index mapping: %v", err), 400) return } } // built in analyzer names _, analyzerNames := registry.AnalyzerTypesAndInstances() // add custom analyzer names for name := range indexMapping.CustomAnalysis.Analyzers { analyzerNames = append(analyzerNames, name) } sort.Strings(analyzerNames) rv := struct { Status string `json:"status"` Analyzers []string `json:"analyzers"` }{ Status: "ok", Analyzers: analyzerNames, } mustEncode(w, rv) }
// indexBuildings builds the search index as well as a way to look items up by SIS/room number. func (s *Server) indexBuildings() { s.idIndex = make(map[string]*models.Index) dir, err := ioutil.TempDir("", "campus") if err != nil { log.Fatal(err) } file := dir + "/index.bleve" log.Printf("Index file %s", file) mapping := bleve.NewIndexMapping() index, err := bleve.New(file, mapping) if err != nil { log.Fatal(err) } s.index = index for _, b := range s.buildings { idx := &models.Index{ Id: b.SIS, Name: b.Name, Type: "building", Description: b.Description, } index.Index(b.SIS, idx) idx.Item = b.Meta() idx.Image = b.Image s.idIndex[b.SIS] = idx for _, f := range b.Floors { for _, r := range f.Rooms { id := b.SIS + " " + r.Id idx := &models.Index{ Id: id, Name: r.Name, Type: r.Type, } index.Index(id, idx) idx.Item = r r.Floor = f.Name r.SIS = b.SIS s.idIndex[id] = idx } } } }
func (be *BleveEngine) BatchIndex(documents []*Document) (int64, error) { start := time.Now().UnixNano() / int64(time.Millisecond) var index bleve.Index mapping := bleve.NewIndexMapping() index, err := bleve.New(INDEX, mapping) if err != nil { index, _ = bleve.Open(INDEX) } batch := index.NewBatch() for _, document := range documents { batch.Index(document.Id, document.Data) } index.Batch(batch) index.Close() return time.Now().UnixNano()/int64(time.Millisecond) - start, nil }