// Returns data model for index
func getIndexMapping() *bv.IndexMapping {
indexMapping := bv.NewIndexMapping()
messageMapping := bv.NewDocumentStaticMapping()
// Will search exact string, e.g.: «hostname.example.org» will search for «hostname.example.org»
mappingKeyword := getTextFieldMapping()
mappingKeyword.Analyzer = bvKeywordAnalyzer.Name
// Tokenized query, e.g.: «hostname example org» will search for «hostname», «example» or «org»
mappingText := getTextFieldMapping()
mappingText.Analyzer = bvStandardAnalyzer.Name
messageMapping.AddFieldMappingsAt("version", mappingKeyword)
messageMapping.AddFieldMappingsAt("host", mappingKeyword)
messageMapping.AddFieldMappingsAt("short_message", mappingText)
messageMapping.AddFieldMappingsAt("full_message", mappingText)
messageMapping.AddFieldMappingsAt("timestamp", bv.NewDateTimeFieldMapping())
messageMapping.AddFieldMappingsAt("level", bv.NewNumericFieldMapping())
messageMapping.AddFieldMappingsAt("facility", mappingKeyword)
messageMapping.AddSubDocumentMapping("extra", bv.NewDocumentMapping())
indexMapping.AddDocumentMapping(DOC_TYPE, messageMapping)
return indexMapping
}
func buildIndexMapping() *bleve.IndexMapping {
enTextFieldMapping := bleve.NewTextFieldMapping()
enTextFieldMapping.Analyzer = textFieldAnalyzer
storeFieldOnlyMapping := bleve.NewTextFieldMapping()
storeFieldOnlyMapping.Index = false
storeFieldOnlyMapping.IncludeTermVectors = false
storeFieldOnlyMapping.IncludeInAll = false
dateTimeMapping := bleve.NewDateTimeFieldMapping()
wikiMapping := bleve.NewDocumentMapping()
wikiMapping.AddFieldMappingsAt("name", enTextFieldMapping)
wikiMapping.AddFieldMappingsAt("body", enTextFieldMapping)
wikiMapping.AddFieldMappingsAt("modified_by", enTextFieldMapping)
wikiMapping.AddFieldMappingsAt("modified_by_name", enTextFieldMapping)
wikiMapping.AddFieldMappingsAt("modified_by_email", enTextFieldMapping)
wikiMapping.AddFieldMappingsAt("modified_by_avatar", storeFieldOnlyMapping)
wikiMapping.AddFieldMappingsAt("modified", dateTimeMapping)
indexMapping := bleve.NewIndexMapping()
indexMapping.AddDocumentMapping("wiki", wikiMapping)
indexMapping.DefaultAnalyzer = textFieldAnalyzer
return indexMapping
}
func buildIndexMapping() (*bleve.IndexMapping, error) {
var err error
// Create the index mapping, configure the analyzer, and set as default.
indexMapping := bleve.NewIndexMapping()
err = indexMapping.AddCustomTokenizer("ekanite_tk",
map[string]interface{}{
"regexp": `[^\W_]+`,
"type": regexp_tokenizer.Name,
})
if err != nil {
return nil, err
}
err = indexMapping.AddCustomAnalyzer("ekanite",
map[string]interface{}{
"type": custom_analyzer.Name,
"char_filters": []interface{}{},
"tokenizer": `ekanite_tk`,
"token_filters": []interface{}{`to_lower`},
})
if err != nil {
return nil, err
}
indexMapping.DefaultAnalyzer = "ekanite"
// Create field-specific mappings.
simpleJustIndexed := bleve.NewTextFieldMapping()
simpleJustIndexed.Store = false
simpleJustIndexed.IncludeInAll = true // XXX Move to false when using AST
simpleJustIndexed.IncludeTermVectors = false
timeJustIndexed := bleve.NewDateTimeFieldMapping()
timeJustIndexed.Store = false
timeJustIndexed.IncludeInAll = false
timeJustIndexed.IncludeTermVectors = false
articleMapping := bleve.NewDocumentMapping()
// Connect field mappings to fields.
articleMapping.AddFieldMappingsAt("Message", simpleJustIndexed)
articleMapping.AddFieldMappingsAt("ReferenceTime", timeJustIndexed)
articleMapping.AddFieldMappingsAt("ReceptionTime", timeJustIndexed)
// Tell the index about field mappings.
indexMapping.DefaultMapping = articleMapping
return indexMapping, nil
}
func getIndex(indexName string) (bleve.Index, error) {
if _, err := os.Stat(indexName); os.IsNotExist(err) {
log.Printf("Creating Bleve index at %s\n", indexName)
log.Println("Setting up index...")
indexMapping := bleve.NewIndexMapping()
// Add Accession as a specific document map
accessionMapping := bleve.NewDocumentMapping()
// Now add specific accession fields
titleMapping := bleve.NewTextFieldMapping()
titleMapping.Analyzer = "en"
titleMapping.Store = true
titleMapping.Index = true
accessionMapping.AddFieldMappingsAt("title", titleMapping)
descriptionMapping := bleve.NewTextFieldMapping()
descriptionMapping.Analyzer = "en"
descriptionMapping.Store = true
descriptionMapping.Index = true
accessionMapping.AddFieldMappingsAt("content_description", descriptionMapping)
subjectsMapping := bleve.NewTextFieldMapping()
subjectsMapping.Analyzer = "en"
subjectsMapping.Store = true
subjectsMapping.Index = true
subjectsMapping.IncludeTermVectors = true
accessionMapping.AddFieldMappingsAt("subjects", subjectsMapping)
subjectsFunctionMapping := bleve.NewTextFieldMapping()
subjectsFunctionMapping.Analyzer = "en"
subjectsFunctionMapping.Store = true
subjectsFunctionMapping.Index = true
subjectsFunctionMapping.IncludeTermVectors = true
accessionMapping.AddFieldMappingsAt("subjects_function", subjectsFunctionMapping)
subjectsTopicalMapping := bleve.NewTextFieldMapping()
subjectsTopicalMapping.Analyzer = "en"
subjectsTopicalMapping.Store = true
subjectsTopicalMapping.Index = true
subjectsTopicalMapping.IncludeTermVectors = true
accessionMapping.AddFieldMappingsAt("subjects_topical", subjectsTopicalMapping)
objectTitleMapping := bleve.NewTextFieldMapping()
objectTitleMapping.Analyzer = "en"
objectTitleMapping.Store = true
objectTitleMapping.Index = false
accessionMapping.AddFieldMappingsAt("digital_objects.title", objectTitleMapping)
objectFileURIMapping := bleve.NewTextFieldMapping()
objectFileURIMapping.Analyzer = ""
objectFileURIMapping.Store = true
objectFileURIMapping.Index = false
accessionMapping.AddFieldMappingsAt("digital_objects.file_uris", objectFileURIMapping)
extentsMapping := bleve.NewTextFieldMapping()
extentsMapping.Analyzer = "en"
extentsMapping.Store = true
extentsMapping.Index = true
accessionMapping.AddFieldMappingsAt("extents", extentsMapping)
accessionDateMapping := bleve.NewTextFieldMapping()
accessionDateMapping.Analyzer = "en"
accessionDateMapping.Store = true
accessionDateMapping.Index = false
accessionMapping.AddFieldMappingsAt("accession_date", accessionDateMapping)
datesMapping := bleve.NewTextFieldMapping()
datesMapping.Store = true
datesMapping.Index = false
accessionMapping.AddFieldMappingsAt("date_expression", datesMapping)
createdMapping := bleve.NewDateTimeFieldMapping()
createdMapping.Store = true
createdMapping.Index = false
accessionMapping.AddFieldMappingsAt("created", createdMapping)
// Finally add this mapping to the main index mapping
indexMapping.AddDocumentMapping("accession", accessionMapping)
index, err := bleve.New(indexName, indexMapping)
if err != nil {
return nil, fmt.Errorf("Can't create new bleve index %s, %s", indexName, err)
}
return index, nil
}
log.Printf("Opening Bleve index at %s", indexName)
index, err := bleve.OpenUsing(indexName, map[string]interface{}{
"read_only": false,
})
if err != nil {
return nil, fmt.Errorf("Can't create new bleve index %s, %s", indexName, err)
}
return index, nil
}
func NewOfferIndex(dir string) (bleve.Index, error) {
err := os.RemoveAll(dir)
if err != nil && !os.IsNotExist(err) {
return nil, err
}
parts := []string{}
for _, exc := range indexExceptions {
parts = append(parts, regexp.QuoteMeta(exc))
}
pattern := strings.Join(parts, "|")
pattern = "(?i)(?:" + pattern + ")"
m := bleve.NewIndexMapping()
apecTokenizer := "apec"
err = m.AddCustomTokenizer(apecTokenizer, map[string]interface{}{
"type": exception.Name,
"exceptions": []string{pattern},
"tokenizer": bleveuni.Name,
})
if err != nil {
return nil, err
}
apecTokens := "apec_tokens"
err = m.AddCustomTokenMap(apecTokens, map[string]interface{}{
"type": tokenmap.Name,
"tokens": stopWords,
})
if err != nil {
return nil, err
}
apecStop := "apec_stop"
err = m.AddCustomTokenFilter(apecStop, map[string]interface{}{
"type": stop.Name,
"stop_token_map": apecTokens,
})
if err != nil {
return nil, err
}
frTokens := []string{
lowercase.Name,
fr.ElisionName,
fr.StopName,
fr.LightStemmerName,
apecStop,
}
fr := map[string]interface{}{
"type": custom.Name,
"tokenizer": apecTokenizer,
"token_filters": frTokens,
}
frHtml := map[string]interface{}{
"type": custom.Name,
"char_filters": []string{
html.Name,
},
"tokenizer": apecTokenizer,
"token_filters": frTokens,
}
err = m.AddCustomAnalyzer("fr", fr)
if err != nil {
return nil, fmt.Errorf("failed to register analyzer fr: %s", err)
}
err = m.AddCustomAnalyzer("fr_html", frHtml)
if err != nil {
return nil, fmt.Errorf("failed to register analyzer fr_html: %s", err)
}
htmlFr := bleve.NewTextFieldMapping()
htmlFr.Store = false
htmlFr.IncludeInAll = false
htmlFr.IncludeTermVectors = false
htmlFr.Analyzer = "fr_html"
textFr := bleve.NewTextFieldMapping()
textFr.Store = false
textFr.IncludeInAll = false
textFr.IncludeTermVectors = false
textFr.Analyzer = "fr"
textAll := bleve.NewTextFieldMapping()
textAll.Store = false
textAll.IncludeInAll = true
textAll.IncludeTermVectors = false
date := bleve.NewDateTimeFieldMapping()
date.Index = false
date.Store = true
date.IncludeInAll = false
date.IncludeTermVectors = false
offer := bleve.NewDocumentStaticMapping()
offer.Dynamic = false
offer.AddFieldMappingsAt("html", htmlFr)
offer.AddFieldMappingsAt("title", textFr)
offer.AddFieldMappingsAt("date", date)
m.AddDocumentMapping("offer", offer)
m.DefaultMapping = offer
index, err := bleve.NewUsing(dir, m, upsidedown.Name, boltdb.Name,
map[string]interface{}{
"nosync": true,
})
if err != nil {
return nil, err
}
return index, nil
}
