예제 #1
0
파일: bleve.go 프로젝트: endeveit/recause
// Returns data model for index
func getIndexMapping() *bv.IndexMapping {
	indexMapping := bv.NewIndexMapping()
	messageMapping := bv.NewDocumentStaticMapping()

	// Will search exact string, e.g.: «hostname.example.org» will search for «hostname.example.org»
	mappingKeyword := getTextFieldMapping()
	mappingKeyword.Analyzer = bvKeywordAnalyzer.Name

	// Tokenized query, e.g.: «hostname example org» will search for «hostname», «example» or «org»
	mappingText := getTextFieldMapping()
	mappingText.Analyzer = bvStandardAnalyzer.Name

	messageMapping.AddFieldMappingsAt("version", mappingKeyword)
	messageMapping.AddFieldMappingsAt("host", mappingKeyword)
	messageMapping.AddFieldMappingsAt("short_message", mappingText)
	messageMapping.AddFieldMappingsAt("full_message", mappingText)
	messageMapping.AddFieldMappingsAt("timestamp", bv.NewDateTimeFieldMapping())
	messageMapping.AddFieldMappingsAt("level", bv.NewNumericFieldMapping())
	messageMapping.AddFieldMappingsAt("facility", mappingKeyword)
	messageMapping.AddSubDocumentMapping("extra", bv.NewDocumentMapping())

	indexMapping.AddDocumentMapping(DOC_TYPE, messageMapping)

	return indexMapping
}
예제 #2
0
func buildIndexMapping() *bleve.IndexMapping {

	enTextFieldMapping := bleve.NewTextFieldMapping()
	enTextFieldMapping.Analyzer = textFieldAnalyzer

	storeFieldOnlyMapping := bleve.NewTextFieldMapping()
	storeFieldOnlyMapping.Index = false
	storeFieldOnlyMapping.IncludeTermVectors = false
	storeFieldOnlyMapping.IncludeInAll = false

	dateTimeMapping := bleve.NewDateTimeFieldMapping()

	wikiMapping := bleve.NewDocumentMapping()
	wikiMapping.AddFieldMappingsAt("name", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("body", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("modified_by", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("modified_by_name", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("modified_by_email", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("modified_by_avatar", storeFieldOnlyMapping)
	wikiMapping.AddFieldMappingsAt("modified", dateTimeMapping)

	indexMapping := bleve.NewIndexMapping()
	indexMapping.AddDocumentMapping("wiki", wikiMapping)

	indexMapping.DefaultAnalyzer = textFieldAnalyzer

	return indexMapping
}
예제 #3
0
파일: index.go 프로젝트: tjyang/ekanite
func buildIndexMapping() (*bleve.IndexMapping, error) {
	var err error

	// Create the index mapping, configure the analyzer, and set as default.
	indexMapping := bleve.NewIndexMapping()
	err = indexMapping.AddCustomTokenizer("ekanite_tk",
		map[string]interface{}{
			"regexp": `[^\W_]+`,
			"type":   regexp_tokenizer.Name,
		})
	if err != nil {
		return nil, err
	}

	err = indexMapping.AddCustomAnalyzer("ekanite",
		map[string]interface{}{
			"type":          custom_analyzer.Name,
			"char_filters":  []interface{}{},
			"tokenizer":     `ekanite_tk`,
			"token_filters": []interface{}{`to_lower`},
		})
	if err != nil {
		return nil, err
	}
	indexMapping.DefaultAnalyzer = "ekanite"

	// Create field-specific mappings.

	simpleJustIndexed := bleve.NewTextFieldMapping()
	simpleJustIndexed.Store = false
	simpleJustIndexed.IncludeInAll = true // XXX Move to false when using AST
	simpleJustIndexed.IncludeTermVectors = false

	timeJustIndexed := bleve.NewDateTimeFieldMapping()
	timeJustIndexed.Store = false
	timeJustIndexed.IncludeInAll = false
	timeJustIndexed.IncludeTermVectors = false

	articleMapping := bleve.NewDocumentMapping()

	// Connect field mappings to fields.
	articleMapping.AddFieldMappingsAt("Message", simpleJustIndexed)
	articleMapping.AddFieldMappingsAt("ReferenceTime", timeJustIndexed)
	articleMapping.AddFieldMappingsAt("ReceptionTime", timeJustIndexed)

	// Tell the index about field mappings.
	indexMapping.DefaultMapping = articleMapping

	return indexMapping, nil
}
예제 #4
0
func getIndex(indexName string) (bleve.Index, error) {
	if _, err := os.Stat(indexName); os.IsNotExist(err) {
		log.Printf("Creating Bleve index at %s\n", indexName)

		log.Println("Setting up index...")
		indexMapping := bleve.NewIndexMapping()
		// Add Accession as a specific document map
		accessionMapping := bleve.NewDocumentMapping()

		// Now add specific accession fields
		titleMapping := bleve.NewTextFieldMapping()
		titleMapping.Analyzer = "en"
		titleMapping.Store = true
		titleMapping.Index = true
		accessionMapping.AddFieldMappingsAt("title", titleMapping)

		descriptionMapping := bleve.NewTextFieldMapping()
		descriptionMapping.Analyzer = "en"
		descriptionMapping.Store = true
		descriptionMapping.Index = true
		accessionMapping.AddFieldMappingsAt("content_description", descriptionMapping)

		subjectsMapping := bleve.NewTextFieldMapping()
		subjectsMapping.Analyzer = "en"
		subjectsMapping.Store = true
		subjectsMapping.Index = true
		subjectsMapping.IncludeTermVectors = true
		accessionMapping.AddFieldMappingsAt("subjects", subjectsMapping)

		subjectsFunctionMapping := bleve.NewTextFieldMapping()
		subjectsFunctionMapping.Analyzer = "en"
		subjectsFunctionMapping.Store = true
		subjectsFunctionMapping.Index = true
		subjectsFunctionMapping.IncludeTermVectors = true
		accessionMapping.AddFieldMappingsAt("subjects_function", subjectsFunctionMapping)

		subjectsTopicalMapping := bleve.NewTextFieldMapping()
		subjectsTopicalMapping.Analyzer = "en"
		subjectsTopicalMapping.Store = true
		subjectsTopicalMapping.Index = true
		subjectsTopicalMapping.IncludeTermVectors = true
		accessionMapping.AddFieldMappingsAt("subjects_topical", subjectsTopicalMapping)

		objectTitleMapping := bleve.NewTextFieldMapping()
		objectTitleMapping.Analyzer = "en"
		objectTitleMapping.Store = true
		objectTitleMapping.Index = false
		accessionMapping.AddFieldMappingsAt("digital_objects.title", objectTitleMapping)

		objectFileURIMapping := bleve.NewTextFieldMapping()
		objectFileURIMapping.Analyzer = ""
		objectFileURIMapping.Store = true
		objectFileURIMapping.Index = false
		accessionMapping.AddFieldMappingsAt("digital_objects.file_uris", objectFileURIMapping)

		extentsMapping := bleve.NewTextFieldMapping()
		extentsMapping.Analyzer = "en"
		extentsMapping.Store = true
		extentsMapping.Index = true
		accessionMapping.AddFieldMappingsAt("extents", extentsMapping)

		accessionDateMapping := bleve.NewTextFieldMapping()
		accessionDateMapping.Analyzer = "en"
		accessionDateMapping.Store = true
		accessionDateMapping.Index = false
		accessionMapping.AddFieldMappingsAt("accession_date", accessionDateMapping)

		datesMapping := bleve.NewTextFieldMapping()
		datesMapping.Store = true
		datesMapping.Index = false
		accessionMapping.AddFieldMappingsAt("date_expression", datesMapping)

		createdMapping := bleve.NewDateTimeFieldMapping()
		createdMapping.Store = true
		createdMapping.Index = false
		accessionMapping.AddFieldMappingsAt("created", createdMapping)

		// Finally add this mapping to the main index mapping
		indexMapping.AddDocumentMapping("accession", accessionMapping)

		index, err := bleve.New(indexName, indexMapping)
		if err != nil {
			return nil, fmt.Errorf("Can't create new bleve index %s, %s", indexName, err)
		}
		return index, nil
	}
	log.Printf("Opening Bleve index at %s", indexName)
	index, err := bleve.OpenUsing(indexName, map[string]interface{}{
		"read_only": false,
	})
	if err != nil {
		return nil, fmt.Errorf("Can't create new bleve index %s, %s", indexName, err)
	}
	return index, nil
}
예제 #5
0
파일: index.go 프로젝트: pmezard/apec
func NewOfferIndex(dir string) (bleve.Index, error) {
	err := os.RemoveAll(dir)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	parts := []string{}
	for _, exc := range indexExceptions {
		parts = append(parts, regexp.QuoteMeta(exc))
	}
	pattern := strings.Join(parts, "|")
	pattern = "(?i)(?:" + pattern + ")"

	m := bleve.NewIndexMapping()
	apecTokenizer := "apec"
	err = m.AddCustomTokenizer(apecTokenizer, map[string]interface{}{
		"type":       exception.Name,
		"exceptions": []string{pattern},
		"tokenizer":  bleveuni.Name,
	})
	if err != nil {
		return nil, err
	}

	apecTokens := "apec_tokens"
	err = m.AddCustomTokenMap(apecTokens, map[string]interface{}{
		"type":   tokenmap.Name,
		"tokens": stopWords,
	})
	if err != nil {
		return nil, err
	}

	apecStop := "apec_stop"
	err = m.AddCustomTokenFilter(apecStop, map[string]interface{}{
		"type":           stop.Name,
		"stop_token_map": apecTokens,
	})
	if err != nil {
		return nil, err
	}

	frTokens := []string{
		lowercase.Name,
		fr.ElisionName,
		fr.StopName,
		fr.LightStemmerName,
		apecStop,
	}
	fr := map[string]interface{}{
		"type":          custom.Name,
		"tokenizer":     apecTokenizer,
		"token_filters": frTokens,
	}
	frHtml := map[string]interface{}{
		"type": custom.Name,
		"char_filters": []string{
			html.Name,
		},
		"tokenizer":     apecTokenizer,
		"token_filters": frTokens,
	}
	err = m.AddCustomAnalyzer("fr", fr)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr: %s", err)
	}
	err = m.AddCustomAnalyzer("fr_html", frHtml)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr_html: %s", err)
	}

	htmlFr := bleve.NewTextFieldMapping()
	htmlFr.Store = false
	htmlFr.IncludeInAll = false
	htmlFr.IncludeTermVectors = false
	htmlFr.Analyzer = "fr_html"

	textFr := bleve.NewTextFieldMapping()
	textFr.Store = false
	textFr.IncludeInAll = false
	textFr.IncludeTermVectors = false
	textFr.Analyzer = "fr"

	textAll := bleve.NewTextFieldMapping()
	textAll.Store = false
	textAll.IncludeInAll = true
	textAll.IncludeTermVectors = false

	date := bleve.NewDateTimeFieldMapping()
	date.Index = false
	date.Store = true
	date.IncludeInAll = false
	date.IncludeTermVectors = false

	offer := bleve.NewDocumentStaticMapping()
	offer.Dynamic = false
	offer.AddFieldMappingsAt("html", htmlFr)
	offer.AddFieldMappingsAt("title", textFr)
	offer.AddFieldMappingsAt("date", date)

	m.AddDocumentMapping("offer", offer)
	m.DefaultMapping = offer

	index, err := bleve.NewUsing(dir, m, upsidedown.Name, boltdb.Name,
		map[string]interface{}{
			"nosync": true,
		})
	if err != nil {
		return nil, err
	}
	return index, nil
}