Esempio n. 1
0
func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
	tokenMap, err := cache.TokenMapNamed(StopName)
	if err != nil {
		return nil, err
	}
	return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil
}
Esempio n. 2
0
func DictionaryCompoundFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {

	minWordSize := defaultMinWordSize
	minSubWordSize := defaultMinSubWordSize
	maxSubWordSize := defaultMaxSubWordSize
	onlyLongestMatch := defaultOnlyLongestMatch

	minVal, ok := config["min_word_size"].(float64)
	if ok {
		minWordSize = int(minVal)
	}
	minSubVal, ok := config["min_subword_size"].(float64)
	if ok {
		minSubWordSize = int(minSubVal)
	}
	maxSubVal, ok := config["max_subword_size"].(float64)
	if ok {
		maxSubWordSize = int(maxSubVal)
	}
	onlyVal, ok := config["only_longest_match"].(bool)
	if ok {
		onlyLongestMatch = onlyVal
	}

	dictTokenMapName, ok := config["dict_token_map"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify dict_token_map")
	}
	dictTokenMap, err := cache.TokenMapNamed(dictTokenMapName)
	if err != nil {
		return nil, fmt.Errorf("error building dict compound words filter: %v", err)
	}
	return NewDictionaryCompoundFilter(dictTokenMap, minWordSize, minSubWordSize, maxSubWordSize, onlyLongestMatch), nil
}
Esempio n. 3
0
func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
	articlesTokenMap, err := cache.TokenMapNamed(ArticlesName)
	if err != nil {
		return nil, fmt.Errorf("error building elision filter: %v", err)
	}
	return elision_filter.NewElisionFilter(articlesTokenMap), nil
}
Esempio n. 4
0
func KeyWordMarkerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
	keywordsTokenMapName, ok := config["keywords_token_map"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify keywords_token_map")
	}
	keywordsTokenMap, err := cache.TokenMapNamed(keywordsTokenMapName)
	if err != nil {
		return nil, fmt.Errorf("error building keyword marker filter: %v", err)
	}
	return NewKeyWordMarkerFilter(keywordsTokenMap), nil
}
Esempio n. 5
0
func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
	articlesTokenMapName, ok := config["articles_token_map"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify articles_token_map")
	}
	articlesTokenMap, err := cache.TokenMapNamed(articlesTokenMapName)
	if err != nil {
		return nil, fmt.Errorf("error building elision filter: %v", err)
	}
	return NewElisionFilter(articlesTokenMap), nil
}
Esempio n. 6
0
func StopTokensFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
	stopTokenMapName, ok := config["stop_token_map"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify stop_token_map")
	}
	stopTokenMap, err := cache.TokenMapNamed(stopTokenMapName)
	if err != nil {
		return nil, fmt.Errorf("error building stop words filter: %v", err)
	}
	return NewStopTokensFilter(stopTokenMap), nil
}