func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { icuTokenizer, err := cache.TokenizerNamed(icu.Name) if err != nil { return nil, err } toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) if err != nil { return nil, err } stopHuFilter, err := cache.TokenFilterNamed(StopName) if err != nil { return nil, err } stemmerHuFilter, err := cache.TokenFilterNamed(StemmerName) if err != nil { return nil, err } rv := analysis.Analyzer{ Tokenizer: icuTokenizer, TokenFilters: []analysis.TokenFilter{ toLowerFilter, stopHuFilter, stemmerHuFilter, }, } return &rv, nil }
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { tokenizer, err := cache.TokenizerNamed(unicode.Name) if err != nil { return nil, err } toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) if err != nil { return nil, err } normalizeFilter := unicode_normalize.MustNewUnicodeNormalizeFilter(unicode_normalize.NFKC) stopArFilter, err := cache.TokenFilterNamed(StopName) if err != nil { return nil, err } normalizeArFilter, err := cache.TokenFilterNamed(NormalizeName) if err != nil { return nil, err } stemmerArFilter, err := cache.TokenFilterNamed(StemmerName) if err != nil { return nil, err } rv := analysis.Analyzer{ Tokenizer: tokenizer, TokenFilters: []analysis.TokenFilter{ toLowerFilter, normalizeFilter, stopArFilter, normalizeArFilter, stemmerArFilter, }, } return &rv, nil }
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { tokenizer, err := cache.TokenizerNamed(unicode.Name) if err != nil { return nil, err } elisionFilter, err := cache.TokenFilterNamed(ElisionName) if err != nil { return nil, err } toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) if err != nil { return nil, err } stopFrFilter, err := cache.TokenFilterNamed(StopName) if err != nil { return nil, err } stemmerFrFilter, err := cache.TokenFilterNamed(LightStemmerName) if err != nil { return nil, err } rv := analysis.Analyzer{ Tokenizer: tokenizer, TokenFilters: []analysis.TokenFilter{ elisionFilter, toLowerFilter, stopFrFilter, stemmerFrFilter, }, } return &rv, nil }
func ExceptionsTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { exceptions := []string{} iexceptions, ok := config["exceptions"].([]interface{}) if ok { for _, exception := range iexceptions { exception, ok := exception.(string) if ok { exceptions = append(exceptions, exception) } } } aexceptions, ok := config["exceptions"].([]string) if ok { exceptions = append(exceptions, aexceptions...) } exceptionPattern := strings.Join(exceptions, "|") r, err := regexp.Compile(exceptionPattern) if err != nil { return nil, fmt.Errorf("unable to build regexp tokenizer: %v", err) } remainingName, ok := config["tokenizer"].(string) if !ok { return nil, fmt.Errorf("must specify tokenizer for remaining input") } remaining, err := cache.TokenizerNamed(remainingName) if err != nil { return nil, err } return NewExceptionsTokenizer(r, remaining), nil }
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { tokenizer, err := cache.TokenizerNamed(unicode.Name) if err != nil { return nil, err } possEnFilter, err := cache.TokenFilterNamed(PossessiveName) if err != nil { return nil, err } toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) if err != nil { return nil, err } stopEnFilter, err := cache.TokenFilterNamed(StopName) if err != nil { return nil, err } stemmerEnFilter, err := cache.TokenFilterNamed(porter.Name) if err != nil { return nil, err } rv := analysis.Analyzer{ Tokenizer: tokenizer, TokenFilters: []analysis.TokenFilter{ possEnFilter, toLowerFilter, stopEnFilter, stemmerEnFilter, }, } return &rv, nil }
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { keywordTokenizer, err := cache.TokenizerNamed(single_token.Name) if err != nil { return nil, err } rv := analysis.Analyzer{ Tokenizer: keywordTokenizer, } return &rv, nil }
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { kagomeTokenizer, err := cache.TokenizerNamed(TokenizerName) if err != nil { return nil, err } normalizeFilter := unicode_normalize.MustNewUnicodeNormalizeFilter(unicode_normalize.NFKD) rv := analysis.Analyzer{ Tokenizer: kagomeTokenizer, TokenFilters: []analysis.TokenFilter{ normalizeFilter, }, } return &rv, nil }
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { zFilter, err := cache.CharFilterNamed(zero_width_non_joiner.Name) if err != nil { return nil, err } icuTokenizer, err := cache.TokenizerNamed(icu.Name) if err != nil { return nil, err } normArFilter, err := cache.TokenFilterNamed(ar.NormalizeName) if err != nil { return nil, err } normFaFilter, err := cache.TokenFilterNamed(NormalizeName) if err != nil { return nil, err } toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) if err != nil { return nil, err } stopFaFilter, err := cache.TokenFilterNamed(StopName) if err != nil { return nil, err } rv := analysis.Analyzer{ CharFilters: []analysis.CharFilter{ zFilter, }, Tokenizer: icuTokenizer, TokenFilters: []analysis.TokenFilter{ toLowerFilter, normArFilter, normFaFilter, stopFaFilter, }, } return &rv, nil }
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { whitespaceTokenizer, err := cache.TokenizerNamed(whitespace_tokenizer.Name) if err != nil { return nil, err } normalizeFilter := unicode_normalize.MustNewUnicodeNormalizeFilter(unicode_normalize.NFKD) toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) if err != nil { return nil, err } bigramFilter, err := cache.TokenFilterNamed(BigramName) if err != nil { return nil, err } rv := analysis.Analyzer{ Tokenizer: whitespaceTokenizer, TokenFilters: []analysis.TokenFilter{ normalizeFilter, toLowerFilter, bigramFilter, }, } return &rv, nil }
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { var err error var charFilters []analysis.CharFilter charFiltersNames, ok := config["char_filters"].([]string) if ok { charFilters, err = getCharFilters(charFiltersNames, cache) if err != nil { return nil, err } } else { charFiltersNamesInterfaceSlice, ok := config["char_filters"].([]interface{}) if ok { charFiltersNames, err := convertInterfaceSliceToStringSlice(charFiltersNamesInterfaceSlice, "char filter") if err != nil { return nil, err } charFilters, err = getCharFilters(charFiltersNames, cache) if err != nil { return nil, err } } } tokenizerName, ok := config["tokenizer"].(string) if !ok { return nil, fmt.Errorf("must specify tokenizer") } tokenizer, err := cache.TokenizerNamed(tokenizerName) if err != nil { return nil, err } var tokenFilters []analysis.TokenFilter tokenFiltersNames, ok := config["token_filters"].([]string) if ok { tokenFilters, err = getTokenFilters(tokenFiltersNames, cache) if err != nil { return nil, err } } else { tokenFiltersNamesInterfaceSlice, ok := config["token_filters"].([]interface{}) if ok { tokenFiltersNames, err := convertInterfaceSliceToStringSlice(tokenFiltersNamesInterfaceSlice, "token filter") if err != nil { return nil, err } tokenFilters, err = getTokenFilters(tokenFiltersNames, cache) if err != nil { return nil, err } } } rv := analysis.Analyzer{ Tokenizer: tokenizer, } if charFilters != nil { rv.CharFilters = charFilters } if tokenFilters != nil { rv.TokenFilters = tokenFilters } return &rv, nil }