Пример #1
0
func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
	remainingTokenizer, err := cache.TokenizerNamed(unicode.Name)
	if err != nil {
		return nil, err
	}
	return exception.NewExceptionsTokenizer(exceptionsRegexp, remainingTokenizer), nil
}
Пример #2
0
func analyzeFn(cfg *Config) error {
	reExc, err := regexp.Compile(`(?i)c\+\+`)
	if err != nil {
		return err
	}
	uni := unicode.NewUnicodeTokenizer()
	tokenizer := exception.NewExceptionsTokenizer(reExc, uni)
	tokens := tokenizer.Tokenize([]byte(*analyzeArg))
	for _, t := range tokens {
		fmt.Println(t)
	}
	return nil
}