func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
remainingTokenizer, err := cache.TokenizerNamed(unicode.Name)
if err != nil {
return nil, err
}
return exception.NewExceptionsTokenizer(exceptionsRegexp, remainingTokenizer), nil
}
func analyzeFn(cfg *Config) error {
reExc, err := regexp.Compile(`(?i)c\+\+`)
if err != nil {
return err
}
uni := unicode.NewUnicodeTokenizer()
tokenizer := exception.NewExceptionsTokenizer(reExc, uni)
tokens := tokenizer.Tokenize([]byte(*analyzeArg))
for _, t := range tokens {
fmt.Println(t)
}
return nil
}