func getStringTokens(term string) ([]string, error) {
tokenizer, err := tok.NewTokenizer([]byte(term))
if err != nil {
return nil, x.Errorf("Could not create tokenizer: %v", term)
}
defer tokenizer.Destroy()
return tokenizer.Tokens(), nil
}
// DefaultIndexKeys tokenizes data as a string and return keys for indexing.
func DefaultIndexKeys(attr string, val *String) []string {
data := []byte((*val).String())
tokenizer, err := tok.NewTokenizer(data)
if err != nil {
return nil
}
defer tokenizer.Destroy()
tokens := make([]string, 0, 5)
for {
s := tokenizer.Next()
if s == nil {
break
}
tokens = append(tokens, string(s))
}
return tokens
}
