func NewTokenizerFromFile(src *os.File) *Tokenizer {
if mostRecentlyUsedFile == src {
return mostRecentFileTokenizer
} else {
t := NewTokenizer(bufrr.NewReader(src))
mostRecentFileTokenizer = t
mostRecentlyUsedFile = src
return t
}
}
func NewTokenizerFromString(src string) *Tokenizer {
return NewTokenizer(bufrr.NewReader(strings.NewReader(src)))
}