示例#1
0
func NewTokenizerFromFile(src *os.File) *Tokenizer {
	if mostRecentlyUsedFile == src {
		return mostRecentFileTokenizer
	} else {
		t := NewTokenizer(bufrr.NewReader(src))
		mostRecentFileTokenizer = t
		mostRecentlyUsedFile = src
		return t
	}
}
示例#2
0
func NewTokenizerFromString(src string) *Tokenizer {
	return NewTokenizer(bufrr.NewReader(strings.NewReader(src)))
}