func newScanner(r io.Reader) *scmScanner {
var s scanner.Scanner
s.Init(r)
s.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanFloats
s.IsIdentRune = func(ch rune, i int) bool {
if i == 0 && unicode.IsDigit(ch) {
return false
}
return ch != '\'' &&
ch != '"' &&
ch != '(' &&
ch != ')' &&
ch != ';' &&
!unicode.IsSpace(ch) &&
unicode.IsPrint(ch)
}
return &scmScanner{Scanner: s}
}
func NewTokenizer(name string, r io.Reader, file *os.File) *Tokenizer {
var s scanner.Scanner
s.Init(r)
// Newline is like a semicolon; other space characters are fine.
s.Whitespace = 1<<'\t' | 1<<'\r' | 1<<' '
// Don't skip comments: we need to count newlines.
s.Mode = scanner.ScanChars |
scanner.ScanFloats |
scanner.ScanIdents |
scanner.ScanInts |
scanner.ScanStrings |
scanner.ScanComments
s.Position.Filename = name
s.IsIdentRune = isIdentRune
if file != nil {
linkCtxt.LineHist.Push(histLine, name)
}
return &Tokenizer{
s: &s,
line: 1,
fileName: name,
file: file,
}
}