func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
var s scanner.Scanner
var h errorCollector
s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &h, scanner.ScanComments)
_, tok0, _ := s.Scan()
_, tok1, _ := s.Scan()
if tok0 != tok {
t.Errorf("%q: got %s, expected %s", src, tok0, tok)
}
if tok1 != token.EOF {
t.Errorf("%q: got %s, expected EOF", src, tok1)
}
cnt := 0
if err != "" {
cnt = 1
}
if h.cnt != cnt {
t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
}
if h.msg != err {
t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
}
if h.pos.Offset != pos {
t.Errorf("%q: got offset %d, expected %d", src, h.pos.Offset, pos)
}
}
// Verify that calling Scan() provides the correct results.
func TestScan(t *testing.T) {
// make source
var src string
for _, e := range tokens {
src += e.lit + whitespace
}
src_linecount := newlineCount(src)
whitespace_linecount := newlineCount(whitespace)
// verify scan
var s scanner.Scanner
s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &testErrorHandler{t}, scanner.ScanComments)
index := 0
epos := token.Position{"", 0, 1, 1} // expected position
for {
pos, tok, litb := s.Scan()
e := elt{token.EOF, "", special}
if index < len(tokens) {
e = tokens[index]
}
lit := string(litb)
if tok == token.EOF {
lit = "<EOF>"
epos.Line = src_linecount
epos.Column = 2
}
checkPos(t, lit, pos, epos)
if tok != e.tok {
t.Errorf("bad token for %q: got %s, expected %s", lit, tok.String(), e.tok.String())
}
if e.tok.IsLiteral() && lit != e.lit {
t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, e.lit)
}
if tokenclass(tok) != e.class {
t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
}
epos.Offset += len(lit) + len(whitespace)
epos.Line += newlineCount(lit) + whitespace_linecount
if tok == token.COMMENT && litb[1] == '/' {
// correct for unaccounted '/n' in //-style comment
epos.Offset++
epos.Line++
}
index++
if tok == token.EOF {
break
}
}
if s.ErrorCount != 0 {
t.Errorf("found %d errors", s.ErrorCount)
}
}
func TestIllegalChars(t *testing.T) {
var s scanner.Scanner
const src = "*?*$*@*"
file := fset.AddFile("", fset.Base(), len(src))
s.Init(file, []byte(src), &testErrorHandler{t}, scanner.AllowIllegalChars)
for offs, ch := range src {
pos, tok, lit := s.Scan()
if poffs := file.Offset(pos); poffs != offs {
t.Errorf("bad position for %s: got %d, expected %d", string(lit), poffs, offs)
}
if tok == token.ILLEGAL && string(lit) != string(ch) {
t.Errorf("bad token: got %s, expected %s", string(lit), string(ch))
}
}
if s.ErrorCount != 0 {
t.Errorf("found %d errors", s.ErrorCount)
}
}
func TestStdErrorHander(t *testing.T) {
const src = "@\n" + // illegal character, cause an error
"@ @\n" + // two errors on the same line
"//line File2:20\n" +
"@\n" + // different file, but same line
"//line File2:1\n" +
"@ @\n" + // same file, decreasing line number
"//line File1:1\n" +
"@ @ @" // original file, line 1 again
v := new(scanner.ErrorVector)
var s scanner.Scanner
s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), v, 0)
for {
if _, tok, _ := s.Scan(); tok == token.EOF {
break
}
}
list := v.GetErrorList(scanner.Raw)
if len(list) != 9 {
t.Errorf("found %d raw errors, expected 9", len(list))
scanner.PrintError(os.Stderr, list)
}
list = v.GetErrorList(scanner.Sorted)
if len(list) != 9 {
t.Errorf("found %d sorted errors, expected 9", len(list))
scanner.PrintError(os.Stderr, list)
}
list = v.GetErrorList(scanner.NoMultiples)
if len(list) != 4 {
t.Errorf("found %d one-per-line errors, expected 4", len(list))
scanner.PrintError(os.Stderr, list)
}
if v.ErrorCount() != s.ErrorCount {
t.Errorf("found %d errors, expected %d", v.ErrorCount(), s.ErrorCount)
}
}
// Verify that comments of the form "//line filename:line" are interpreted correctly.
func TestLineComments(t *testing.T) {
// make source
var src string
for _, e := range segments {
src += e.srcline
}
// verify scan
var S scanner.Scanner
file := fset.AddFile("dir/TestLineComments", fset.Base(), len(src))
S.Init(file, []byte(src), nil, 0)
for _, s := range segments {
p, _, lit := S.Scan()
pos := file.Position(p)
checkPos(t, string(lit), p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
}
if S.ErrorCount != 0 {
t.Errorf("found %d errors", S.ErrorCount)
}
}
// Verify that initializing the same scanner more then once works correctly.
func TestInit(t *testing.T) {
var s scanner.Scanner
// 1st init
src1 := "if true { }"
f1 := fset.AddFile("src1", fset.Base(), len(src1))
s.Init(f1, []byte(src1), nil, 0)
if f1.Size() != len(src1) {
t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
}
s.Scan() // if
s.Scan() // true
_, tok, _ := s.Scan() // {
if tok != token.LBRACE {
t.Errorf("bad token: got %s, expected %s", tok.String(), token.LBRACE)
}
// 2nd init
src2 := "go true { ]"
f2 := fset.AddFile("src2", fset.Base(), len(src2))
s.Init(f2, []byte(src2), nil, 0)
if f2.Size() != len(src2) {
t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
}
_, tok, _ = s.Scan() // go
if tok != token.GO {
t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO)
}
if s.ErrorCount != 0 {
t.Errorf("found %d errors", s.ErrorCount)
}
}
func checkSemi(t *testing.T, line string, mode uint) {
var S scanner.Scanner
file := fset.AddFile("TestSemis", fset.Base(), len(line))
S.Init(file, []byte(line), nil, mode)
pos, tok, lit := S.Scan()
for tok != token.EOF {
if tok == token.ILLEGAL {
// the illegal token literal indicates what
// kind of semicolon literal to expect
semiLit := "\n"
if lit[0] == '#' {
semiLit = ";"
}
// next token must be a semicolon
semiPos := file.Position(pos)
semiPos.Offset++
semiPos.Column++
pos, tok, lit = S.Scan()
if tok == token.SEMICOLON {
if string(lit) != semiLit {
t.Errorf(`bad literal for %q: got %q, expected %q`, line, lit, semiLit)
}
checkPos(t, line, pos, semiPos)
} else {
t.Errorf("bad token for %q: got %s, expected ;", line, tok.String())
}
} else if tok == token.SEMICOLON {
t.Errorf("bad token for %q: got ;, expected no ;", line)
}
pos, tok, lit = S.Scan()
}
}