func ExampleScanner_Scan() {
// src is the input that we want to tokenize.
src := []byte("cos(x) + 1i*sin(x) // Euler")
// Initialize the scanner.
var s scanner.Scanner
fset := token.NewFileSet() // positions are relative to fset
file := fset.AddFile("", fset.Base(), len(src)) // register input "file"
s.Init(file, src, nil /* no error handler */, scanner.ScanComments)
// Repeated calls to Scan yield the token sequence found in the input.
for {
pos, tok, lit := s.Scan()
if tok == token.EOF {
break
}
fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit)
}
// output:
// 1:1 IDENT "cos"
// 1:4 ( ""
// 1:5 IDENT "x"
// 1:6 ) ""
// 1:8 + ""
// 1:10 IMAG "1i"
// 1:12 * ""
// 1:13 IDENT "sin"
// 1:16 ( ""
// 1:17 IDENT "x"
// 1:18 ) ""
// 1:20 ; "\n"
// 1:20 COMMENT "// Euler"
}
// errMap collects the regular expressions of ERROR comments found
// in files and returns them as a map of error positions to error messages.
//
func errMap(t *testing.T, testname string, files []*ast.File) map[string][]string {
// map of position strings to lists of error message patterns
errmap := make(map[string][]string)
for _, file := range files {
filename := fset.Position(file.Package).Filename
src, err := ioutil.ReadFile(filename)
if err != nil {
t.Fatalf("%s: could not read %s", testname, filename)
}
var s scanner.Scanner
s.Init(fset.AddFile(filename, -1, len(src)), src, nil, scanner.ScanComments)
var prev token.Pos // position of last non-comment, non-semicolon token
var here token.Pos // position immediately after the token at position prev
scanFile:
for {
pos, tok, lit := s.Scan()
switch tok {
case token.EOF:
break scanFile
case token.COMMENT:
if lit[1] == '*' {
lit = lit[:len(lit)-2] // strip trailing */
}
if s := errRx.FindStringSubmatch(lit[2:]); len(s) == 3 {
pos := prev
if s[1] == "HERE" {
pos = here
}
p := fset.Position(pos).String()
// TODO: Use also column once we don't mess them up anymore.
p = p[:strings.LastIndex(p, ":")]
errmap[p] = append(errmap[p], strings.TrimSpace(s[2]))
}
case token.SEMICOLON:
// ignore automatically inserted semicolon
if lit == "\n" {
continue scanFile
}
fallthrough
default:
prev = pos
var l int // token length
if tok.IsLiteral() {
l = len(lit)
} else {
l = len(tok.String())
}
here = prev + token.Pos(l)
}
}
}
return errmap
}
// expectedErrors collects the regular expressions of ERROR comments found
// in files and returns them as a map of error positions to error messages.
//
func expectedErrors(t *testing.T, fset *token.FileSet, filename string, src []byte) map[token.Pos]string {
errors := make(map[token.Pos]string)
var s scanner.Scanner
// file was parsed already - do not add it again to the file
// set otherwise the position information returned here will
// not match the position information collected by the parser
s.Init(getFile(fset, filename), src, nil, scanner.ScanComments)
var prev token.Pos // position of last non-comment, non-semicolon token
var here token.Pos // position immediately after the token at position prev
for {
pos, tok, lit := s.Scan()
switch tok {
case token.EOF:
return errors
case token.COMMENT:
s := errRx.FindStringSubmatch(lit)
if len(s) == 3 {
pos := prev
if s[1] == "HERE" {
pos = here
}
errors[pos] = string(s[2])
}
default:
prev = pos
var l int // token length
if tok.IsLiteral() {
l = len(lit)
} else {
l = len(tok.String())
}
here = prev + token.Pos(l)
}
}
}