func ExampleScanner_Scan() {
// src is the input that we want to tokenize.
src := []byte("cos(x) + 1*sin(x) // Euler")
// Initialize the scanner.
var s scanner.Scanner
fset := token.NewFileSet() // positions are relative to fset
file := fset.AddFile("", fset.Base(), len(src)) // register input "file"
s.Init(file, src, nil /* no error handler */, scanner.ScanComments)
// Repeated calls to Scan yield the token sequence found in the input.
for {
pos, tok, lit := s.Scan()
if tok == token.EOF {
break
}
fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit)
}
// output:
// 1:1 IDENT "cos"
// 1:4 ( ""
// 1:5 IDENT "x"
// 1:6 ) ""
// 1:8 + ""
// 1:10 INT "1"
// 1:11 * ""
// 1:12 IDENT "sin"
// 1:15 ( ""
// 1:16 IDENT "x"
// 1:17 ) ""
// 1:19 ; "\n"
// 1:19 COMMENT "// Euler"
}
// expectedErrors collects the regular expressions of ERROR comments found
// in files and returns them as a map of error positions to error messages.
//
func expectedErrors(t *testing.T, fset *token.FileSet, filename string, src []byte) map[token.Pos]string {
errors := make(map[token.Pos]string)
var s scanner.Scanner
// file was parsed already - do not add it again to the file
// set otherwise the position information returned here will
// not match the position information collected by the parser
s.Init(getFile(fset, filename), src, nil, scanner.ScanComments)
var prev token.Pos // position of last non-comment, non-semicolon token
var here token.Pos // position immediately after the token at position prev
for {
pos, tok, lit := s.Scan()
switch tok {
case token.EOF:
return errors
case token.COMMENT:
s := errRx.FindStringSubmatch(lit)
if len(s) == 3 {
pos := prev
if s[1] == "HERE" {
pos = here
}
errors[pos] = string(s[2])
}
default:
prev = pos
var l int // token length
if tok.IsLiteral() {
l = len(lit)
} else {
l = len(tok.String())
}
here = prev + token.Pos(l)
}
}
}