func (d *Decoder) Token() (Token, error) {
t := d.scan.Scan()
if d.err != nil {
return nil, d.err
}
if d.depth == 0 && t != '(' && t != scanner.EOF {
return nil, fmt.Errorf("expecting '(', got %s", scanner.TokenString(t))
}
switch t {
case scanner.EOF:
return nil, io.EOF
case scanner.Ident:
return Symbol(d.scan.TokenText()), nil
case scanner.String:
text := d.scan.TokenText()
// Assume all strings are quoted.
return String(text[1 : len(text)-1]), nil
case scanner.Int:
n, err := strconv.ParseInt(d.scan.TokenText(), 10, 64)
if err != nil {
return nil, err
}
return Int(n), nil
case '(':
d.depth++
return StartList{}, nil
case ')':
d.depth--
return EndList{}, nil
default:
pos := d.scan.Pos()
return nil, fmt.Errorf("unexpected token %s at L%d:C%d", scanner.TokenString(t), pos.Line, pos.Column)
}
}
func (p *parser) parseDefinitions() (defs []Definition) {
for {
switch p.tok {
case scanner.Ident:
ident := p.scanner.TokenText()
pos := p.scanner.Position
p.accept(scanner.Ident)
switch p.tok {
case '+':
p.accept('+')
defs = append(defs, p.parseAssignment(ident, pos, "+="))
case '=':
defs = append(defs, p.parseAssignment(ident, pos, "="))
case '{', '(':
defs = append(defs, p.parseModule(ident, pos))
default:
p.errorf("expected \"=\" or \"+=\" or \"{\" or \"(\", found %s",
scanner.TokenString(p.tok))
}
case scanner.EOF:
return
default:
p.errorf("expected assignment or module definition, found %s",
scanner.TokenString(p.tok))
return
}
}
}
func (p *parser) expect(tok rune) string {
lit := p.lit
if p.tok != tok {
p.errorf("expected %s, got %s (%s)", scanner.TokenString(tok), scanner.TokenString(p.tok), lit)
}
p.next()
return lit
}
func (p *gcParser) expect(tok int) string {
lit := p.lit
if p.tok != tok {
p.errorf("expected %q, got %q (%q)", scanner.TokenString(tok), scanner.TokenString(p.tok), lit)
}
p.next()
return lit
}
func (p *parser) accept(toks ...rune) bool {
for _, tok := range toks {
if p.tok != tok {
p.errorf("expected %s, found %s", scanner.TokenString(tok),
scanner.TokenString(p.tok))
return false
}
p.next()
}
return true
}
func (this *import_data_parser) read_int() string {
val := ""
if this.toktype == '-' {
this.next()
val += "-"
}
if this.toktype != scanner.Int {
this.errorf("expected: %s, got: %s", scanner.TokenString(scanner.Int), scanner.TokenString(this.toktype))
}
val += this.token()
this.next()
return val
}
// ConstDecl = "const" ExportedName [ Type ] "=" Literal .
// Literal = bool_lit | int_lit | float_lit | complex_lit | string_lit .
// bool_lit = "true" | "false" .
// complex_lit = "(" float_lit "+" float_lit ")" .
// rune_lit = "(" int_lit "+" int_lit ")" .
// string_lit = `"` { unicode_char } `"` .
//
func (p *gcParser) parseConstDecl() {
p.expectKeyword("const")
pkg, name := p.parseExportedName()
obj := p.declare(pkg.Data.(*ast.Scope), ast.Con, name)
var x Const
var typ Type
if p.tok != '=' {
obj.Type = p.parseType()
}
p.expect('=')
switch p.tok {
case scanner.Ident:
// bool_lit
if p.lit != "true" && p.lit != "false" {
p.error("expected true or false")
}
x = Const{p.lit == "true"}
typ = Bool.Underlying
p.next()
case '-', scanner.Int:
// int_lit
x = p.parseNumber()
typ = Int.Underlying
if _, ok := x.val.(*big.Rat); ok {
typ = Float64.Underlying
}
case '(':
// complex_lit or rune_lit
p.next()
if p.tok == scanner.Char {
p.next()
p.expect('+')
p.parseNumber()
p.expect(')')
// TODO: x = ...
break
}
re := p.parseNumber()
p.expect('+')
im := p.parseNumber()
p.expect(')')
x = Const{cmplx{re.val.(*big.Rat), im.val.(*big.Rat)}}
typ = Complex128.Underlying
case scanner.Char:
// TODO: x = ...
p.next()
case scanner.String:
// string_lit
x = MakeConst(token.STRING, p.lit)
p.next()
typ = String.Underlying
default:
println(p.tok)
p.errorf("expected literal got %s", scanner.TokenString(p.tok))
}
if obj.Type == nil {
obj.Type = typ
}
obj.Data = x
}
// Export = "PackageClause { Decl } "$$" .
// PackageClause = "package" PackageName [ "safe" ] "\n" .
//
func (p *parser) parseExport() *types.Package {
p.expectKeyword("package")
name := p.parsePackageName()
if p.tok == scanner.Ident && p.lit == "safe" {
// package was compiled with -u option - ignore
p.next()
}
p.expect('\n')
pkg := p.getPkg(p.id, name)
for p.tok != '$' && p.tok != scanner.EOF {
p.parseDecl()
}
if ch := p.scanner.Peek(); p.tok != '$' || ch != '$' {
// don't call next()/expect() since reading past the
// export data may cause scanner errors (e.g. NUL chars)
p.errorf("expected '$$', got %s %c", scanner.TokenString(p.tok), ch)
}
if n := p.scanner.ErrorCount; n != 0 {
p.errorf("expected no scanner errors, got %d", n)
}
// package was imported completely and without errors
pkg.MarkComplete()
return pkg
}
// Export = "PackageClause { Decl } "$$" .
// PackageClause = "package" identifier [ "safe" ] "\n" .
//
func (p *gcParser) parseExport() *ast.Object {
p.expectKeyword("package")
name := p.expect(scanner.Ident)
if p.tok != '\n' {
// A package is safe if it was compiled with the -u flag,
// which disables the unsafe package.
// TODO(gri) remember "safe" package
p.expectKeyword("safe")
}
p.expect('\n')
assert(p.imports[p.id] == nil)
pkg := ast.NewObj(ast.Pkg, name)
pkg.Data = ast.NewScope(nil)
p.imports[p.id] = pkg
for p.tok != '$' && p.tok != scanner.EOF {
p.parseDecl()
}
if ch := p.scanner.Peek(); p.tok != '$' || ch != '$' {
// don't call next()/expect() since reading past the
// export data may cause scanner errors (e.g. NUL chars)
p.errorf("expected '$$', got %s %c", scanner.TokenString(p.tok), ch)
}
if n := p.scanner.ErrorCount; n != 0 {
p.errorf("expected no scanner errors, got %d", n)
}
return pkg
}
// internal, use expect(scanner.Ident) instead
func (this *import_data_parser) read_ident() string {
id := ""
prev := rune(0)
loop:
for {
switch this.toktype {
case scanner.Ident:
if prev == scanner.Ident {
break loop
}
prev = this.toktype
id += this.token()
this.next()
case '.', '?', '$':
prev = this.toktype
id += string(this.toktype)
this.next()
default:
break loop
}
}
if id == "" {
this.errorf("identifier expected, got %s", scanner.TokenString(this.toktype))
}
return id
}
func (p *Parser) parseValue(tok rune) error {
if tok == '{' {
return p.parseObject()
} else if tok == '[' {
return p.parseArray()
} else if tok == scanner.String {
p.c.StringValue(p.s.TokenText())
return nil
} else if tok == '-' {
tok = p.s.Scan()
if tok == scanner.Int || tok == scanner.Float {
p.c.NumberValue("-" + p.s.TokenText())
return nil
}
return p.createError("expected number, but got %s",
scanner.TokenString(tok))
} else if tok == scanner.Int || tok == scanner.Float {
p.c.NumberValue(p.s.TokenText())
return nil
} else if tok == scanner.Ident && isLiteral(p.s.TokenText()) {
p.c.LiteralValue(toLiteral(p.s.TokenText()))
return nil
}
return p.createError("expected object, array, string, number or literal, but got %s", scanner.TokenString(tok))
}
func (p *Parser) parseArray() error {
p.c.StartArray()
for {
tok := p.s.Scan()
if tok == ']' {
p.c.EndArray()
return nil
}
p.c.StartValue()
if err := p.parseValue(tok); err != nil {
return err
}
tok = p.s.Scan()
if tok == ']' {
p.c.EndValue(false)
p.c.EndArray()
return nil
}
if tok != ',' {
return p.createError("in array, expected ',', but got %s",
scanner.TokenString(tok))
}
p.c.EndValue(true)
}
return nil
}
// Export = "PackageClause { Decl } "$$" .
// PackageClause = "package" PackageName [ "safe" ] "\n" .
//
func (p *gcParser) parseExport() *Package {
p.expectKeyword("package")
name := p.parsePackageName()
if p.tok != '\n' {
// A package is safe if it was compiled with the -u flag,
// which disables the unsafe package.
// TODO(gri) remember "safe" package
p.expectKeyword("safe")
}
p.expect('\n')
pkg := p.getPkg(p.id, name)
for p.tok != '$' && p.tok != scanner.EOF {
p.parseDecl()
}
if ch := p.scanner.Peek(); p.tok != '$' || ch != '$' {
// don't call next()/expect() since reading past the
// export data may cause scanner errors (e.g. NUL chars)
p.errorf("expected '$$', got %s %c", scanner.TokenString(p.tok), ch)
}
if n := p.scanner.ErrorCount; n != 0 {
p.errorf("expected no scanner errors, got %d", n)
}
// package was imported completely and without errors
pkg.Complete = true
return pkg
}
// Type = "<" "type" ( "-" int | int [ TypeDefinition ] ) ">" .
func (p *parser) parseType(pkg *types.Package) (t types.Type) {
p.expect('<')
p.expectKeyword("type")
switch p.tok {
case scanner.Int:
n := p.parseInt()
if p.tok == '>' {
t = p.typeMap[int(n)]
} else {
t = p.parseTypeDefinition(pkg, int(n))
}
case '-':
p.next()
n := p.parseInt()
t = lookupBuiltinType(int(n))
default:
p.errorf("expected type number, got %s (%q)", scanner.TokenString(p.tok), p.lit)
return nil
}
p.expect('>')
return
}
func (p *parser) expect(tok rune) scanner.Position {
pos := p.pos
if p.tok != tok {
p.errorExpected(pos, scanner.TokenString(tok))
}
p.next() // make progress in any case
return pos
}
func main() {
d := freenect.NewFreenectDevice(StartWithDevice)
var s scanner.Scanner
s.Init(os.Stdin)
var cmd string
var arg int = OutOfRange
fmt.Print("gonect> ")
tok := s.Scan()
for tok != scanner.EOF {
if scanner.TokenString(tok) == "Ident" {
cmd = s.TokenText()
}
switch {
case cmd == "help":
showHelpMessage()
break
case cmd == "tilt":
s.Scan()
arg, _ = strconv.Atoi(s.TokenText())
if arg == OutOfRange { break }
d.SetTiltDegs(arg)
cmd = ""
arg = OutOfRange
case cmd == "led":
s.Scan()
led_string := s.TokenText()
flash_led(d, led_string)
break
case cmd == "rgb_frame":
s.Scan()
filename := s.TokenText()
freenect.SaveRGBAFrame(d, filename)
break
case cmd == "ir_frame":
s.Scan()
filename := s.TokenText()
freenect.SaveIRFrame(d, filename)
break
case cmd == "depth_frame":
s.Scan()
filename := s.TokenText()
freenect.SaveDepthFrame(d, filename)
break
case cmd == "quit":
return
}
fmt.Printf("gonect> ")
s.Scan()
}
}
// makes sure that the current token is 'x', returns it and reads the next one
func (this *import_data_parser) expect(x rune) string {
if x == scanner.Ident {
// special case, in gccgo import data identifier is not exactly a scanner.Ident
return this.read_ident()
}
if x == scanner.Int {
// another special case, handle negative ints as well
return this.read_int()
}
if this.toktype != x {
this.errorf("expected: %s, got: %s", scanner.TokenString(x), scanner.TokenString(this.toktype))
}
tok := this.token()
this.next()
return tok
}
// Type =
// BasicType | TypeName | ArrayType | SliceType | StructType |
// PointerType | FuncType | InterfaceType | MapType | ChanType |
// "(" Type ")" .
// BasicType = ident .
// TypeName = ExportedName .
// SliceType = "[" "]" Type .
// PointerType = "*" Type .
// FuncType = "func" Signature .
//
func (p *gcParser) parseType() Type {
switch p.tok {
case scanner.Ident:
switch p.lit {
default:
return p.parseBasicType()
case "struct":
return p.parseStructType()
case "func":
// FuncType
p.next()
return p.parseSignature()
case "interface":
return p.parseInterfaceType()
case "map":
return p.parseMapType()
case "chan":
return p.parseChanType()
}
case '@':
// TypeName
pkg, name := p.parseExportedName()
typ := p.declare(pkg.Data.(*ast.Scope), ast.Typ, name).Type.(Type)
switch typ := typ.(type) {
case *Struct:
typ.Package = p.pkgids[pkg]
case *Name:
typ.Package = p.pkgids[pkg]
}
return typ
case '[':
p.next() // look ahead
if p.tok == ']' {
// SliceType
p.next()
return &Slice{Elt: p.parseType()}
}
return p.parseArrayType()
case '*':
// PointerType
p.next()
return &Pointer{Base: p.parseType()}
case '<':
return p.parseChanType()
case '(':
// "(" Type ")"
p.next()
typ := p.parseType()
p.expect(')')
return typ
}
p.errorf("expected type, got %s (%q)", scanner.TokenString(p.tok), p.lit)
return nil
}
func main() {
const src = "hello 1"
var s scanner.Scanner
s.Init(bytes.NewBufferString(src))
tok := s.Scan()
for tok != scanner.EOF {
fmt.Println(scanner.TokenString(tok))
// do something with tok
tok = s.Scan()
}
}
func scanBracketedKey(s *scanner.Scanner) (key string, err error) {
s.Scan() // scan the '['
key, err = scanKey(s)
if err == nil {
t := s.Scan()
if t != ']' {
err = fmt.Errorf("Unexpected token at %s. Expected ']', had %s", s.Pos(), scanner.TokenString(t))
}
}
return
}
func (p *parser) errorExpected(pos scanner.Position, msg string) {
msg = `expected "` + msg + `"`
if pos.Offset == p.pos.Offset {
// the error happened at the current position;
// make the error message more specific
msg += ", found " + scanner.TokenString(p.tok)
if p.tok < 0 {
msg += " " + p.lit
}
}
p.error(pos, msg)
}
func (p *parser) next() {
p.tok = p.scanner.Scan()
switch p.tok {
case scanner.Ident, scanner.Int, scanner.Char, scanner.String, '·':
p.lit = p.scanner.TokenText()
default:
p.lit = ""
}
if debug {
fmt.Printf("%s: %q -> %q\n", scanner.TokenString(p.tok), p.scanner.TokenText(), p.lit)
}
}
func (p *gcParser) next() {
p.tok = p.scanner.Scan()
switch p.tok {
case scanner.Ident, scanner.Int, scanner.String:
p.lit = p.scanner.TokenText()
default:
p.lit = ""
}
if trace {
fmt.Printf("%s: %q -> %q\n", scanner.TokenString(p.tok), p.scanner.TokenText(), p.lit)
}
}
func (p *Parser) parseObject() error {
p.c.StartObject()
for {
tok := p.s.Scan()
if tok == '}' {
p.c.EndObject()
return nil
}
if tok != scanner.String {
return p.createError("expected string, but got %s ",
scanner.TokenString(tok))
}
p.c.StartMember(p.s.TokenText())
tok = p.s.Scan()
if tok != ':' {
return p.createError("expected ':', but got %s",
scanner.TokenString(tok))
}
if err := p.parseValue(p.s.Scan()); err != nil {
return err
}
tok = p.s.Scan()
if tok == '}' {
p.c.EndMember(false)
p.c.EndObject()
return nil
}
if tok != ',' {
return p.createError("in object, expected ',', but got %s",
scanner.TokenString(tok))
}
p.c.EndMember(true)
}
// NOTREACHED
return nil
}
func (p *gcParser) next() {
p.tok = p.scanner.Scan()
switch p.tok {
case scanner.Ident, scanner.Int, scanner.Char, scanner.String, '·':
p.lit = p.scanner.TokenText()
default:
p.lit = ""
}
// leave for debugging
if false {
fmt.Printf("%s: %q -> %q\n", scanner.TokenString(p.tok), p.scanner.TokenText(), p.lit)
}
}
func (s *scmScanner) read() object {
var tok = s.Scan()
for tok != EOF {
switch tok {
case scanner.Ident:
return scmSymbol(s.TokenText())
case scanner.Int:
i, _ := strconv.ParseInt(s.TokenText(), 10, 64)
return scmNumber(i)
case scanner.Float:
f, _ := strconv.ParseFloat(s.TokenText(), 64)
return scmNumber(f)
case scanner.String:
str := s.TokenText()
return scmString(str[1 : len(str)-1])
case ';':
s.skipComment()
tok = s.Scan()
case '\'':
return s.readQuoted()
case '(':
return s.readList()
case ')':
if s.expectedCloses > 0 {
return ')'
}
fallthrough
default:
return raiseError(
"Syntax error, invalid token",
scanner.TokenString(tok),
)
}
}
return EOF
}
func (p *parser) parseValue() (value Value) {
switch p.tok {
case scanner.Ident:
return p.parseVariable()
case scanner.String:
return p.parseStringValue()
case '[':
return p.parseListValue()
case '{':
return p.parseMapValue()
default:
p.errorf("expected bool, list, or string value; found %s",
scanner.TokenString(p.tok))
return
}
}
// Type =
// BasicType | TypeName | ArrayType | SliceType | StructType |
// PointerType | FuncType | InterfaceType | MapType | ChanType |
// "(" Type ")" .
//
// BasicType = ident .
// TypeName = ExportedName .
// SliceType = "[" "]" Type .
// PointerType = "*" Type .
// FuncType = "func" Signature .
//
func (p *parser) parseType(parent *types.Package) types.Type {
switch p.tok {
case scanner.Ident:
switch p.lit {
default:
return p.parseBasicType()
case "struct":
return p.parseStructType(parent)
case "func":
// FuncType
p.next()
return p.parseSignature(nil)
case "interface":
return p.parseInterfaceType(parent)
case "map":
return p.parseMapType(parent)
case "chan":
return p.parseChanType(parent)
}
case '@':
// TypeName
pkg, name := p.parseExportedName()
return declTypeName(pkg, name).Type()
case '[':
p.next() // look ahead
if p.tok == ']' {
// SliceType
p.next()
return types.NewSlice(p.parseType(parent))
}
return p.parseArrayType(parent)
case '*':
// PointerType
p.next()
return types.NewPointer(p.parseType(parent))
case '<':
return p.parseChanType(parent)
case '(':
// "(" Type ")"
p.next()
typ := p.parseType(parent)
p.expect(')')
return typ
}
p.errorf("expected type, got %s (%q)", scanner.TokenString(p.tok), p.lit)
return nil
}
// Type =
// BasicType | TypeName | ArrayType | SliceType | StructType |
// PointerType | FuncType | InterfaceType | MapType | ChanType |
// "(" Type ")" .
// BasicType = ident .
// TypeName = ExportedName .
// SliceType = "[" "]" Type .
// PointerType = "*" Type .
// FuncType = "func" Signature .
//
func (p *gcParser) parseType() Type {
switch p.tok {
case scanner.Ident:
switch p.lit {
default:
return p.parseBasicType()
case "struct":
return p.parseStructType()
case "func":
// FuncType
p.next()
return p.parseSignature()
case "interface":
return p.parseInterfaceType()
case "map":
return p.parseMapType()
case "chan":
return p.parseChanType()
}
case scanner.String:
// TypeName
return p.parseExportedName(ast.Typ).Type.(Type)
case '[':
p.next() // look ahead
if p.tok == ']' {
// SliceType
p.next()
return &Slice{Elt: p.parseType()}
}
return p.parseArrayType()
case '*':
// PointerType
p.next()
return &Pointer{Base: p.parseType()}
case '<':
return p.parseChanType()
case '(':
// "(" Type ")"
p.next()
typ := p.parseType()
p.expect(')')
return typ
}
p.errorf("expected type, got %s (%q)", scanner.TokenString(p.tok), p.lit)
return nil
}
func (p *Parser) Parse() error {
tok := p.s.Scan()
if tok == scanner.EOF {
return nil
}
var err error
switch tok {
case '{':
err = p.parseObject()
case '[':
err = p.parseArray()
default:
err = p.createError("expected object or array, but got %s",
scanner.TokenString(tok))
}
return err
}