Example #1
0
func ExampleParseFile() {
	tokens, err := lexer.ParseFile("../../testdata/incorrect/lexer/long-char.c")
	if err != nil {
		log.Fatal(err)
	}
	for _, tok := range tokens {
		fmt.Printf("%-9v   %q\n", tok.Kind.GoString(), tok.Val)
	}
	// Output:
	// Ident       "int"
	// Ident       "main"
	// Lparen      "("
	// Ident       "void"
	// Rparen      ")"
	// Lbrace      "{"
	// Ident       "char"
	// Ident       "c"
	// Semicolon   ";"
	// Ident       "c"
	// Assign      "="
	// CharLit     "'c'"
	// Semicolon   ";"
	// Comment     "// OK"
	// Ident       "c"
	// Assign      "="
	// Error       "unterminated character literal"
	// Ident       "cc"
	// Error       "unterminated character literal"
	// Semicolon   ";"
	// Comment     "// Not OK"
	// Rbrace      "}"
	// EOF         ""
}
Example #2
0
File: scanner.go Project: mewmew/uc
// Open returns a new scanner lexing from path.
func Open(path string) (Scanner, error) {
	toks, err := lexer.ParseFile(path)
	if err != nil {
		return nil, err
	}
	return &scanner{toks: toks}, nil
}
Example #3
0
File: ulex.go Project: mewmew/uc
// lexFileHand lexes the given file and pretty-prints the n first tokens to
// standard output, using the hand-written lexer.
func lexFileHand(path string, n int) (err error) {
	var toks []token.Token
	if path == "-" {
		fmt.Fprintln(os.Stderr, "Lexing from standard input")
		toks, err = lexer.Parse(os.Stdin)
	} else {
		fmt.Fprintf(os.Stderr, "Lexing %q\n", path)
		toks, err = lexer.ParseFile(path)
	}
	if err != nil {
		return errutil.Err(err)
	}

	ntoks := len(toks)
	if n > ntoks {
		ntoks = n
	}
	pad := int(math.Ceil(math.Log10(float64(ntoks))))
	for i, tok := range toks {
		if n != 0 && i == n {
			break
		}
		if tok.Kind == token.Error {
			elog.Printf("ERROR %*d:   %v\n", pad, i, tok)
		} else {
			fmt.Printf("token %*d:   %v\n", pad, i, tok)
		}
	}
	fmt.Fprintln(os.Stderr)
	return nil
}