func testStringVsTokens(t *testing.T, str string, toks []*tstTok) {
tb := NewTokenBuffer(lexer.NewLexer(srcbuf.NewSourceFromBuffer(strings.Bytes(str))))
var tok common.Token
var i int
for tok, i = tb.GetToken(), 0; tok.Type() != common.TOK_EOF && i < len(toks); tok, i = tb.GetToken(), i+1 {
switch typ := tok.(type) {
case *lexer.IntTok:
if toks[i].typ != typ.Type() {
t.Errorf("%d: Expected token type %v, but got: %v.\n", i, toks[i].typ, typ.Type())
}
if toks[i].numVal != typ.Value() {
t.Errorf("%d: Expected value %v, but got: %v.\n", i, toks[i].numVal, typ.Value())
}
case *lexer.CharTok:
if toks[i].typ != typ.Type() {
t.Errorf("%d: Expected token type %v, but got: %v.\n", i, toks[i].typ, typ.Type())
}
if toks[i].numVal != int64(typ.Value()) {
t.Errorf("%d: Expected value %v, but got: %v.\n", i, toks[i].numVal, typ.Value())
}
case *lexer.StringTok:
if toks[i].typ != typ.Type() {
t.Errorf("%d: Expected token type %v, but got: %v.\n", i, toks[i].typ, typ.Type())
}
if toks[i].strVal != typ.Value() {
t.Errorf("%d: Expected value %v, but got: %v.\n", i, toks[i].strVal, typ.Value())
}
case *lexer.SpaceTok:
if toks[i].typ != typ.Type() {
t.Errorf("%d: Expected token type %v, but got: %v.\n", i, toks[i].typ, typ.Type())
}
space := int64(typ.Space())
if typ.AtStartOfLine() {
space += 1000
}
if toks[i].numVal != space {
t.Errorf("%d: Expected space %v, but got: %v.\n", i, toks[i].numVal, space)
}
default:
if toks[i].typ != typ.Type() {
t.Errorf("%d: Expected token type %v, but got: %v.\n", i, toks[i].typ, typ.Type())
}
if toks[i].checkContent && toks[i].content != typ.Content() {
t.Errorf("%d: Expected content %v, but got: %v.\n", i, toks[i].content, typ.Content())
}
}
}
if i != len(toks) {
t.Error("Got too few tokens!")
}
if tok.Type() != common.TOK_EOF {
t.Error("Got too many tokens!")
}
}
func main() {
flag.Parse() // Scans the arg list and sets up flags
var sb common.SrcBuffer
// fill the source buffer either from the command line or from a file:
if *useCommandLine {
if flag.NArg() <= 0 {
fmt.Fprintln(os.Stderr, "FATAL ERROR: Need source line(s) as argument(s)!")
os.Exit(1)
}
srcStr := ""
for i := 0; i < flag.NArg(); i++ {
srcStr += flag.Arg(i) + "\n"
}
sb = srcbuf.NewSourceFromBuffer(strings.Bytes(srcStr), "command line")
} else {
if flag.NArg() <= 0 {
fmt.Fprintln(os.Stderr, "FATAL ERROR: Need name of source file as argument!")
os.Exit(1)
}
// Initialize the source buffer:
var err os.Error
sb, err = srcbuf.NewSourceFromFile(flag.Arg(0))
if err != nil {
fmt.Fprintf(os.Stderr, "FATAL ERROR: Unable to read source file '%s': %s\n",
flag.Arg(0), err)
os.Exit(1)
}
}
//for ch := sb.Getch(); ch != common.EOF; ch = sb.Getch() {
// fmt.Println("Found char:", ch, string(ch));
//}
// Initialize the lexer:
lx := lexer.NewLexer(sb)
// Test output:
for tok := lx.GetToken(); tok.Type() != common.TOK_EOF; tok = lx.GetToken() {
switch t := tok.(type) {
case *lexer.IntTok:
fmt.Println("Got int:", t.Value(), t)
default:
fmt.Println("Got token:", t.Type(), t)
}
}
}