func testStringVsTokens(t *testing.T, str string, toks []*tstTok) {
lx := NewLexer(srcbuf.NewSourceFromBuffer(strings.Bytes(str)))
var tok common.Token
var i int
for tok, i = lx.GetToken(), 0; tok.Type() != common.TOK_EOF && i < len(toks); tok, i = lx.GetToken(), i+1 {
switch typ := tok.(type) {
case *IntTok:
if toks[i].typ != typ.Type() {
t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
}
if toks[i].numVal != typ.Value() {
t.Errorf("Expected value %v, but got: %v.\n", toks[i].numVal, typ.Value())
}
case *CharTok:
if toks[i].typ != typ.Type() {
t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
}
if toks[i].numVal != int64(typ.Value()) {
t.Errorf("Expected value %v, but got: %v.\n", toks[i].numVal, typ.Value())
}
case *StringTok:
if toks[i].typ != typ.Type() {
t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
}
if toks[i].strVal != typ.Value() {
t.Errorf("Expected value %v, but got: %v.\n", toks[i].strVal, typ.Value())
}
case *SpaceTok:
if toks[i].typ != typ.Type() {
t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
}
space := int64(typ.Space())
if typ.AtStartOfLine() {
space += 1000
}
if toks[i].numVal != space {
t.Errorf("Expected space %v, but got: %v.\n", toks[i].numVal, space)
}
default:
if toks[i].typ != typ.Type() {
t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
}
if toks[i].checkContent && toks[i].content != typ.Content() {
t.Errorf("Expected content %v, but got: %v.\n", toks[i].content, typ.Content())
}
}
}
if i != len(toks) {
t.Error("Got too few tokens!")
}
if tok.Type() != common.TOK_EOF {
t.Error("Got too many tokens!")
}
}
// special handling of EOF (so we have valid code if possible)
func handleEof(tok common.Token, tb *tokBuf) bool {
if tok.Type() == common.TOK_EOF {
if tb.indentLevel > 0 {
handleIndent(tb.lx.NewSpaceTok(tok, 0, true), tok, tb);
} else {
tb.curTok = tb.tokBuf.PushBack(tok);
}
return true;
}
return false;
}
func handleSpace(tok common.Token, tb *tokBuf) bool {
if tok.Type() == common.TOK_SPACE {
spaceTok := lexer.Token2space(tok);
if spaceTok.AtStartOfLine() {
handlePossibleIndent(tok, tb);
} else {
tb.curTok = tb.tokBuf.PushBack(tok);
}
return true;
}
return false;
}
func handleColon(tok common.Token, tb *tokBuf) bool {
if tok.Type() == common.TOK_COLON {
curTok := tb.tokBuf.PushBack(tok);
tok2 := tb.getFilteredToken(); // whats behind the colon?
if tok2.Type() != common.TOK_NL {
// No TOK_NL: just 2 normal tokens then
tb.tokBuf.PushBack(tok2);
} else {
curTok.Value = tb.lx.NewAnyTok(common.TOK_BLOCK_START,
tok.SourcePiece().Start(), tok2.SourcePiece().End());
}
tb.curTok = curTok;
return true;
}
return false;
}