func testStringVsTokens(t *testing.T, str string, toks []*tstTok) { lx := NewLexer(srcbuf.NewSourceFromBuffer(strings.Bytes(str))) var tok common.Token var i int for tok, i = lx.GetToken(), 0; tok.Type() != common.TOK_EOF && i < len(toks); tok, i = lx.GetToken(), i+1 { switch typ := tok.(type) { case *IntTok: if toks[i].typ != typ.Type() { t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type()) } if toks[i].numVal != typ.Value() { t.Errorf("Expected value %v, but got: %v.\n", toks[i].numVal, typ.Value()) } case *CharTok: if toks[i].typ != typ.Type() { t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type()) } if toks[i].numVal != int64(typ.Value()) { t.Errorf("Expected value %v, but got: %v.\n", toks[i].numVal, typ.Value()) } case *StringTok: if toks[i].typ != typ.Type() { t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type()) } if toks[i].strVal != typ.Value() { t.Errorf("Expected value %v, but got: %v.\n", toks[i].strVal, typ.Value()) } case *SpaceTok: if toks[i].typ != typ.Type() { t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type()) } space := int64(typ.Space()) if typ.AtStartOfLine() { space += 1000 } if toks[i].numVal != space { t.Errorf("Expected space %v, but got: %v.\n", toks[i].numVal, space) } default: if toks[i].typ != typ.Type() { t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type()) } if toks[i].checkContent && toks[i].content != typ.Content() { t.Errorf("Expected content %v, but got: %v.\n", toks[i].content, typ.Content()) } } } if i != len(toks) { t.Error("Got too few tokens!") } if tok.Type() != common.TOK_EOF { t.Error("Got too many tokens!") } }
// special handling of EOF (so we have valid code if possible) func handleEof(tok common.Token, tb *tokBuf) bool { if tok.Type() == common.TOK_EOF { if tb.indentLevel > 0 { handleIndent(tb.lx.NewSpaceTok(tok, 0, true), tok, tb); } else { tb.curTok = tb.tokBuf.PushBack(tok); } return true; } return false; }
func handleSpace(tok common.Token, tb *tokBuf) bool { if tok.Type() == common.TOK_SPACE { spaceTok := lexer.Token2space(tok); if spaceTok.AtStartOfLine() { handlePossibleIndent(tok, tb); } else { tb.curTok = tb.tokBuf.PushBack(tok); } return true; } return false; }
func handleColon(tok common.Token, tb *tokBuf) bool { if tok.Type() == common.TOK_COLON { curTok := tb.tokBuf.PushBack(tok); tok2 := tb.getFilteredToken(); // whats behind the colon? if tok2.Type() != common.TOK_NL { // No TOK_NL: just 2 normal tokens then tb.tokBuf.PushBack(tok2); } else { curTok.Value = tb.lx.NewAnyTok(common.TOK_BLOCK_START, tok.SourcePiece().Start(), tok2.SourcePiece().End()); } tb.curTok = curTok; return true; } return false; }