func Token2eof(tok common.Token) *EofTok {
et, ok := tok.(*EofTok)
if !ok {
tok.Error("Not an EOF token")
}
return et
}
func Token2int(tok common.Token) *IntTok {
it, ok := tok.(*IntTok)
if !ok {
tok.Error("Not an integer token")
}
return it
}
func Token2operator(tok common.Token) *OperatorTok {
ot, ok := tok.(*OperatorTok)
if !ok {
tok.Error("Not an operator token")
}
return ot
}
func Token2simple(tok common.Token) *SimpleToken {
st, ok := tok.(*SimpleToken)
if !ok {
tok.Error("Not a simple token")
}
return st
}
func Token2id(tok common.Token) *IdTok {
it, ok := tok.(*IdTok)
if !ok {
tok.Error("Not an ID token")
}
return it
}
func Token2string(tok common.Token) *StringTok {
st, ok := tok.(*StringTok)
if !ok {
tok.Error("Not a string token")
}
return st
}
func Token2space(tok common.Token) *SpaceTok {
st, ok := tok.(*SpaceTok)
if !ok {
tok.Error("Not a space token")
}
return st
}
func Token2char(tok common.Token) *CharTok {
ct, ok := tok.(*CharTok)
if !ok {
tok.Error("Not a character token")
}
return ct
}
func testStringVsTokens(t *testing.T, str string, toks []*tstTok) {
lx := NewLexer(srcbuf.NewSourceFromBuffer(strings.Bytes(str)))
var tok common.Token
var i int
for tok, i = lx.GetToken(), 0; tok.Type() != common.TOK_EOF && i < len(toks); tok, i = lx.GetToken(), i+1 {
switch typ := tok.(type) {
case *IntTok:
if toks[i].typ != typ.Type() {
t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
}
if toks[i].numVal != typ.Value() {
t.Errorf("Expected value %v, but got: %v.\n", toks[i].numVal, typ.Value())
}
case *CharTok:
if toks[i].typ != typ.Type() {
t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
}
if toks[i].numVal != int64(typ.Value()) {
t.Errorf("Expected value %v, but got: %v.\n", toks[i].numVal, typ.Value())
}
case *StringTok:
if toks[i].typ != typ.Type() {
t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
}
if toks[i].strVal != typ.Value() {
t.Errorf("Expected value %v, but got: %v.\n", toks[i].strVal, typ.Value())
}
case *SpaceTok:
if toks[i].typ != typ.Type() {
t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
}
space := int64(typ.Space())
if typ.AtStartOfLine() {
space += 1000
}
if toks[i].numVal != space {
t.Errorf("Expected space %v, but got: %v.\n", toks[i].numVal, space)
}
default:
if toks[i].typ != typ.Type() {
t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
}
if toks[i].checkContent && toks[i].content != typ.Content() {
t.Errorf("Expected content %v, but got: %v.\n", toks[i].content, typ.Content())
}
}
}
if i != len(toks) {
t.Error("Got too few tokens!")
}
if tok.Type() != common.TOK_EOF {
t.Error("Got too many tokens!")
}
}
// special handling of EOF (so we have valid code if possible)
func handleEof(tok common.Token, tb *tokBuf) bool {
if tok.Type() == common.TOK_EOF {
if tb.indentLevel > 0 {
handleIndent(tb.lx.NewSpaceTok(tok, 0, true), tok, tb);
} else {
tb.curTok = tb.tokBuf.PushBack(tok);
}
return true;
}
return false;
}
func handleSpace(tok common.Token, tb *tokBuf) bool {
if tok.Type() == common.TOK_SPACE {
spaceTok := lexer.Token2space(tok);
if spaceTok.AtStartOfLine() {
handlePossibleIndent(tok, tb);
} else {
tb.curTok = tb.tokBuf.PushBack(tok);
}
return true;
}
return false;
}
func recordIndent(indent int, tok common.Token, tb *tokBuf) {
// we can have half indentations (2 spaces) and
// full indentations (4 spaces)
switch indent {
case 2:
tb.indentLevel++;
tb.curTok = tb.tokBuf.PushBack(tb.lx.NewCopyTok(common.TOK_HALF_INDENT, tok));
case 4:
tb.indentLevel += 2;
tb.curTok = tb.tokBuf.PushBack(tb.lx.NewCopyTok(common.TOK_INDENT, tok));
default:
tok.Error("Indentation error");
}
}
func handleColon(tok common.Token, tb *tokBuf) bool {
if tok.Type() == common.TOK_COLON {
curTok := tb.tokBuf.PushBack(tok);
tok2 := tb.getFilteredToken(); // whats behind the colon?
if tok2.Type() != common.TOK_NL {
// No TOK_NL: just 2 normal tokens then
tb.tokBuf.PushBack(tok2);
} else {
curTok.Value = tb.lx.NewAnyTok(common.TOK_BLOCK_START,
tok.SourcePiece().Start(), tok2.SourcePiece().End());
}
tb.curTok = curTok;
return true;
}
return false;
}
func (lx *Lexer) NewCopyTok(typ common.TokEnum, tok common.Token) common.Token {
return &SimpleToken{typ, tok.SourcePiece()}
}
// remove old lines from the underlying source buffer
func (tb *tokBuf) ClearUpTo(tok common.Token) {
tb.lx.ClearUpTo(tok.SourcePiece().Start());
}
func recordDedent(dedent int, tok common.Token, tb *tokBuf) {
if dedent & 1 != 0 { tok.Error("Uneven dedentation error"); }
recordOtherDedents(recordFirstDedent(dedent/2, tok, tb), tok, tb);
}