Пример #1
0
func testStringVsTokens(t *testing.T, str string, toks []*tstTok) {
	lx := NewLexer(srcbuf.NewSourceFromBuffer(strings.Bytes(str)))
	var tok common.Token
	var i int
	for tok, i = lx.GetToken(), 0; tok.Type() != common.TOK_EOF && i < len(toks); tok, i = lx.GetToken(), i+1 {
		switch typ := tok.(type) {
		case *IntTok:
			if toks[i].typ != typ.Type() {
				t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
			}
			if toks[i].numVal != typ.Value() {
				t.Errorf("Expected value %v, but got: %v.\n", toks[i].numVal, typ.Value())
			}
		case *CharTok:
			if toks[i].typ != typ.Type() {
				t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
			}
			if toks[i].numVal != int64(typ.Value()) {
				t.Errorf("Expected value %v, but got: %v.\n", toks[i].numVal, typ.Value())
			}
		case *StringTok:
			if toks[i].typ != typ.Type() {
				t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
			}
			if toks[i].strVal != typ.Value() {
				t.Errorf("Expected value %v, but got: %v.\n", toks[i].strVal, typ.Value())
			}
		case *SpaceTok:
			if toks[i].typ != typ.Type() {
				t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
			}
			space := int64(typ.Space())
			if typ.AtStartOfLine() {
				space += 1000
			}
			if toks[i].numVal != space {
				t.Errorf("Expected space %v, but got: %v.\n", toks[i].numVal, space)
			}
		default:
			if toks[i].typ != typ.Type() {
				t.Errorf("Expected token type %v, but got: %v.\n", toks[i].typ, typ.Type())
			}
			if toks[i].checkContent && toks[i].content != typ.Content() {
				t.Errorf("Expected content %v, but got: %v.\n", toks[i].content, typ.Content())
			}
		}
	}
	if i != len(toks) {
		t.Error("Got too few tokens!")
	}
	if tok.Type() != common.TOK_EOF {
		t.Error("Got too many tokens!")
	}
}
Пример #2
0
// special handling of EOF (so we have valid code if possible)
func handleEof(tok common.Token, tb *tokBuf) bool {
  if tok.Type() == common.TOK_EOF {
    if tb.indentLevel > 0 {
      handleIndent(tb.lx.NewSpaceTok(tok, 0, true), tok, tb);
    } else {
      tb.curTok = tb.tokBuf.PushBack(tok);
    }
    return true;
  }
  return false;
}
Пример #3
0
func handleSpace(tok common.Token, tb *tokBuf) bool {
  if tok.Type() == common.TOK_SPACE {
    spaceTok := lexer.Token2space(tok);
    if spaceTok.AtStartOfLine() {
      handlePossibleIndent(tok, tb);
    } else {
      tb.curTok = tb.tokBuf.PushBack(tok);
    }
    return true;
  }
  return false;
}
Пример #4
0
func handleColon(tok common.Token, tb *tokBuf) bool {
  if tok.Type() == common.TOK_COLON {
    curTok := tb.tokBuf.PushBack(tok);
    tok2 := tb.getFilteredToken();  // whats behind the colon?
    if tok2.Type() != common.TOK_NL {
      // No TOK_NL: just 2 normal tokens then
      tb.tokBuf.PushBack(tok2);
    } else {
      curTok.Value = tb.lx.NewAnyTok(common.TOK_BLOCK_START,
          tok.SourcePiece().Start(), tok2.SourcePiece().End());
    }
    tb.curTok = curTok;
    return true;
  }
  return false;
}