예제 #1
0
func makeNewLineTok(lx *Lexer, mark common.SrcMark) common.Token {
	tok := common.Token(nil)
	if lx.inParens <= 0 {
		tok = &SimpleToken{common.TOK_NL, lx.srcBuf.NewPiece(mark)}
	}
	return tok
}
예제 #2
0
func (lx *Lexer) GetToken() common.Token {
  tok := common.Token(nil);
  lxFuncs := []lexFunc{
      trySpace, tryEof, tryComment, tryNewLine, trySemicolon, tryColon, tryParen,
      tryNumber, tryOperator, tryId, tryChar, tryString, signalUndefined
  };

  tok = lx.getFirstTok(lxFuncs);
//fmt.Println(">>> Got token:", tok.Type(), tok);

  return tok;
}
예제 #3
0
func (lx *Lexer) getFirstTok(lxFuncs []lexFunc) common.Token {
  tok := common.Token(nil);
  moved := false;

  // move until a token is found:
  N := len(lxFuncs);
  for tok == nil {
    moved = false;
    for i := 0; !moved && i < N; i++ {
      tok, moved = lxFuncs[i](lx);
    }
  }
  return tok;
}