Beispiel #1
0
// errorf returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.nextItem.
func (l *lexer) errorf(format string, args ...interface{}) LexFn {
	l.tokens <- token.NewToken(token.ILLEGAL, l.start, fmt.Sprintf(format, args...))
	return nil
}
Beispiel #2
0
/*func (i token.TokenType) String() string {
	s := tokenName[i]
	if s == "" {
		return fmt.Sprintf("token%d", int(i))
	}
	return s
}*/

type lexTest struct {
	name   string
	input  string
	tokens []token.Token
}

var (
	tEOF    = token.NewToken(token.EOF, 0, "")
	tLParen = token.NewToken(token.LPAREN, 0, "(")
	tRParen = token.NewToken(token.RPAREN, 0, ")")
)

var lexTests = []lexTest{
	{"empty", "", []token.Token{tEOF}},
	{"number", "2", []token.Token{
		token.NewToken(token.INT, 0, "2"),
		token.NewToken(token.EOF, 0, ""),
	}},
	{"add", "2 + 2", []token.Token{
		token.NewToken(token.INT, 0, "2"),
		token.NewToken(token.ADD, 0, "+"),
		token.NewToken(token.INT, 0, "2"),
		tEOF,
Beispiel #3
0
// emit passes an item back to the client.
func (l *lexer) emit(t token.TokenType) {
	l.tokens <- token.NewToken(t, l.start, l.input[l.start:l.pos])
	l.start = l.pos
}