func lex(program *bytes.Reader) (string, int) {
lexeme := make([]rune, 0, 128)
index := 0
//Get next Non-Whitespace Character
char, _, err := program.ReadRune()
for charClass(char) == WHITESPACE {
errCheck(err)
char, _, err = program.ReadRune()
}
switch charClass(char) {
case LETTER: //IDENT or KEYWORD
for ; charClass(char) == LETTER || charClass(char) == DIGIT; char, _, err = program.ReadRune() {
errCheck(err)
lexeme = lexeme[:index+1]
lexeme[index] = char
index++
}
program.UnreadRune()
result := string(lexeme)
return result, getKeyword(result)
case DIGIT:
for ; charClass(char) == DIGIT; char, _, err = program.ReadRune() {
errCheck(err)
lexeme = lexeme[:index+1]
lexeme[index] = char
index++
}
program.UnreadRune()
return string(lexeme), DIGIT
}
lexeme = lexeme[:index+1]
lexeme[index] = char
return string(lexeme), charClass(char)
}