Пример #1
0
func lex(program *bytes.Reader) (string, int) {
	lexeme := make([]rune, 0, 128)
	index := 0

	//Get next Non-Whitespace Character
	char, _, err := program.ReadRune()
	for charClass(char) == WHITESPACE {
		errCheck(err)
		char, _, err = program.ReadRune()
	}

	switch charClass(char) {
	case LETTER: //IDENT or KEYWORD
		for ; charClass(char) == LETTER || charClass(char) == DIGIT; char, _, err = program.ReadRune() {
			errCheck(err)
			lexeme = lexeme[:index+1]
			lexeme[index] = char
			index++
		}
		program.UnreadRune()
		result := string(lexeme)
		return result, getKeyword(result)
	case DIGIT:
		for ; charClass(char) == DIGIT; char, _, err = program.ReadRune() {
			errCheck(err)
			lexeme = lexeme[:index+1]
			lexeme[index] = char
			index++
		}
		program.UnreadRune()
		return string(lexeme), DIGIT
	}
	lexeme = lexeme[:index+1]
	lexeme[index] = char
	return string(lexeme), charClass(char)
}