// Scan returns a Token.
func (scanner *MultiCharacterScanner) Scan() (token.Token, error) {
for index, tokenRune := range scanner.TokenKind {
var (
firstRune rune
err error
)
// The first rune can be t of ' true'.
// So this uses NextNonWhiteSpaceRune for first rune.
// But following runes must be continuous.
// e.g.
// ' true' is valid.
// ' t ru e' is invalid.
if index == 0 {
firstRune, err = scanner.Internal.NextNonWhiteSpaceRune()
} else {
firstRune, err = scanner.Internal.NextRune()
}
if err != nil {
return token.NewEmptyToken(), err
}
if tokenRune != firstRune {
return token.NewEmptyToken(), fmt.Errorf("False index at %d is %#v. expected %#v", index, firstRune, tokenRune)
}
}
return token.Token{Kind: scanner.TokenKind, Value: string(scanner.TokenKind)}, nil
}
// Scan returns a Token.
func (scanner *SingleCharacterScanner) Scan() (token.Token, error) {
targetRune, err := scanner.Internal.NextNonWhiteSpaceRune()
if err != nil {
return token.NewEmptyToken(), err
}
if !scanner.CanProcessRune(targetRune) {
return token.NewEmptyToken(), fmt.Errorf("SingleCharacter must start with '%s' actual='%s'", scanner.TokenKind, string(targetRune))
}
return token.Token{Kind: scanner.TokenKind, Value: string(scanner.TokenKind)}, nil
}
// Scan returns a Token.
func (scanner *StringScanner) Scan() (token.Token, error) {
firstRune, err := scanner.Internal.NextNonWhiteSpaceRune()
if err != nil {
return token.NewEmptyToken(), err
}
if !scanner.CanProcessRune(firstRune) {
return token.NewEmptyToken(), fmt.Errorf("Token[String] must start with \", actual=%s", string(firstRune))
}
value, err := ScanString(scanner.Internal)
return token.Token{Kind: token.KindString, Value: value}, nil
}
// Scan returns a Token.
func (scanner *NumberScanner) Scan() (token.Token, error) {
firstRune, err := scanner.Internal.NextNonWhiteSpaceRune()
if err != nil {
return token.NewEmptyToken(), err
}
if !scanner.CanProcessRune(firstRune) {
return token.NewEmptyToken(), fmt.Errorf("Token[Number] must start with digit or '-', actual=%s", string(firstRune))
}
value, err := ScanNumber(scanner.Internal, firstRune)
if err != nil {
return token.NewEmptyToken(), err
}
return token.Token{Kind: token.KindNumber, Value: value}, nil
}
// PeekToken returns a token at index from current position.
func (tokenizer *Tokenizer) PeekToken(index int) (token.Token, error) {
err := tokenizer.queueTokensTo(index)
if err == io.EOF {
return token.NewTokenEOF(), nil
}
if err != nil {
return token.NewEmptyToken(), err
}
return tokenizer.buffer.peek(index), nil
}