Example #1
0
// Scan returns a Token.
func (scanner *MultiCharacterScanner) Scan() (token.Token, error) {
	for index, tokenRune := range scanner.TokenKind {
		var (
			firstRune rune
			err       error
		)

		// The first rune can be t of '     true'.
		// So this uses NextNonWhiteSpaceRune for first rune.
		// But following runes must be continuous.
		// e.g.
		//		'    true' is valid.
		//		'  t ru e' is invalid.
		if index == 0 {
			firstRune, err = scanner.Internal.NextNonWhiteSpaceRune()
		} else {
			firstRune, err = scanner.Internal.NextRune()
		}

		if err != nil {
			return token.NewEmptyToken(), err
		}
		if tokenRune != firstRune {
			return token.NewEmptyToken(), fmt.Errorf("False index at %d is %#v. expected %#v", index, firstRune, tokenRune)
		}
	}

	return token.Token{Kind: scanner.TokenKind, Value: string(scanner.TokenKind)}, nil
}
Example #2
0
// Scan returns a Token.
func (scanner *SingleCharacterScanner) Scan() (token.Token, error) {
	targetRune, err := scanner.Internal.NextNonWhiteSpaceRune()
	if err != nil {
		return token.NewEmptyToken(), err
	}
	if !scanner.CanProcessRune(targetRune) {
		return token.NewEmptyToken(), fmt.Errorf("SingleCharacter must start with '%s' actual='%s'", scanner.TokenKind, string(targetRune))
	}
	return token.Token{Kind: scanner.TokenKind, Value: string(scanner.TokenKind)}, nil
}
Example #3
0
// Scan returns a Token.
func (scanner *StringScanner) Scan() (token.Token, error) {
	firstRune, err := scanner.Internal.NextNonWhiteSpaceRune()
	if err != nil {
		return token.NewEmptyToken(), err
	}
	if !scanner.CanProcessRune(firstRune) {
		return token.NewEmptyToken(), fmt.Errorf("Token[String] must start with \", actual=%s", string(firstRune))
	}

	value, err := ScanString(scanner.Internal)
	return token.Token{Kind: token.KindString, Value: value}, nil
}
Example #4
0
// Scan returns a Token.
func (scanner *NumberScanner) Scan() (token.Token, error) {
	firstRune, err := scanner.Internal.NextNonWhiteSpaceRune()
	if err != nil {
		return token.NewEmptyToken(), err
	}
	if !scanner.CanProcessRune(firstRune) {
		return token.NewEmptyToken(), fmt.Errorf("Token[Number] must start with digit or '-', actual=%s", string(firstRune))
	}

	value, err := ScanNumber(scanner.Internal, firstRune)
	if err != nil {
		return token.NewEmptyToken(), err
	}
	return token.Token{Kind: token.KindNumber, Value: value}, nil
}
Example #5
0
// PeekToken returns a token at index from current position.
func (tokenizer *Tokenizer) PeekToken(index int) (token.Token, error) {
	err := tokenizer.queueTokensTo(index)
	if err == io.EOF {
		return token.NewTokenEOF(), nil
	}
	if err != nil {
		return token.NewEmptyToken(), err
	}
	return tokenizer.buffer.peek(index), nil
}