示例#1
0
文件: lexer_test.go 项目: mewmew/uc
func BenchmarkLexer(b *testing.B) {
	buf, err := ioutil.ReadFile("../../testdata/noisy/advanced/eval.c")
	if err != nil {
		b.Fatal(err)
	}
	b.SetBytes(int64(len(buf)))
	for i := 0; i < b.N; i++ {
		l := lexer.NewLexer(buf)
		for {
			tok := l.Scan()
			if tok.Type == token.EOF {
				break
			}
		}
	}
}
示例#2
0
文件: scanner.go 项目: mewmew/uc
// NewFromBytes returns a new scanner lexing from input.
func NewFromBytes(input []byte) Scanner {
	// Append new line to files not ending with new line.
	appendNewLine := false
	lastNewLine := bytes.LastIndexByte(input, '\n')
	for _, r := range string(input[lastNewLine+1:]) {
		if !strings.ContainsRune(whitespace, r) {
			// Non-whitespace character located after the last new line character.
			appendNewLine = true
			break
		}
	}
	if appendNewLine {
		input = append(input, '\n')
	}

	return lexer.NewLexer(input)
}