func BenchmarkLexer(b *testing.B) {
buf, err := ioutil.ReadFile("../../testdata/noisy/advanced/eval.c")
if err != nil {
b.Fatal(err)
}
b.SetBytes(int64(len(buf)))
for i := 0; i < b.N; i++ {
l := lexer.NewLexer(buf)
for {
tok := l.Scan()
if tok.Type == token.EOF {
break
}
}
}
}
// NewFromBytes returns a new scanner lexing from input.
func NewFromBytes(input []byte) Scanner {
// Append new line to files not ending with new line.
appendNewLine := false
lastNewLine := bytes.LastIndexByte(input, '\n')
for _, r := range string(input[lastNewLine+1:]) {
if !strings.ContainsRune(whitespace, r) {
// Non-whitespace character located after the last new line character.
appendNewLine = true
break
}
}
if appendNewLine {
input = append(input, '\n')
}
return lexer.NewLexer(input)
}