// Stat calculates statistics for all runes read from r.
func (m *Main) Stat(r io.RuneReader) (Stats, error) {
var stats Stats
for {
// Read next character.
ch, sz, err := r.ReadRune()
if err == io.EOF {
break
} else if err != nil {
return stats, err
}
// Calculate stats.
stats.TotalN++
if unicode.IsControl(ch) {
stats.ControlN++
}
if unicode.IsDigit(ch) {
stats.DigitN++
}
if unicode.IsGraphic(ch) {
stats.GraphicN++
}
if unicode.IsLetter(ch) {
stats.LetterN++
}
if unicode.IsLower(ch) {
stats.LowerN++
}
if unicode.IsMark(ch) {
stats.MarkN++
}
if unicode.IsNumber(ch) {
stats.NumberN++
}
if unicode.IsPrint(ch) {
stats.PrintN++
}
if unicode.IsPunct(ch) {
stats.PunctN++
}
if unicode.IsSpace(ch) {
stats.SpaceN++
}
if unicode.IsSymbol(ch) {
stats.SymbolN++
}
if unicode.IsTitle(ch) {
stats.TitleN++
}
if unicode.IsUpper(ch) {
stats.UpperN++
}
if sz > 1 {
stats.MultiByteN++
}
}
return stats, nil
}
// CharCount reads input and returns stats about unicode frequency
func CharCount(in io.RuneReader) UnicodeStat {
var results UnicodeStat
results.Count = make(map[rune]int) // counts of Unicode characters
results.UtfLen = make(map[int]int) // count of lengths of UTF-8 encodings
results.Category = make(map[string]int) // number of categorys like Unicode.IsNumber
results.Invalid = 0 // count of invalid UTF-8 characters
for {
r, n, err := in.ReadRune() // returns rune, nbytes, error
if err == io.EOF {
break
}
if err != nil {
fmt.Fprintf(os.Stderr, "charcount: %v\n", err)
os.Exit(1)
}
if r == unicode.ReplacementChar && n == 1 {
results.Invalid++
continue
}
results.Count[r]++
results.Category[CharType(r)]++
results.UtfLen[n]++
}
return results
}
func Draw(r io.RuneReader, f font.Face, dst draw.Image, src image.Image, bounds fixed.Rectangle26_6, dir Direction) (err error) {
var prev rune
var char rune
var origin = dir.Origin(f, bounds)
var dot = origin
var next fixed.Point26_6
for {
if char, _, err = r.ReadRune(); err != nil {
if err == io.EOF {
err = nil
}
return
}
dot, next = dir.Advance(f, origin, dot, prev, char)
if char != '\n' {
dr, mask, maskp := glyph(f, char, dot)
draw.DrawMask(dst, dr, src, dr.Min, mask, maskp, draw.Over)
}
dot, prev = next, char
}
}
func Measure(r io.RuneReader, f font.Face) (size fixed.Point26_6, err error) {
var char rune
var prev rune
w := fixed.Int26_6(0)
m := f.Metrics()
size.Y = m.Height + m.Descent
for {
if char, _, err = r.ReadRune(); err != nil {
if err == io.EOF {
err = nil
}
size.X = maxInt26_6(size.X, w)
return
}
if char == '\n' {
size.X = maxInt26_6(size.X, w)
size.Y += m.Height
w, prev = 0, 0
continue
}
if prev != 0 {
w += f.Kern(prev, char)
}
w += advance(f, char)
prev = char
}
}
func Lex(input io.RuneReader, start StateFn) *Lexer {
tokens := make(chan *Item)
go func() {
r, _, err := input.ReadRune()
buf := &RuneBuffer{
in: input,
r: r,
more: (err == nil),
}
for state := start; state != nil; {
state = state(buf, tokens)
}
close(tokens)
}()
first, ok := <-tokens
return &Lexer{
tokens: tokens,
next: first,
more: ok,
}
}
func byteOffset(r io.RuneReader, off int) (bo int, err error) {
for i := 0; i != off; i++ {
_, s, err := r.ReadRune()
if err != nil {
return 0, err
}
bo += s
}
return
}
func readRunes(r io.RuneReader, buffer []rune) (int, error) {
for i, _ := range buffer {
ch, _, err := r.ReadRune()
if err != nil {
return i, err
}
buffer[i] = ch
}
return len(buffer), nil
}
func byteOff(q int, r io.RuneReader) (off int, err error) {
for i := 0; i != q; i++ {
_, s, err := r.ReadRune()
if err != nil {
return 0, err
}
off += s
}
return
}
func nextRune(rd io.RuneReader) (r rune, err error) {
for {
r, _, err = rd.ReadRune()
if err != nil {
return
}
if !unicode.IsSpace(r) {
return
}
}
return
}
func readAllRunes(r io.RuneReader) ([]rune, error) {
p := make([]rune, 0)
for {
rn, _, err := r.ReadRune()
if err != nil {
if err == io.EOF {
return p, nil
}
return p, err
}
p = append(p, rn)
}
return p, nil
}
func parseCell(r, c rune, sudoku Sudoku, rr io.RuneReader) (Sudoku, error) {
var x rune
var err error
for x = ' '; x != '.' && (x < '0' || x > '9'); x, _, err = rr.ReadRune() {
if err != nil {
return sudoku, err
}
}
if x >= '1' && x <= '9' {
return sudoku.withAssignment(coord(r, c), uint8(x-'0'))
}
return sudoku, nil
}
func (g globSetImpl) MatchReader(r io.RuneReader) bool {
// Drain the reader and convert to a byte array
b := make([]byte, 0, 10)
for {
rn, _, err := r.ReadRune()
if err == io.EOF {
break
} else if err != nil {
return false
}
b = append(b, byte(rn))
}
return g.Match(b)
}
func fromReader(r io.RuneReader) []byte {
b := make([]byte, numReadBufferStartSize)
offset := 0
var err error = nil
for err == nil {
rune, runeWidth, err := r.ReadRune()
if err == nil {
b = grow_buffer(b, offset, runeWidth)
writeWidth := utf8.EncodeRune(b[offset:], rune)
if runeWidth != writeWidth {
panic("reading rune width not equal to the written rune width")
}
offset += writeWidth
} else {
break
}
}
return b[:offset]
}
func runesUntil(in io.RuneReader, stop map[rune]bool) ([]rune, rune, error) {
v := []rune{}
for {
switch r, _, e := in.ReadRune(); {
case e != nil:
return v, r, e
case inMap(r, stop):
return v, r, nil
case r == '\\':
next, _, e := in.ReadRune()
if e != nil {
return v, next, e
}
v = append(v, next)
default:
v = append(v, r)
}
}
}
// Loads same file as above benchmarks, but just counts
// newlines. This is to see how much of the time is spent
// actually parsing vs just reading runes.
func BenchmarkNoopParse(b *testing.B) {
b.StopTimer()
data, err := ioutil.ReadFile("parsebench.tcl")
if err != nil {
panic(err.String())
}
b.SetBytes(int64(len(data)))
b.StartTimer()
for i := 0; i < b.N; i++ {
nlcount := 0
var reader io.RuneReader = bufio.NewReader(bytes.NewBuffer(data))
done := false
for !done {
r, _, e := reader.ReadRune()
if e != nil {
done = true
}
if r == '\n' {
nlcount++
}
}
}
}
// NewL parses a .l source fname from src, returns L or an error if any.
// Currently it is not reentrant and not invokable more than once in an application
// (which is assumed tolerable for a "lex" tool).
// The unoptdfa argument allows to disable optimization of the produced DFA.
// The mode32 parameter is not yet supported and must be false.
func NewL(fname string, src io.RuneReader, unoptdfa, mode32 bool) (l *L, err error) {
if mode32 {
return nil, errors.New("lex.NewL: mode32 unsupported yet")
}
nodfaopt, bits32 = unoptdfa, mode32
l = &L{}
if !hook {
defer func() {
if e := recover(); e != nil {
l = nil
err = e.(error)
}
}()
}
// Support \r\n line separators too
in := []rune{}
loop:
for {
r, _, err := src.ReadRune()
switch {
case err == nil:
in = append(in, r)
case fileutil.IsEOF(err):
break loop
default:
return nil, err
}
}
src = bytes.NewBufferString(strings.Replace(string(in), "\r\n", "\n", -1))
scanner := lxr.Scanner(fname, src)
if y := yyParse(newTokenizer(scanner)); y != 0 || len(errors_) != 0 {
return nil, errors.New(strings.Join(errors_, "\n"))
}
computePartialDFAs()
if len(errors_) != 0 {
return nil, errors.New(strings.Join(errors_, "\n"))
}
computeAllNfa()
allDfa = allNfa.powerSet()
for _, irule := range allDfa.acceptRule {
delete(unreachableRules, irule)
}
for irule := range unreachableRules {
logErr(fmt.Sprintf("%s - pattern `%s` unreachable", rulePos[irule], rules[irule].pattern))
}
if len(errors_) != 0 {
return nil, errors.New(strings.Join(errors_, "\n"))
}
l.DefCode = defCode
l.UserCode = usrCode
l.StartConditions = iStarts
l.StartConditionsStates = make(map[int]*lexer.NfaState)
l.StartConditionsBolStates = make(map[int]*lexer.NfaState)
for _, edge0 := range allDfa.nfa.in.Consuming {
switch edge := edge0.(type) {
default:
panic(errors.New("internal error"))
case *lexer.RuneEdge:
if _, ok := l.StartConditionsStates[int(edge.Rune)]; ok {
panic(errors.New("internal error"))
}
if edge.Rune < 128 {
l.StartConditionsStates[int(edge.Rune)] = edge.Target()
} else {
l.StartConditionsBolStates[int(edge.Rune)-128] = edge.Target()
}
case *lexer.RangesEdge:
for _, rng := range edge.Ranges.R32 {
for arune := rng.Lo; arune <= rng.Hi; arune += rng.Stride {
if _, ok := l.StartConditionsStates[int(arune)]; ok {
panic(errors.New("internal error"))
}
if arune < 128 {
l.StartConditionsStates[int(arune)] = edge.Target()
} else {
l.StartConditionsBolStates[int(arune)-128] = edge.Target()
}
}
}
}
}
for _, rule := range rules {
l.Rules = append(l.Rules, Rule{Conds: rule.conds, Pattern: rule.pattern, RE: rule.re, Action: rule.action, BOL: rule.bol, EOL: rule.eol})
}
l.Dfa = allDfa.nfa.nfa[1:]
l.Accepts = map[*lexer.NfaState]int{}
for id, state := range allDfa.accept {
l.Accepts[state] = allDfa.acceptRule[id]
}
l.YYT = _yyt
l.YYB = _yyb
l.YYC = _yyc
l.YYN = _yyn
l.YYM = _yym
return
}