Beispiel #1
0
// Stat calculates statistics for all runes read from r.
func (m *Main) Stat(r io.RuneReader) (Stats, error) {
	var stats Stats

	for {
		// Read next character.
		ch, sz, err := r.ReadRune()
		if err == io.EOF {
			break
		} else if err != nil {
			return stats, err
		}

		// Calculate stats.
		stats.TotalN++
		if unicode.IsControl(ch) {
			stats.ControlN++
		}
		if unicode.IsDigit(ch) {
			stats.DigitN++
		}
		if unicode.IsGraphic(ch) {
			stats.GraphicN++
		}
		if unicode.IsLetter(ch) {
			stats.LetterN++
		}
		if unicode.IsLower(ch) {
			stats.LowerN++
		}
		if unicode.IsMark(ch) {
			stats.MarkN++
		}
		if unicode.IsNumber(ch) {
			stats.NumberN++
		}
		if unicode.IsPrint(ch) {
			stats.PrintN++
		}
		if unicode.IsPunct(ch) {
			stats.PunctN++
		}
		if unicode.IsSpace(ch) {
			stats.SpaceN++
		}
		if unicode.IsSymbol(ch) {
			stats.SymbolN++
		}
		if unicode.IsTitle(ch) {
			stats.TitleN++
		}
		if unicode.IsUpper(ch) {
			stats.UpperN++
		}
		if sz > 1 {
			stats.MultiByteN++
		}
	}

	return stats, nil
}
Beispiel #2
0
// CharCount reads input and returns stats about unicode frequency
func CharCount(in io.RuneReader) UnicodeStat {
	var results UnicodeStat
	results.Count = make(map[rune]int)      // counts of Unicode characters
	results.UtfLen = make(map[int]int)      // count of lengths of UTF-8 encodings
	results.Category = make(map[string]int) // number of categorys like Unicode.IsNumber
	results.Invalid = 0                     // count of invalid UTF-8 characters

	for {
		r, n, err := in.ReadRune() // returns rune, nbytes, error
		if err == io.EOF {
			break
		}
		if err != nil {
			fmt.Fprintf(os.Stderr, "charcount: %v\n", err)
			os.Exit(1)
		}
		if r == unicode.ReplacementChar && n == 1 {
			results.Invalid++
			continue
		}
		results.Count[r]++
		results.Category[CharType(r)]++
		results.UtfLen[n]++
	}

	return results
}
Beispiel #3
0
func Draw(r io.RuneReader, f font.Face, dst draw.Image, src image.Image, bounds fixed.Rectangle26_6, dir Direction) (err error) {
	var prev rune
	var char rune

	var origin = dir.Origin(f, bounds)
	var dot = origin
	var next fixed.Point26_6

	for {
		if char, _, err = r.ReadRune(); err != nil {
			if err == io.EOF {
				err = nil
			}
			return
		}

		dot, next = dir.Advance(f, origin, dot, prev, char)

		if char != '\n' {
			dr, mask, maskp := glyph(f, char, dot)
			draw.DrawMask(dst, dr, src, dr.Min, mask, maskp, draw.Over)
		}

		dot, prev = next, char
	}
}
Beispiel #4
0
func Measure(r io.RuneReader, f font.Face) (size fixed.Point26_6, err error) {
	var char rune
	var prev rune

	w := fixed.Int26_6(0)
	m := f.Metrics()
	size.Y = m.Height + m.Descent

	for {
		if char, _, err = r.ReadRune(); err != nil {
			if err == io.EOF {
				err = nil
			}
			size.X = maxInt26_6(size.X, w)
			return
		}

		if char == '\n' {
			size.X = maxInt26_6(size.X, w)
			size.Y += m.Height
			w, prev = 0, 0
			continue
		}

		if prev != 0 {
			w += f.Kern(prev, char)
		}

		w += advance(f, char)
		prev = char
	}
}
func Lex(input io.RuneReader, start StateFn) *Lexer {
	tokens := make(chan *Item)

	go func() {
		r, _, err := input.ReadRune()
		buf := &RuneBuffer{
			in:   input,
			r:    r,
			more: (err == nil),
		}

		for state := start; state != nil; {
			state = state(buf, tokens)
		}

		close(tokens)
	}()

	first, ok := <-tokens
	return &Lexer{
		tokens: tokens,
		next:   first,
		more:   ok,
	}
}
Beispiel #6
0
Datei: main.go Projekt: aoeu/acme
func byteOffset(r io.RuneReader, off int) (bo int, err error) {
	for i := 0; i != off; i++ {
		_, s, err := r.ReadRune()
		if err != nil {
			return 0, err
		}
		bo += s
	}
	return
}
Beispiel #7
0
func readRunes(r io.RuneReader, buffer []rune) (int, error) {
	for i, _ := range buffer {
		ch, _, err := r.ReadRune()
		if err != nil {
			return i, err
		}
		buffer[i] = ch
	}
	return len(buffer), nil
}
Beispiel #8
0
func byteOff(q int, r io.RuneReader) (off int, err error) {
	for i := 0; i != q; i++ {
		_, s, err := r.ReadRune()
		if err != nil {
			return 0, err
		}
		off += s
	}
	return
}
Beispiel #9
0
func nextRune(rd io.RuneReader) (r rune, err error) {
	for {
		r, _, err = rd.ReadRune()
		if err != nil {
			return
		}
		if !unicode.IsSpace(r) {
			return
		}
	}
	return
}
Beispiel #10
0
func readAllRunes(r io.RuneReader) ([]rune, error) {
	p := make([]rune, 0)
	for {
		rn, _, err := r.ReadRune()
		if err != nil {
			if err == io.EOF {
				return p, nil
			}
			return p, err
		}
		p = append(p, rn)
	}

	return p, nil
}
Beispiel #11
0
func parseCell(r, c rune, sudoku Sudoku, rr io.RuneReader) (Sudoku, error) {
	var x rune
	var err error

	for x = ' '; x != '.' && (x < '0' || x > '9'); x, _, err = rr.ReadRune() {
		if err != nil {
			return sudoku, err
		}
	}

	if x >= '1' && x <= '9' {
		return sudoku.withAssignment(coord(r, c), uint8(x-'0'))
	}
	return sudoku, nil
}
Beispiel #12
0
func (g globSetImpl) MatchReader(r io.RuneReader) bool {
	// Drain the reader and convert to a byte array
	b := make([]byte, 0, 10)
	for {
		rn, _, err := r.ReadRune()
		if err == io.EOF {
			break
		} else if err != nil {
			return false
		}
		b = append(b, byte(rn))
	}

	return g.Match(b)
}
Beispiel #13
0
func fromReader(r io.RuneReader) []byte {
	b := make([]byte, numReadBufferStartSize)
	offset := 0
	var err error = nil
	for err == nil {
		rune, runeWidth, err := r.ReadRune()
		if err == nil {
			b = grow_buffer(b, offset, runeWidth)
			writeWidth := utf8.EncodeRune(b[offset:], rune)
			if runeWidth != writeWidth {
				panic("reading rune width not equal to the written rune width")
			}
			offset += writeWidth
		} else {
			break
		}
	}
	return b[:offset]
}
Beispiel #14
0
func runesUntil(in io.RuneReader, stop map[rune]bool) ([]rune, rune, error) {
	v := []rune{}
	for {
		switch r, _, e := in.ReadRune(); {
		case e != nil:
			return v, r, e
		case inMap(r, stop):
			return v, r, nil
		case r == '\\':
			next, _, e := in.ReadRune()
			if e != nil {
				return v, next, e
			}
			v = append(v, next)
		default:
			v = append(v, r)
		}
	}
}
Beispiel #15
0
// Loads same file as above benchmarks, but just counts
// newlines. This is to see how much of the time is spent
// actually parsing vs just reading runes.
func BenchmarkNoopParse(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("parsebench.tcl")
	if err != nil {
		panic(err.String())
	}
	b.SetBytes(int64(len(data)))
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		nlcount := 0
		var reader io.RuneReader = bufio.NewReader(bytes.NewBuffer(data))
		done := false
		for !done {
			r, _, e := reader.ReadRune()
			if e != nil {
				done = true
			}
			if r == '\n' {
				nlcount++
			}
		}
	}
}
Beispiel #16
0
Datei: lex.go Projekt: cznic/lex
// NewL parses a .l source fname from src, returns L or an error if any.
// Currently it is not reentrant and not invokable more than once in an application
// (which is assumed tolerable for a "lex" tool).
// The unoptdfa argument allows to disable optimization of the produced DFA.
// The mode32 parameter is not yet supported and must be false.
func NewL(fname string, src io.RuneReader, unoptdfa, mode32 bool) (l *L, err error) {
	if mode32 {
		return nil, errors.New("lex.NewL: mode32 unsupported yet")
	}

	nodfaopt, bits32 = unoptdfa, mode32
	l = &L{}

	if !hook {
		defer func() {
			if e := recover(); e != nil {
				l = nil
				err = e.(error)
			}
		}()
	}

	// Support \r\n line separators too
	in := []rune{}
loop:
	for {
		r, _, err := src.ReadRune()
		switch {
		case err == nil:
			in = append(in, r)
		case fileutil.IsEOF(err):
			break loop
		default:
			return nil, err
		}
	}
	src = bytes.NewBufferString(strings.Replace(string(in), "\r\n", "\n", -1))

	scanner := lxr.Scanner(fname, src)
	if y := yyParse(newTokenizer(scanner)); y != 0 || len(errors_) != 0 {
		return nil, errors.New(strings.Join(errors_, "\n"))
	}

	computePartialDFAs()
	if len(errors_) != 0 {
		return nil, errors.New(strings.Join(errors_, "\n"))
	}

	computeAllNfa()
	allDfa = allNfa.powerSet()
	for _, irule := range allDfa.acceptRule {
		delete(unreachableRules, irule)
	}
	for irule := range unreachableRules {
		logErr(fmt.Sprintf("%s - pattern `%s` unreachable", rulePos[irule], rules[irule].pattern))
	}
	if len(errors_) != 0 {
		return nil, errors.New(strings.Join(errors_, "\n"))
	}

	l.DefCode = defCode
	l.UserCode = usrCode
	l.StartConditions = iStarts
	l.StartConditionsStates = make(map[int]*lexer.NfaState)
	l.StartConditionsBolStates = make(map[int]*lexer.NfaState)
	for _, edge0 := range allDfa.nfa.in.Consuming {
		switch edge := edge0.(type) {
		default:
			panic(errors.New("internal error"))
		case *lexer.RuneEdge:
			if _, ok := l.StartConditionsStates[int(edge.Rune)]; ok {
				panic(errors.New("internal error"))
			}
			if edge.Rune < 128 {
				l.StartConditionsStates[int(edge.Rune)] = edge.Target()
			} else {
				l.StartConditionsBolStates[int(edge.Rune)-128] = edge.Target()
			}
		case *lexer.RangesEdge:
			for _, rng := range edge.Ranges.R32 {
				for arune := rng.Lo; arune <= rng.Hi; arune += rng.Stride {
					if _, ok := l.StartConditionsStates[int(arune)]; ok {
						panic(errors.New("internal error"))
					}
					if arune < 128 {
						l.StartConditionsStates[int(arune)] = edge.Target()
					} else {
						l.StartConditionsBolStates[int(arune)-128] = edge.Target()
					}
				}
			}
		}

	}
	for _, rule := range rules {
		l.Rules = append(l.Rules, Rule{Conds: rule.conds, Pattern: rule.pattern, RE: rule.re, Action: rule.action, BOL: rule.bol, EOL: rule.eol})
	}
	l.Dfa = allDfa.nfa.nfa[1:]
	l.Accepts = map[*lexer.NfaState]int{}
	for id, state := range allDfa.accept {
		l.Accepts[state] = allDfa.acceptRule[id]
	}
	l.YYT = _yyt
	l.YYB = _yyb
	l.YYC = _yyc
	l.YYN = _yyn
	l.YYM = _yym
	return
}