Beispiel #1
0
func iter_words_backward(data []byte, cb func(word []byte)) {
	for {
		if len(data) == 0 {
			return
		}

		r, rlen := utf8.DecodeLastRune(data)
		// skip non-word runes
		for !is_word(r) {
			data = data[:len(data)-rlen]
			if len(data) == 0 {
				return
			}
			r, rlen = utf8.DecodeLastRune(data)
		}

		// must be on a word rune
		i := len(data)
		for is_word(r) && i > 0 {
			i -= rlen
			r, rlen = utf8.DecodeLastRune(data[:i])
		}
		cb(data[i:])
		data = data[:i]
	}
}
Beispiel #2
0
func (s *PossessiveFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
	for _, token := range input {
		lastRune, lastRuneSize := utf8.DecodeLastRune(token.Term)
		if lastRune == 's' || lastRune == 'S' {
			nextLastRune, nextLastRuneSize := utf8.DecodeLastRune(token.Term[:len(token.Term)-lastRuneSize])
			if nextLastRune == rightSingleQuotationMark ||
				nextLastRune == apostrophe ||
				nextLastRune == fullWidthApostrophe {
				token.Term = token.Term[:len(token.Term)-lastRuneSize-nextLastRuneSize]
			}
		}
	}
	return input
}
Beispiel #3
0
func (r *UTF8Reader) Read(p []byte) (n int, err error) {
	size := 0

	if cap(p) < utf8.UTFMax {
		return size, SmallBufferError
	}

	if r.buffer.Len() > 0 {
		n, err = r.buffer.Read(p)
		size += n
		if err != nil {
			return size, err
		}
	}

	n, err = r.reader.Read(p[size:])
	size += n
	if err != nil {
		return size, err
	}

	leftOver := 0
	for ; leftOver < utf8.UTFMax && size-leftOver > 0; leftOver++ {
		rune, _ := utf8.DecodeLastRune(p[:size-leftOver])
		if rune != utf8.RuneError {
			break
		}
	}

	r.buffer.Write(p[size-leftOver : size])

	return size - leftOver, nil
}
Beispiel #4
0
func (f *wordBoundaryFiber) Resume() (output, error) {
	if f.cnt == 0 {
		f.cnt++
		match := false
		if len(f.I.b) > 0 {
			if f.I.begin > 0 && isASCIIWord(rune(f.I.b[0])) != isASCIIWord(rune(f.I.o[f.I.begin-1])) {
				match = true
			}
			if f.I.begin == 0 && isASCIIWord(rune(f.I.b[0])) {
				match = true
			}
		}
		if len(f.I.o) > 0 && f.I.begin == len(f.I.o) {
			r, _ := utf8.DecodeLastRune(f.I.o)
			if isASCIIWord(r) {
				match = true
			}
		}
		if f.node.Reversed {
			match = !match
		}
		if match {
			return output{offset: 0}, nil
		}
	}
	return output{}, errDeadFiber
}
Beispiel #5
0
// MultipleChoice computes the score of a multiple choice exercise
// with student answers provided in fileName, and the answers provided
// in the answerKey object. The function requires a Score object, and
// will produce both string output and JSON output.
func MultipleChoice(t *testing.T, sc *score.Score, fileName string, answers Choices) {
	defer sc.WriteString(os.Stdout)
	defer sc.WriteJSON(os.Stdout)

	// Read the whole file
	bytes, err := ioutil.ReadFile(fileName)
	if err != nil {
		sc.Score = 0
		t.Fatalf(fmt.Sprintf("%v: error reading the file: %v", fileName, err))
		return
	}

	for i := range answers {
		// Find the user's answer to the corresponding question number
		regexStr := "\n" + strconv.Itoa(answers[i].Number) + "[.)]*[ \t\v\r\n\f]*[A-Za-z]*"
		regex := regexp.MustCompile(regexStr)
		userAnswer := regex.Find(bytes)

		if userAnswer == nil {
			t.Errorf("%v %d: Answer not found.\n", sc.TestName, answers[i].Number)
			sc.Dec()
		} else {
			r, _ := utf8.DecodeLastRune(userAnswer)
			got, _ := utf8.DecodeLastRuneInString(strings.ToUpper(string(r)))
			if got != answers[i].Want {
				t.Errorf("%v %d: %q is incorrect.\n", sc.TestName, answers[i].Number, got)
				sc.Dec()
			}
		}
	}
}
Beispiel #6
0
func (sr *stringReader) Read(p []byte) (int, error) {
	if sr.remLen > 0 {
		copy(p, sr.rem[:sr.remLen])
	}
	n, err := sr.r.Read(p[sr.remLen:])
	n += int(sr.remLen)
	sr.remLen = 0
	if err != nil {
		return n, err
	}
	// find the last full rune
	var i int
	for i = n; i >= 0; {
		r, size := utf8.DecodeLastRune(p[:i])
		if !(size == 1 && r == utf8.RuneError) {
			break
		}
		i -= size
	}
	if i == n {
		return n, err
	}
	sr.remLen = uint8(n - i)
	log.Printf("remlLen=%d i=%d n=%d", sr.remLen, i, n)
	copy(sr.rem[:sr.remLen], p[i:n])
	return i, err
}
Beispiel #7
0
Datei: gap.go Projekt: vron/sem
// TODO
// Got to hate utf8 for making it complicated... Guess there is not much to do..
func (f *File) OffsetRune(cn, start int) (offset int, e error) {
	if start < 0 || start > len(f.b) {
		return 0, memfile.OutOfBounds
	}
	if cn <= 0 {
		ind := start
		for ; cn < 0; cn++ {
			if ind < 1 {
				return 0, nil
			}
			_, s := utf8.DecodeLastRune(f.b[:ind])
			ind -= s
		}
		return ind, nil
	}
	ind := start
	for ; cn > 0; cn-- {
		if ind >= len(f.b)-1 {
			return len(f.b), nil
		}
		_, s := utf8.DecodeRune(f.b[ind:])
		ind += s
	}
	return ind, nil
}
Beispiel #8
0
// Allowable parses the pattern and determines the minimum and maximum allowable
// values that the pattern can represent.
// When the max cannot be determined, 'true' will be returned
// for infinite.
func Allowable(pattern string) (min, max string) {
	if pattern == "" || pattern[0] == '*' {
		return "", ""
	}

	minb := make([]byte, 0, len(pattern))
	maxb := make([]byte, 0, len(pattern))
	var wild bool
	for i := 0; i < len(pattern); i++ {
		if pattern[i] == '*' {
			wild = true
			break
		}
		if pattern[i] == '?' {
			minb = append(minb, 0)
			maxb = append(maxb, maxRuneBytes...)
		} else {
			minb = append(minb, pattern[i])
			maxb = append(maxb, pattern[i])
		}
	}
	if wild {
		r, n := utf8.DecodeLastRune(maxb)
		if r != utf8.RuneError {
			if r < utf8.MaxRune {
				r++
				if r > 0x7f {
					b := make([]byte, 4)
					nn := utf8.EncodeRune(b, r)
					maxb = append(maxb[:len(maxb)-n], b[:nn]...)
				} else {
					maxb = append(maxb[:len(maxb)-n], byte(r))
				}
			}
		}
	}
	return string(minb), string(maxb)
	/*
		return
		if wild {
			r, n := utf8.DecodeLastRune(maxb)
			if r != utf8.RuneError {
				if r < utf8.MaxRune {
					infinite = true
				} else {
					r++
					if r > 0x7f {
						b := make([]byte, 4)
						nn := utf8.EncodeRune(b, r)
						maxb = append(maxb[:len(maxb)-n], b[:nn]...)
					} else {
						maxb = append(maxb[:len(maxb)-n], byte(r))
					}
				}
			}
		}
		return string(minb), string(maxb), infinite
	*/
}
Beispiel #9
0
func (r *Reader) UnreadRune() rune {
	run, size := utf8.DecodeLastRune(r.buf[:r.pos])
	if size == 0 {
		panic(r.GetFileLineInfo() + " [UnreadRune] last is not valid utf8 code.")
	}
	r.pos -= size
	return run
}
Beispiel #10
0
func reverse(str []byte) {
	for i, j, s1, s2, len1, len2 := 0, len(str), rune(0), rune(0), 0, 0; i < j-1; i, j = i+len2, j-len1 {
		s1, len1 = utf8.DecodeRune(str[i:])
		s2, len2 = utf8.DecodeLastRune(str[:j])
		copy(str[i+len2:j-len1], str[i+len1:j-len2])
		copy(str[j-len1:j], []byte(string(s1)))
		copy(str[i:i+len2], []byte(string(s2)))
	}
}
func main() {
	b := []byte("안녕하세요")

	r, size := utf8.DecodeRune(b)
	fmt.Printf("%c %d\n", r, size) // 안 3: "안녕하세요"의 첫 글자를 디코딩하여 '안', 바이트 수 3

	r, size = utf8.DecodeRune(b[3:]) // '안'의 길이가 3이므로 인덱스 3부터 부분 슬라이스를 만들면 "녕하세요"가 됨
	fmt.Printf("%c %d\n", r, size)   // 녕 3: "녕하세요"를 첫 글자를 디코딩하여 '녕', 바이트 수 3

	r, size = utf8.DecodeLastRune(b)
	fmt.Printf("%c %d\n", r, size) // 요 3: "안녕하세요"의 마지막 글자를 디코딩하여 '요', 바이트 수 3

	// '요'의 길이가 3이므로 // 문자열 길이-3을 하여 부분 슬라이스를 만들면
	// "안녕하세"가 됨
	r, size = utf8.DecodeLastRune(b[:len(b)-3])

	fmt.Printf("%c %d\n", r, size) // 세 3: "안녕하세"의 마지막 글자를 디코딩하여 '세', 바이트 수 3
}
Beispiel #12
0
// backup one rune
func (s *Scanner) backup() {
	w := utf8.RuneLen(s.ch)
	s.rdOffset -= w

	// Copy of slice, this is expensive
	r, w := utf8.DecodeLastRune(s.src[:s.rdOffset])
	s.offset = s.rdOffset - w
	s.ch = r
}
Beispiel #13
0
Datei: win.go Projekt: aoeu/acme
func (w *win) typing(q0, q1 int) {
	if *debug {
		defer func(p, e int) {
			w.Addr("#%d", w.eAddr)
			text, err := w.ReadAll("data")
			if err != nil {
				panic(err)
			}
			w.Addr("#%d", w.pAddr)
			log.Printf("typing pAddr before: %d, pAddr after: %d, eAddr before: %d, eAddr after: %d [%s]\n", p, w.pAddr, e, w.eAddr, text)
		}(w.pAddr, w.eAddr)
	}

	if q0 < w.pAddr {
		d("typing before prompt")
		w.pAddr += q1 - q0
	}
	if q0 < w.eAddr {
		d("typing before entry")
		w.eAddr += q1 - q0
		return
	}
	if q0 < w.pAddr {
		return
	}

	defer w.Addr("#%d", w.pAddr)

	w.Addr("#%d", w.eAddr)
	text, err := w.ReadAll("data")
	if err != nil {
		panic("Failed to read from window: " + err.Error())
	}

	// If the last character after the prompt isn't a newline then
	// wait.  This fixes a bug where Send sends two typing
	// events, the sent text and a new line.  The text won't
	// be issued to w.send() until the final newline is received.
	// Otherwise the first insert event messes up the
	// addresses and the subsequent event (with the newline)
	// appears to have inserted a newline before pAddr.
	if r, _ := utf8.DecodeLastRune(text); r != '\n' {
		return
	}
	for {
		i := bytes.IndexRune(text, '\n')
		if i < 0 {
			break
		}

		t := string(text[:i+1])
		w.Addr("#%d,#%d", w.pAddr, w.eAddr+utf8.RuneCountInString(t))
		w.send(t)
		text = text[i+1:]
	}
}
Beispiel #14
0
func (i *inputBytes) context(pos int) syntax.EmptyOp {
	r1, r2 := endOfText, endOfText
	if pos > 0 && pos <= len(i.str) {
		r1, _ = utf8.DecodeLastRune(i.str[:pos])
	}
	if pos < len(i.str) {
		r2, _ = utf8.DecodeRune(i.str[pos:])
	}
	return syntax.EmptyOpContext(r1, r2)
}
Beispiel #15
0
// lastIndexFunc is the same as LastIndexFunc except that if
// truth==false, the sense of the predicate function is
// inverted.
func lastIndexFunc(s []byte, f func(r rune) bool, truth bool) int {
	for i := len(s); i > 0; {
		r, size := utf8.DecodeLastRune(s[0:i])
		i -= size
		if f(r) == truth {
			return i
		}
	}
	return -1
}
Beispiel #16
0
Datei: gap.go Projekt: vron/sem
func (f *File) Context(pos int) syntax.EmptyOp {
	pos += f.pos
	r1, r2 := endOfText, endOfText
	if pos > 0 && pos <= f.Length() {
		if pos < f.gapStart {
			r1, _ = utf8.DecodeLastRune(f.b[:pos])
		} else {
			r1, _ = utf8.DecodeLastRune(f.b[:pos+(f.gapEnd-f.gapStart)])
		}
	}
	if pos < f.Length() {
		if pos < f.gapStart {
			r2, _ = utf8.DecodeRune(f.b[pos:])
		} else {
			r2, _ = utf8.DecodeRune(f.b[pos+(f.gapEnd-f.gapStart):])
		}
	}
	return syntax.EmptyOpContext(r1, r2)
}
Beispiel #17
0
// UnreadRune unreads the last rune returned by ReadRune.
// If the most recent read or write operation on the buffer was
// not a ReadRune, UnreadRune returns an error.  (In this regard
// it is stricter than UnreadByte, which will unread the last byte
// from any read operation.)
func (b *Buffer) UnreadRune() error {
	if b.lastRead != opReadRune {
		return errors.New("bytes.Buffer: UnreadRune: previous operation was not ReadRune")
	}
	b.lastRead = opInvalid
	if b.off > 0 {
		_, n := utf8.DecodeLastRune(b.buf[0:b.off])
		b.off -= n
	}
	return nil
}
Beispiel #18
0
func main() {
	var arr [5]int
	arr = [5]int{1, 2, 3, 4, 5}
	for i := 0; i < len(arr); i++ {
		fmt.Println(arr[i])
	}

	var pt *int
	pt = &arr[0]
	fmt.Println(*pt)

	var sl []int = arr[0:1]
	fmt.Println(sl)

	ap := new([5]int)
	fmt.Println(*ap)
	fmt.Println(ap[1])

	mk := make([]int, 0)
	mk = append(mk, 100)
	fmt.Println(mk)

	for _, item := range arr {
		item *= 2
	}
	fmt.Println(arr)

	sa := []int{1, 2}
	sa = append(sa, 3)
	fmt.Println(sa)

	s := "hello"
	fmt.Println(s[0])
	c := []byte(s)
	c[0] = 'H'
	fmt.Println(c[0])
	p(sa)
	more.Test()
	more.Test1()

	s_r := "你好"
	fmt.Println(len(s_r))
	fmt.Println(s_r[0])

	k_r := []byte("墨迹阿婆")
	slen := 0
	sizet := 0
	for i := 0; i < len(k_r); i = i + sizet {
		_, sizet = utf8.DecodeLastRune(k_r)
		slen++
	}
	fmt.Println(slen)
}
Beispiel #19
0
Datei: gap.go Projekt: vron/sem
// TODO: (skip the gap)
func (b *br) ReadRune() (rune, int, error) {
	// Try to decode the last rune in this part of the string
	if b.ind == 0 {
		return 0, 0, io.EOF
	}
	r, n := utf8.DecodeLastRune(b.f.b[:b.ind])
	b.ind -= n
	if r == utf8.RuneError && n == 1 {
		return r, n, errors.New("Invalid")
	}
	return r, n, nil
}
Beispiel #20
0
// draws from right to left, 'off' is the end position
// (DrawLabel uses that method)
func (this *Buffer) draw_n_last_runes(off, n int, params *LabelParams, text []byte) {
	for n > 0 {
		r, size := utf8.DecodeLastRune(text)
		this.Cells[off] = termbox.Cell{
			Ch: r,
			Fg: params.Fg,
			Bg: params.Bg,
		}
		text = text[:len(text)-size]
		off--
		n--
	}
}
Beispiel #21
0
// reverse reverses a slice of characters in place.
func reverse(bytes []byte) {
	var r1, r2 rune
	var l1, l2 int
	for i, j := 0, len(bytes); i < j; i, j = i+l2, j-l1 {
		r1, l1 = utf8.DecodeRune(bytes[i:])
		r2, l2 = utf8.DecodeLastRune(bytes[:j])
		if l2 > l1 {
			copy(bytes[i+l2:], bytes[i+l1:j-l2])
		}
		copy(bytes[i:], []byte(string(r2)))
		copy(bytes[j-l1:], []byte(string(r1)))
	}
}
Beispiel #22
0
// LastIndexAny interprets s as a sequence of UTF-8-encoded Unicode code
// points.  It returns the byte index of the last occurrence in s of any of
// the Unicode code points in chars.  It returns -1 if chars is empty or if
// there is no code point in common.
func LastIndexAny(s []byte, chars string) int {
	if len(chars) > 0 {
		for i := len(s); i > 0; {
			r, size := utf8.DecodeLastRune(s[0:i])
			i -= size
			for _, ch := range chars {
				if r == ch {
					return i
				}
			}
		}
	}
	return -1
}
Beispiel #23
0
func main() {
	var s string = "한"
	var s2 string = "안녕하세요"
	var r1 rune = '한'
	fmt.Println(unicode.Is(unicode.Hangul, r1))
	fmt.Println(unicode.Is(unicode.Latin, r1))
	fmt.Println(unicode.In(r1, unicode.Latin, unicode.Han, unicode.Hangul))

	fmt.Println(len(s))
	fmt.Println(utf8.RuneLen(r1))
	fmt.Println(utf8.RuneCountInString(s2))

	b := []byte("안녕하세요")
	r, size := utf8.DecodeRune(b)
	fmt.Printf("%c %d\n", r, size)

	r, size = utf8.DecodeRune(b[3:])
	fmt.Printf("%c %d\n", r, size)

	r, size = utf8.DecodeLastRune(b)
	fmt.Printf("%c %d\n", r, size)

	r, size = utf8.DecodeLastRune(b[:len(b)-3])
	fmt.Printf("%c %d\n", r, size)

	s = "Hello, World!"
	fmt.Printf("%c\n", s[0])
	fmt.Printf("%c\n", s[len(s)-1])

	s = "안녕하세요"
	r, _ = utf8.DecodeRuneInString(s)
	fmt.Printf("%c\n", r)

	r, _ = utf8.DecodeLastRuneInString(s)
	fmt.Printf("%c\n", r)

}
Beispiel #24
0
func (c *ChatPanel) HandleInput(ev termbox.Event) {
	c.m.Lock()
	defer c.m.Unlock()
	switch ev.Type {
	case termbox.EventKey:
		if ev.Ch != 0 {
			c.WriteRune(ev.Ch)
		} else {
			switch ev.Key {
			case termbox.KeySpace:
				// just add a space
				c.WriteRune(' ')

			case termbox.KeyBackspace:
				fallthrough

			case termbox.KeyBackspace2:
				// on backspace, remove the last rune in the buffer
				if c.Len() > 0 {
					_, size := utf8.DecodeLastRune(c.Bytes())
					c.Truncate(c.Len() - size)
				}

			case termbox.KeyCtrlU:
				// clear the buffer, like a UNIX terminal
				c.Reset()

			case termbox.KeyEnter:
				// input confirmed, send it
				if c.Len() > 0 {
					c.g.SendPacket(gnet.NewPacket("Tchat", c.String()))
					c.Reset()
					c.term.SetInputHandler(nil)

				}
			case termbox.KeyEsc:
				// input cancelled
				c.Reset()
				c.term.SetInputHandler(nil)
			}
		}
	case termbox.EventResize:
		w, h := termbox.Size()
		r := image.Rect(w-1, h-2, w/2, h-1)
		c.Buffered = panel.NewBuffered(r, termbox.Cell{'s', termbox.ColorGreen, 0})

	}

}
Beispiel #25
0
func DetectEmphasis(s *Context) (consumed int) {
	rest := s.Buf[s.Pos:]
	if !isEmph(rest[0]) {
		return 0
	}
	// find substring composed solely of '*' and '_'
	i := 1
	for i < len(rest) && isEmph(rest[i]) {
		i++
	}
	indicator := rest[:i]
	// "right-fringe-mark"
	r, _ := utf8.DecodeRune(rest[len(indicator):])
	rightFringe := emphasisFringeRank(r)
	r, _ = utf8.DecodeLastRune(s.Buf[:s.Pos])
	leftFringe := emphasisFringeRank(r)
	// <0 means "left-flanking", >0 "right-flanking", 0 "non-flanking"
	flanking := leftFringe - rightFringe
	if flanking == 0 {
		return len(indicator)
	}
	// split into "emphasis-tag-strings" - subslices of the same char
	tags := [][]byte{}
	prev := 0
	for curr := 1; curr <= len(indicator); curr++ {
		if curr == len(indicator) || indicator[curr] != indicator[prev] {
			tags = append(tags, indicator[prev:curr])
			prev = curr
		}
	}
	// left-flanking? if yes, add some openings
	if flanking < 0 {
		for _, tag := range tags {
			pos, _ := mdutils.OffsetIn(s.Buf, tag)
			s.Openings.Push(MaybeOpening{
				Tag: string(tag),
				Pos: pos,
			})
		}
		return len(indicator)
	}

	// right-flanking; maybe a closing tag
	closingEmphasisTags(s, tags)
	return len(indicator)
}
Beispiel #26
0
func (writer *dbEventWriter) Write(data []byte) (int, error) {
	text := append(writer.dangling, data...)

	checkEncoding, _ := utf8.DecodeLastRune(text)
	if checkEncoding == utf8.RuneError {
		writer.dangling = text
		return len(data), nil
	}

	writer.dangling = nil

	writer.db.SaveBuildEvent(writer.buildID, event.Log{
		Payload: string(text),
		Origin:  writer.origin,
	})

	return len(data), nil
}
Beispiel #27
0
func (ctx *Context) run(path string, src interface{}) ([]byte, error) {

	ctx.fset = token.NewFileSet()
	// ctx.mode = parser.Trace
	pf, err := parser.ParseFile(ctx.fset, path, src, ctx.mode)
	if err != nil {
		return nil, err
	}

	ast.Walk(ctx, pf)
	lr, _ := utf8.DecodeLastRune(ctx.buf.Bytes())
	_ = lr
	if ctx.buf.Len() > 0 && lr != '\n' {
		ctx.out("\n")
	}
	// ctx.printSels(pf.Decls)
	return ctx.buf.Bytes(), nil
}
Beispiel #28
0
Datei: css.go Projekt: 2thetop/go
// endsWithCSSKeyword reports whether b ends with an ident that
// case-insensitively matches the lower-case kw.
func endsWithCSSKeyword(b []byte, kw string) bool {
	i := len(b) - len(kw)
	if i < 0 {
		// Too short.
		return false
	}
	if i != 0 {
		r, _ := utf8.DecodeLastRune(b[:i])
		if isCSSNmchar(r) {
			// Too long.
			return false
		}
	}
	// Many CSS keywords, such as "!important" can have characters encoded,
	// but the URI production does not allow that according to
	// http://www.w3.org/TR/css3-syntax/#TOK-URI
	// This does not attempt to recognize encoded keywords. For example,
	// given "\75\72\6c" and "url" this return false.
	return string(bytes.ToLower(b[i:])) == kw
}
Beispiel #29
0
func ExampleDecodeLastRune() {
	b := []byte("Hello, 世界")

	for len(b) > 0 {
		r, size := utf8.DecodeLastRune(b)
		fmt.Printf("%c %v\n", r, size)

		b = b[:len(b)-size]
	}
	// Output:
	// 界 3
	// 世 3
	//   1
	// , 1
	// o 1
	// l 1
	// l 1
	// e 1
	// H 1
}
Beispiel #30
0
// Reverse reverses (in place) a slice of bytes containing UTF-8
// encoded characters.
func Reverse(s []byte) {
	var p, q rune  // first and last runes
	var pz, qz int // sizes of first and last runes

	for i, j := 0, len(s); i < j-1; i, j = i+qz, j-pz {
		p, pz = utf8.DecodeRune(s[i:])
		q, qz = utf8.DecodeLastRune(s[:j])

		// shift bytes between first and last runes
		// if size of last > first
		if qz > pz {
			copy(s[i+qz:], s[i+pz:j-qz])
		}

		// copy last rune to first
		copy(s[i:], []byte(string(q)))

		// copy first rune to last
		copy(s[j-pz:], []byte(string(p)))
	}
}