func iter_words_backward(data []byte, cb func(word []byte)) {
for {
if len(data) == 0 {
return
}
r, rlen := utf8.DecodeLastRune(data)
// skip non-word runes
for !is_word(r) {
data = data[:len(data)-rlen]
if len(data) == 0 {
return
}
r, rlen = utf8.DecodeLastRune(data)
}
// must be on a word rune
i := len(data)
for is_word(r) && i > 0 {
i -= rlen
r, rlen = utf8.DecodeLastRune(data[:i])
}
cb(data[i:])
data = data[:i]
}
}
func (s *PossessiveFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
for _, token := range input {
lastRune, lastRuneSize := utf8.DecodeLastRune(token.Term)
if lastRune == 's' || lastRune == 'S' {
nextLastRune, nextLastRuneSize := utf8.DecodeLastRune(token.Term[:len(token.Term)-lastRuneSize])
if nextLastRune == rightSingleQuotationMark ||
nextLastRune == apostrophe ||
nextLastRune == fullWidthApostrophe {
token.Term = token.Term[:len(token.Term)-lastRuneSize-nextLastRuneSize]
}
}
}
return input
}
func (r *UTF8Reader) Read(p []byte) (n int, err error) {
size := 0
if cap(p) < utf8.UTFMax {
return size, SmallBufferError
}
if r.buffer.Len() > 0 {
n, err = r.buffer.Read(p)
size += n
if err != nil {
return size, err
}
}
n, err = r.reader.Read(p[size:])
size += n
if err != nil {
return size, err
}
leftOver := 0
for ; leftOver < utf8.UTFMax && size-leftOver > 0; leftOver++ {
rune, _ := utf8.DecodeLastRune(p[:size-leftOver])
if rune != utf8.RuneError {
break
}
}
r.buffer.Write(p[size-leftOver : size])
return size - leftOver, nil
}
func (f *wordBoundaryFiber) Resume() (output, error) {
if f.cnt == 0 {
f.cnt++
match := false
if len(f.I.b) > 0 {
if f.I.begin > 0 && isASCIIWord(rune(f.I.b[0])) != isASCIIWord(rune(f.I.o[f.I.begin-1])) {
match = true
}
if f.I.begin == 0 && isASCIIWord(rune(f.I.b[0])) {
match = true
}
}
if len(f.I.o) > 0 && f.I.begin == len(f.I.o) {
r, _ := utf8.DecodeLastRune(f.I.o)
if isASCIIWord(r) {
match = true
}
}
if f.node.Reversed {
match = !match
}
if match {
return output{offset: 0}, nil
}
}
return output{}, errDeadFiber
}
// MultipleChoice computes the score of a multiple choice exercise
// with student answers provided in fileName, and the answers provided
// in the answerKey object. The function requires a Score object, and
// will produce both string output and JSON output.
func MultipleChoice(t *testing.T, sc *score.Score, fileName string, answers Choices) {
defer sc.WriteString(os.Stdout)
defer sc.WriteJSON(os.Stdout)
// Read the whole file
bytes, err := ioutil.ReadFile(fileName)
if err != nil {
sc.Score = 0
t.Fatalf(fmt.Sprintf("%v: error reading the file: %v", fileName, err))
return
}
for i := range answers {
// Find the user's answer to the corresponding question number
regexStr := "\n" + strconv.Itoa(answers[i].Number) + "[.)]*[ \t\v\r\n\f]*[A-Za-z]*"
regex := regexp.MustCompile(regexStr)
userAnswer := regex.Find(bytes)
if userAnswer == nil {
t.Errorf("%v %d: Answer not found.\n", sc.TestName, answers[i].Number)
sc.Dec()
} else {
r, _ := utf8.DecodeLastRune(userAnswer)
got, _ := utf8.DecodeLastRuneInString(strings.ToUpper(string(r)))
if got != answers[i].Want {
t.Errorf("%v %d: %q is incorrect.\n", sc.TestName, answers[i].Number, got)
sc.Dec()
}
}
}
}
func (sr *stringReader) Read(p []byte) (int, error) {
if sr.remLen > 0 {
copy(p, sr.rem[:sr.remLen])
}
n, err := sr.r.Read(p[sr.remLen:])
n += int(sr.remLen)
sr.remLen = 0
if err != nil {
return n, err
}
// find the last full rune
var i int
for i = n; i >= 0; {
r, size := utf8.DecodeLastRune(p[:i])
if !(size == 1 && r == utf8.RuneError) {
break
}
i -= size
}
if i == n {
return n, err
}
sr.remLen = uint8(n - i)
log.Printf("remlLen=%d i=%d n=%d", sr.remLen, i, n)
copy(sr.rem[:sr.remLen], p[i:n])
return i, err
}
// TODO
// Got to hate utf8 for making it complicated... Guess there is not much to do..
func (f *File) OffsetRune(cn, start int) (offset int, e error) {
if start < 0 || start > len(f.b) {
return 0, memfile.OutOfBounds
}
if cn <= 0 {
ind := start
for ; cn < 0; cn++ {
if ind < 1 {
return 0, nil
}
_, s := utf8.DecodeLastRune(f.b[:ind])
ind -= s
}
return ind, nil
}
ind := start
for ; cn > 0; cn-- {
if ind >= len(f.b)-1 {
return len(f.b), nil
}
_, s := utf8.DecodeRune(f.b[ind:])
ind += s
}
return ind, nil
}
// Allowable parses the pattern and determines the minimum and maximum allowable
// values that the pattern can represent.
// When the max cannot be determined, 'true' will be returned
// for infinite.
func Allowable(pattern string) (min, max string) {
if pattern == "" || pattern[0] == '*' {
return "", ""
}
minb := make([]byte, 0, len(pattern))
maxb := make([]byte, 0, len(pattern))
var wild bool
for i := 0; i < len(pattern); i++ {
if pattern[i] == '*' {
wild = true
break
}
if pattern[i] == '?' {
minb = append(minb, 0)
maxb = append(maxb, maxRuneBytes...)
} else {
minb = append(minb, pattern[i])
maxb = append(maxb, pattern[i])
}
}
if wild {
r, n := utf8.DecodeLastRune(maxb)
if r != utf8.RuneError {
if r < utf8.MaxRune {
r++
if r > 0x7f {
b := make([]byte, 4)
nn := utf8.EncodeRune(b, r)
maxb = append(maxb[:len(maxb)-n], b[:nn]...)
} else {
maxb = append(maxb[:len(maxb)-n], byte(r))
}
}
}
}
return string(minb), string(maxb)
/*
return
if wild {
r, n := utf8.DecodeLastRune(maxb)
if r != utf8.RuneError {
if r < utf8.MaxRune {
infinite = true
} else {
r++
if r > 0x7f {
b := make([]byte, 4)
nn := utf8.EncodeRune(b, r)
maxb = append(maxb[:len(maxb)-n], b[:nn]...)
} else {
maxb = append(maxb[:len(maxb)-n], byte(r))
}
}
}
}
return string(minb), string(maxb), infinite
*/
}
func (r *Reader) UnreadRune() rune {
run, size := utf8.DecodeLastRune(r.buf[:r.pos])
if size == 0 {
panic(r.GetFileLineInfo() + " [UnreadRune] last is not valid utf8 code.")
}
r.pos -= size
return run
}
func reverse(str []byte) {
for i, j, s1, s2, len1, len2 := 0, len(str), rune(0), rune(0), 0, 0; i < j-1; i, j = i+len2, j-len1 {
s1, len1 = utf8.DecodeRune(str[i:])
s2, len2 = utf8.DecodeLastRune(str[:j])
copy(str[i+len2:j-len1], str[i+len1:j-len2])
copy(str[j-len1:j], []byte(string(s1)))
copy(str[i:i+len2], []byte(string(s2)))
}
}
func main() {
b := []byte("안녕하세요")
r, size := utf8.DecodeRune(b)
fmt.Printf("%c %d\n", r, size) // 안 3: "안녕하세요"의 첫 글자를 디코딩하여 '안', 바이트 수 3
r, size = utf8.DecodeRune(b[3:]) // '안'의 길이가 3이므로 인덱스 3부터 부분 슬라이스를 만들면 "녕하세요"가 됨
fmt.Printf("%c %d\n", r, size) // 녕 3: "녕하세요"를 첫 글자를 디코딩하여 '녕', 바이트 수 3
r, size = utf8.DecodeLastRune(b)
fmt.Printf("%c %d\n", r, size) // 요 3: "안녕하세요"의 마지막 글자를 디코딩하여 '요', 바이트 수 3
// '요'의 길이가 3이므로 // 문자열 길이-3을 하여 부분 슬라이스를 만들면
// "안녕하세"가 됨
r, size = utf8.DecodeLastRune(b[:len(b)-3])
fmt.Printf("%c %d\n", r, size) // 세 3: "안녕하세"의 마지막 글자를 디코딩하여 '세', 바이트 수 3
}
// backup one rune
func (s *Scanner) backup() {
w := utf8.RuneLen(s.ch)
s.rdOffset -= w
// Copy of slice, this is expensive
r, w := utf8.DecodeLastRune(s.src[:s.rdOffset])
s.offset = s.rdOffset - w
s.ch = r
}
func (w *win) typing(q0, q1 int) {
if *debug {
defer func(p, e int) {
w.Addr("#%d", w.eAddr)
text, err := w.ReadAll("data")
if err != nil {
panic(err)
}
w.Addr("#%d", w.pAddr)
log.Printf("typing pAddr before: %d, pAddr after: %d, eAddr before: %d, eAddr after: %d [%s]\n", p, w.pAddr, e, w.eAddr, text)
}(w.pAddr, w.eAddr)
}
if q0 < w.pAddr {
d("typing before prompt")
w.pAddr += q1 - q0
}
if q0 < w.eAddr {
d("typing before entry")
w.eAddr += q1 - q0
return
}
if q0 < w.pAddr {
return
}
defer w.Addr("#%d", w.pAddr)
w.Addr("#%d", w.eAddr)
text, err := w.ReadAll("data")
if err != nil {
panic("Failed to read from window: " + err.Error())
}
// If the last character after the prompt isn't a newline then
// wait. This fixes a bug where Send sends two typing
// events, the sent text and a new line. The text won't
// be issued to w.send() until the final newline is received.
// Otherwise the first insert event messes up the
// addresses and the subsequent event (with the newline)
// appears to have inserted a newline before pAddr.
if r, _ := utf8.DecodeLastRune(text); r != '\n' {
return
}
for {
i := bytes.IndexRune(text, '\n')
if i < 0 {
break
}
t := string(text[:i+1])
w.Addr("#%d,#%d", w.pAddr, w.eAddr+utf8.RuneCountInString(t))
w.send(t)
text = text[i+1:]
}
}
func (i *inputBytes) context(pos int) syntax.EmptyOp {
r1, r2 := endOfText, endOfText
if pos > 0 && pos <= len(i.str) {
r1, _ = utf8.DecodeLastRune(i.str[:pos])
}
if pos < len(i.str) {
r2, _ = utf8.DecodeRune(i.str[pos:])
}
return syntax.EmptyOpContext(r1, r2)
}
// lastIndexFunc is the same as LastIndexFunc except that if
// truth==false, the sense of the predicate function is
// inverted.
func lastIndexFunc(s []byte, f func(r rune) bool, truth bool) int {
for i := len(s); i > 0; {
r, size := utf8.DecodeLastRune(s[0:i])
i -= size
if f(r) == truth {
return i
}
}
return -1
}
func (f *File) Context(pos int) syntax.EmptyOp {
pos += f.pos
r1, r2 := endOfText, endOfText
if pos > 0 && pos <= f.Length() {
if pos < f.gapStart {
r1, _ = utf8.DecodeLastRune(f.b[:pos])
} else {
r1, _ = utf8.DecodeLastRune(f.b[:pos+(f.gapEnd-f.gapStart)])
}
}
if pos < f.Length() {
if pos < f.gapStart {
r2, _ = utf8.DecodeRune(f.b[pos:])
} else {
r2, _ = utf8.DecodeRune(f.b[pos+(f.gapEnd-f.gapStart):])
}
}
return syntax.EmptyOpContext(r1, r2)
}
// UnreadRune unreads the last rune returned by ReadRune.
// If the most recent read or write operation on the buffer was
// not a ReadRune, UnreadRune returns an error. (In this regard
// it is stricter than UnreadByte, which will unread the last byte
// from any read operation.)
func (b *Buffer) UnreadRune() error {
if b.lastRead != opReadRune {
return errors.New("bytes.Buffer: UnreadRune: previous operation was not ReadRune")
}
b.lastRead = opInvalid
if b.off > 0 {
_, n := utf8.DecodeLastRune(b.buf[0:b.off])
b.off -= n
}
return nil
}
func main() {
var arr [5]int
arr = [5]int{1, 2, 3, 4, 5}
for i := 0; i < len(arr); i++ {
fmt.Println(arr[i])
}
var pt *int
pt = &arr[0]
fmt.Println(*pt)
var sl []int = arr[0:1]
fmt.Println(sl)
ap := new([5]int)
fmt.Println(*ap)
fmt.Println(ap[1])
mk := make([]int, 0)
mk = append(mk, 100)
fmt.Println(mk)
for _, item := range arr {
item *= 2
}
fmt.Println(arr)
sa := []int{1, 2}
sa = append(sa, 3)
fmt.Println(sa)
s := "hello"
fmt.Println(s[0])
c := []byte(s)
c[0] = 'H'
fmt.Println(c[0])
p(sa)
more.Test()
more.Test1()
s_r := "你好"
fmt.Println(len(s_r))
fmt.Println(s_r[0])
k_r := []byte("墨迹阿婆")
slen := 0
sizet := 0
for i := 0; i < len(k_r); i = i + sizet {
_, sizet = utf8.DecodeLastRune(k_r)
slen++
}
fmt.Println(slen)
}
// TODO: (skip the gap)
func (b *br) ReadRune() (rune, int, error) {
// Try to decode the last rune in this part of the string
if b.ind == 0 {
return 0, 0, io.EOF
}
r, n := utf8.DecodeLastRune(b.f.b[:b.ind])
b.ind -= n
if r == utf8.RuneError && n == 1 {
return r, n, errors.New("Invalid")
}
return r, n, nil
}
// draws from right to left, 'off' is the end position
// (DrawLabel uses that method)
func (this *Buffer) draw_n_last_runes(off, n int, params *LabelParams, text []byte) {
for n > 0 {
r, size := utf8.DecodeLastRune(text)
this.Cells[off] = termbox.Cell{
Ch: r,
Fg: params.Fg,
Bg: params.Bg,
}
text = text[:len(text)-size]
off--
n--
}
}
// reverse reverses a slice of characters in place.
func reverse(bytes []byte) {
var r1, r2 rune
var l1, l2 int
for i, j := 0, len(bytes); i < j; i, j = i+l2, j-l1 {
r1, l1 = utf8.DecodeRune(bytes[i:])
r2, l2 = utf8.DecodeLastRune(bytes[:j])
if l2 > l1 {
copy(bytes[i+l2:], bytes[i+l1:j-l2])
}
copy(bytes[i:], []byte(string(r2)))
copy(bytes[j-l1:], []byte(string(r1)))
}
}
// LastIndexAny interprets s as a sequence of UTF-8-encoded Unicode code
// points. It returns the byte index of the last occurrence in s of any of
// the Unicode code points in chars. It returns -1 if chars is empty or if
// there is no code point in common.
func LastIndexAny(s []byte, chars string) int {
if len(chars) > 0 {
for i := len(s); i > 0; {
r, size := utf8.DecodeLastRune(s[0:i])
i -= size
for _, ch := range chars {
if r == ch {
return i
}
}
}
}
return -1
}
func main() {
var s string = "한"
var s2 string = "안녕하세요"
var r1 rune = '한'
fmt.Println(unicode.Is(unicode.Hangul, r1))
fmt.Println(unicode.Is(unicode.Latin, r1))
fmt.Println(unicode.In(r1, unicode.Latin, unicode.Han, unicode.Hangul))
fmt.Println(len(s))
fmt.Println(utf8.RuneLen(r1))
fmt.Println(utf8.RuneCountInString(s2))
b := []byte("안녕하세요")
r, size := utf8.DecodeRune(b)
fmt.Printf("%c %d\n", r, size)
r, size = utf8.DecodeRune(b[3:])
fmt.Printf("%c %d\n", r, size)
r, size = utf8.DecodeLastRune(b)
fmt.Printf("%c %d\n", r, size)
r, size = utf8.DecodeLastRune(b[:len(b)-3])
fmt.Printf("%c %d\n", r, size)
s = "Hello, World!"
fmt.Printf("%c\n", s[0])
fmt.Printf("%c\n", s[len(s)-1])
s = "안녕하세요"
r, _ = utf8.DecodeRuneInString(s)
fmt.Printf("%c\n", r)
r, _ = utf8.DecodeLastRuneInString(s)
fmt.Printf("%c\n", r)
}
func (c *ChatPanel) HandleInput(ev termbox.Event) {
c.m.Lock()
defer c.m.Unlock()
switch ev.Type {
case termbox.EventKey:
if ev.Ch != 0 {
c.WriteRune(ev.Ch)
} else {
switch ev.Key {
case termbox.KeySpace:
// just add a space
c.WriteRune(' ')
case termbox.KeyBackspace:
fallthrough
case termbox.KeyBackspace2:
// on backspace, remove the last rune in the buffer
if c.Len() > 0 {
_, size := utf8.DecodeLastRune(c.Bytes())
c.Truncate(c.Len() - size)
}
case termbox.KeyCtrlU:
// clear the buffer, like a UNIX terminal
c.Reset()
case termbox.KeyEnter:
// input confirmed, send it
if c.Len() > 0 {
c.g.SendPacket(gnet.NewPacket("Tchat", c.String()))
c.Reset()
c.term.SetInputHandler(nil)
}
case termbox.KeyEsc:
// input cancelled
c.Reset()
c.term.SetInputHandler(nil)
}
}
case termbox.EventResize:
w, h := termbox.Size()
r := image.Rect(w-1, h-2, w/2, h-1)
c.Buffered = panel.NewBuffered(r, termbox.Cell{'s', termbox.ColorGreen, 0})
}
}
func DetectEmphasis(s *Context) (consumed int) {
rest := s.Buf[s.Pos:]
if !isEmph(rest[0]) {
return 0
}
// find substring composed solely of '*' and '_'
i := 1
for i < len(rest) && isEmph(rest[i]) {
i++
}
indicator := rest[:i]
// "right-fringe-mark"
r, _ := utf8.DecodeRune(rest[len(indicator):])
rightFringe := emphasisFringeRank(r)
r, _ = utf8.DecodeLastRune(s.Buf[:s.Pos])
leftFringe := emphasisFringeRank(r)
// <0 means "left-flanking", >0 "right-flanking", 0 "non-flanking"
flanking := leftFringe - rightFringe
if flanking == 0 {
return len(indicator)
}
// split into "emphasis-tag-strings" - subslices of the same char
tags := [][]byte{}
prev := 0
for curr := 1; curr <= len(indicator); curr++ {
if curr == len(indicator) || indicator[curr] != indicator[prev] {
tags = append(tags, indicator[prev:curr])
prev = curr
}
}
// left-flanking? if yes, add some openings
if flanking < 0 {
for _, tag := range tags {
pos, _ := mdutils.OffsetIn(s.Buf, tag)
s.Openings.Push(MaybeOpening{
Tag: string(tag),
Pos: pos,
})
}
return len(indicator)
}
// right-flanking; maybe a closing tag
closingEmphasisTags(s, tags)
return len(indicator)
}
func (writer *dbEventWriter) Write(data []byte) (int, error) {
text := append(writer.dangling, data...)
checkEncoding, _ := utf8.DecodeLastRune(text)
if checkEncoding == utf8.RuneError {
writer.dangling = text
return len(data), nil
}
writer.dangling = nil
writer.db.SaveBuildEvent(writer.buildID, event.Log{
Payload: string(text),
Origin: writer.origin,
})
return len(data), nil
}
func (ctx *Context) run(path string, src interface{}) ([]byte, error) {
ctx.fset = token.NewFileSet()
// ctx.mode = parser.Trace
pf, err := parser.ParseFile(ctx.fset, path, src, ctx.mode)
if err != nil {
return nil, err
}
ast.Walk(ctx, pf)
lr, _ := utf8.DecodeLastRune(ctx.buf.Bytes())
_ = lr
if ctx.buf.Len() > 0 && lr != '\n' {
ctx.out("\n")
}
// ctx.printSels(pf.Decls)
return ctx.buf.Bytes(), nil
}
// endsWithCSSKeyword reports whether b ends with an ident that
// case-insensitively matches the lower-case kw.
func endsWithCSSKeyword(b []byte, kw string) bool {
i := len(b) - len(kw)
if i < 0 {
// Too short.
return false
}
if i != 0 {
r, _ := utf8.DecodeLastRune(b[:i])
if isCSSNmchar(r) {
// Too long.
return false
}
}
// Many CSS keywords, such as "!important" can have characters encoded,
// but the URI production does not allow that according to
// http://www.w3.org/TR/css3-syntax/#TOK-URI
// This does not attempt to recognize encoded keywords. For example,
// given "\75\72\6c" and "url" this return false.
return string(bytes.ToLower(b[i:])) == kw
}
func ExampleDecodeLastRune() {
b := []byte("Hello, 世界")
for len(b) > 0 {
r, size := utf8.DecodeLastRune(b)
fmt.Printf("%c %v\n", r, size)
b = b[:len(b)-size]
}
// Output:
// 界 3
// 世 3
// 1
// , 1
// o 1
// l 1
// l 1
// e 1
// H 1
}
// Reverse reverses (in place) a slice of bytes containing UTF-8
// encoded characters.
func Reverse(s []byte) {
var p, q rune // first and last runes
var pz, qz int // sizes of first and last runes
for i, j := 0, len(s); i < j-1; i, j = i+qz, j-pz {
p, pz = utf8.DecodeRune(s[i:])
q, qz = utf8.DecodeLastRune(s[:j])
// shift bytes between first and last runes
// if size of last > first
if qz > pz {
copy(s[i+qz:], s[i+pz:j-qz])
}
// copy last rune to first
copy(s[i:], []byte(string(q)))
// copy first rune to last
copy(s[j-pz:], []byte(string(p)))
}
}