func (l *lexer) fetchText(breakers []rune) {
var data []rune
var escaped bool
reading:
for {
r := l.read()
if r == eof {
break
}
if !escaped {
if r == char_escape {
escaped = true
continue
}
if runes.IndexRune(breakers, r) != -1 {
l.unread()
break reading
}
}
escaped = false
data = append(data, r)
}
if len(data) > 0 {
l.tokens.push(Token{Text, string(data)})
}
}
func (self Single) Match(s string) bool {
r, w := utf8.DecodeRuneInString(s)
if len(s) > w {
return false
}
return runes.IndexRune(self.Separators, r) == -1
}
func (self List) Index(s string) (int, []int) {
for i, r := range s {
if self.Not == (runes.IndexRune(self.List, r) == -1) {
return i, segmentsByRuneLength[utf8.RuneLen(r)]
}
}
return -1, nil
}
func (self List) Match(s string) bool {
r, w := utf8.DecodeRuneInString(s)
if len(s) > w {
return false
}
inList := runes.IndexRune(self.List, r) != -1
return inList == !self.Not
}
func (self Single) Index(s string) (int, []int) {
for i, r := range s {
if runes.IndexRune(self.Separators, r) == -1 {
return i, segmentsByRuneLength[utf8.RuneLen(r)]
}
}
return -1, nil
}