// Return true if t is either a space or a symbol
func (tok *JSTokenizer) isSymbol(t string) bool {
if len(t) != 1 {
return false
}
r := rune(t[0])
return tokenize.SearchRunes(tok.symbols, r) != -1
}
func (tok *JSTokenizer) firstSymbol(text string) (int, rune) {
// Return the index of the first symbol in text string
for i, t := range text {
if unicode.IsSpace(t) {
return i, t
}
if tokenize.SearchRunes(tok.symbols, t) != -1 {
return i, t
}
}
// symbol not found, return special end code
return len(text) - 1, rune(0)
}
