// Return true if t is either a space or a symbol func (tok *JSTokenizer) isSymbol(t string) bool { if len(t) != 1 { return false } r := rune(t[0]) return tokenize.SearchRunes(tok.symbols, r) != -1 }
func (tok *JSTokenizer) firstSymbol(text string) (int, rune) { // Return the index of the first symbol in text string for i, t := range text { if unicode.IsSpace(t) { return i, t } if tokenize.SearchRunes(tok.symbols, t) != -1 { return i, t } } // symbol not found, return special end code return len(text) - 1, rune(0) }