func TokenizeOld(source string) []string {
tokens, _ := lexer.Lex("\n("+source+")", token_exprs)
toks := []string{}
for _, v := range tokens {
toks = append(toks, v.Text)
}
return toks
}
// This is where we handle all the new style rules, we transform to old style simply.
func Tokenize(source string) []string {
source = clearNewlines(source)
tokens, _ := lexer.Lex("(\n"+source+"\n)", token_exrps_clear)
toks := []string{}
last_ind := 0
for i := 0; i < len(tokens); i++ {
v := tokens[i]
if v.Text == "\n" {
var next_ind int
if tokens[i+1].Text != "\t" {
next_ind = 0
} else {
next_ind = tokens[i+1].Occ
}
diff := next_ind - last_ind
last_ind = next_ind
if len(tokens) != i+1 && i > 0 && tokens[i-1].Text != "(" {
if diff <= 0 {
toks = append(toks, ")") // 1 implicit záró
for i := 0; i < diff*(-1); i++ { // plusz amennyit csökken
toks = append(toks, ")")
}
}
}
if len(toks) > 0 && len(tokens) > i+2 && tokens[i+2].Text != ")" {
toks = append(toks, "(")
}
} else if v.Text == ";" {
toks = append(toks, ")")
toks = append(toks, "(")
} else if v.Text != "\t" {
toks = append(toks, v.Text)
}
}
return toks
}
