func (this *Lexicon) AddSpaces() {
tokenizer := strutils.NewStrTokens(this.buffer.String())
buf := strutils.NewStringBuffer("")
for tokenizer.HasMoreTokens() {
next := strutils.NewStringBuffer(tokenizer.NextToken())
for _, op := range operators {
count := strings.Count(next.String(), op)
index := 0
for i := 0; i < count; i++ {
index := strings.Index(next.String()[index:], op)
next.Replace(index, index+len(op), " "+op+" ")
index += 2
}
}
for _, sp := range separators {
count := strings.Count(next.String(), sp)
index := 0
for i := 0; i < count; i++ {
index := strings.Index(next.String()[index:], sp)
next.Replace(index, index+len(sp), " "+sp+" ")
index += 2
}
}
buf.Append(next.String() + " ")
}
this.buffer = buf
}
func (this *Lexicon) FilterStrings() {
tokenizer := strutils.NewStrTokensDelims(this.buffer.String(), "\"", true)
buf := strutils.NewStringBuffer("")
for tokenizer.HasMoreTokens() {
next := tokenizer.NextToken()
if next == "\"" {
tmp := "\""
for tokenizer.HasMoreTokens() {
next = tokenizer.NextToken()
if next == "\"" {
if len(tmp) > 0 && string(tmp[len(tmp)-1]) == "\\" {
tmp += next
} else {
tmp += next
break
}
} else {
tmp += next
}
}
this.literals.Push(tmp)
} else {
buf.Append(next + " ")
}
}
this.buffer = buf
}
func (this *Lexicon) Init(filename string) { //initialize and remove comments
this.buffer = strutils.NewStringBuffer("")
file, err := os.Open(filename, os.O_RDONLY, 0666)
if err != nil {
println(err.String())
return
}
defer file.Close()
reader := bufio.NewReader(file)
commentStart := [2]string{"/*", "//"}
commentEnd := [2]string{"*/", "\n"}
for {
line, err := readLine(reader)
index := strings.Index(line, commentStart[0])
if index != -1 {
this.buffer.Append("\n" + line[0:index])
for {
line, err = readLine(reader)
index := strings.Index(line, commentEnd[0])
if index != -1 {
this.buffer.Append("\n" + line[0:index])
break
}
}
continue
}
index = strings.Index(line, commentStart[1])
if index != -1 {
this.buffer.Append("\n" + line[0:index])
} else {
this.buffer.Append("\n" + line)
}
if err != nil {
break
}
}
}