Exemplo n.º 1
0
func (this *Lexicon) AddSpaces() {
	tokenizer := strutils.NewStrTokens(this.buffer.String())
	buf := strutils.NewStringBuffer("")
	for tokenizer.HasMoreTokens() {
		next := strutils.NewStringBuffer(tokenizer.NextToken())
		for _, op := range operators {
			count := strings.Count(next.String(), op)
			index := 0
			for i := 0; i < count; i++ {
				index := strings.Index(next.String()[index:], op)
				next.Replace(index, index+len(op), " "+op+" ")
				index += 2
			}
		}
		for _, sp := range separators {
			count := strings.Count(next.String(), sp)
			index := 0
			for i := 0; i < count; i++ {
				index := strings.Index(next.String()[index:], sp)
				next.Replace(index, index+len(sp), " "+sp+" ")
				index += 2
			}
		}
		buf.Append(next.String() + " ")
	}
	this.buffer = buf
}
Exemplo n.º 2
0
func (this *Lexicon) FilterStrings() {
	tokenizer := strutils.NewStrTokensDelims(this.buffer.String(), "\"", true)
	buf := strutils.NewStringBuffer("")
	for tokenizer.HasMoreTokens() {
		next := tokenizer.NextToken()
		if next == "\"" {
			tmp := "\""
			for tokenizer.HasMoreTokens() {
				next = tokenizer.NextToken()
				if next == "\"" {
					if len(tmp) > 0 && string(tmp[len(tmp)-1]) == "\\" {
						tmp += next
					} else {
						tmp += next
						break
					}
				} else {
					tmp += next
				}
			}
			this.literals.Push(tmp)
		} else {
			buf.Append(next + " ")
		}
	}
	this.buffer = buf
}
Exemplo n.º 3
0
func (this *Lexicon) Init(filename string) { //initialize and remove comments
	this.buffer = strutils.NewStringBuffer("")
	file, err := os.Open(filename, os.O_RDONLY, 0666)
	if err != nil {
		println(err.String())
		return
	}
	defer file.Close()
	reader := bufio.NewReader(file)
	commentStart := [2]string{"/*", "//"}
	commentEnd := [2]string{"*/", "\n"}
	for {
		line, err := readLine(reader)
		index := strings.Index(line, commentStart[0])
		if index != -1 {
			this.buffer.Append("\n" + line[0:index])
			for {
				line, err = readLine(reader)
				index := strings.Index(line, commentEnd[0])
				if index != -1 {
					this.buffer.Append("\n" + line[0:index])
					break
				}
			}
			continue
		}
		index = strings.Index(line, commentStart[1])
		if index != -1 {
			this.buffer.Append("\n" + line[0:index])
		} else {
			this.buffer.Append("\n" + line)
		}
		if err != nil {
			break
		}
	}
}