func awkTokenizer(input util.Chars) ([]util.Chars, int) {
// 9, 32
ret := []util.Chars{}
prefixLength := 0
state := awkNil
numChars := input.Length()
begin := 0
end := 0
for idx := 0; idx < numChars; idx++ {
r := input.Get(idx)
white := r == 9 || r == 32
switch state {
case awkNil:
if white {
prefixLength++
} else {
state, begin, end = awkBlack, idx, idx+1
}
case awkBlack:
end = idx + 1
if white {
state = awkWhite
}
case awkWhite:
if white {
end = idx + 1
} else {
ret = append(ret, input.Slice(begin, end))
state, begin, end = awkBlack, idx, idx+1
}
}
}
if begin < end {
ret = append(ret, input.Slice(begin, end))
}
return ret, prefixLength
}
