//标准分词测试
func TestStandard(dawgFile string, text string) {
fmt.Println("测试：" + text)
tokenizer := cwsharp.NewStandardTokenizer(dawgFile, true)
for token, next := tokenizer.Traverse(text)(); next != nil; token, next = next() {
fmt.Print(token.String())
fmt.Print(" / ")
}
fmt.Println()
}
func main() {
file := "data\\cwsharp.dawg"
tokenizer := cwsharp.NewStandardTokenizer(file, true)
for _, text := range []string{"长春市长春药店", "研究生命起源", "Hello,World!"} {
for token, next := tokenizer.Traverse(text)(); next != nil; token, next = next() {
fmt.Printf(token.String())
fmt.Print(" / ")
}
fmt.Println()
}
}