Example #1
0
//标准分词测试
func TestStandard(dawgFile string, text string) {
	fmt.Println("测试:" + text)
	tokenizer := cwsharp.NewStandardTokenizer(dawgFile, true)
	for token, next := tokenizer.Traverse(text)(); next != nil; token, next = next() {
		fmt.Print(token.String())
		fmt.Print(" / ")
	}
	fmt.Println()
}
Example #2
0
func main() {
	file := "data\\cwsharp.dawg"
	tokenizer := cwsharp.NewStandardTokenizer(file, true)
	for _, text := range []string{"长春市长春药店", "研究生命起源", "Hello,World!"} {
		for token, next := tokenizer.Traverse(text)(); next != nil; token, next = next() {
			fmt.Printf(token.String())
			fmt.Print(" / ")
		}
		fmt.Println()
	}
}