//标准分词测试 func TestStandard(dawgFile string, text string) { fmt.Println("测试:" + text) tokenizer := cwsharp.NewStandardTokenizer(dawgFile, true) for token, next := tokenizer.Traverse(text)(); next != nil; token, next = next() { fmt.Print(token.String()) fmt.Print(" / ") } fmt.Println() }
func main() { file := "data\\cwsharp.dawg" tokenizer := cwsharp.NewStandardTokenizer(file, true) for _, text := range []string{"长春市长春药店", "研究生命起源", "Hello,World!"} { for token, next := tokenizer.Traverse(text)(); next != nil; token, next = next() { fmt.Printf(token.String()) fmt.Print(" / ") } fmt.Println() } }