Golang NewStringReader示例

示例#1

文件： tokenizer_test.go 项目： zhengchun/cwsharp-go

func Test(t *testing.T) {
	var input string = "Hello,World!你好，世界!14.4%,abc1233,11a-b,2015.2.1"
	tokenizer := New()
	for iter := tokenizer.Traverse(cwsharp.NewStringReader(input)); iter.Next(); {
		fmt.Println(iter.Cur())
	}
}

示例#2

文件： tokenizer_test.go 项目： zhengchun/cwsharp-go

func Test(t *testing.T) {
	input := "一次性交100元"
	tokenizer := New()
	iter := tokenizer.Traverse(cwsharp.NewStringReader(input))
	for iter.Next() {
		fmt.Println(iter.Cur())
	}
}

示例#3

文件： tokenizer_test.go 项目： zhengchun/cwsharp-go

func Test1(t *testing.T) {
	input := "研究生命起源.一次性交100元"
	tokenizer := New("https://github.com/zhengchun/cwsharp-go/raw/master/data/cwsharp.dawg")
	iter := tokenizer.Traverse(cwsharp.NewStringReader(input))
	for iter.Next() {
		fmt.Println(iter.Cur())
	}
}

示例#4

文件： main.go 项目： zhengchun/cwsharp-go

//标准分词测试
func TestStandard(dawgFile string, text string) {
	fmt.Println("测试：" + text)
	tokenizer := mmseg.New(dawgFile)
	for iter := tokenizer.Traverse(cwsharp.NewStringReader(text)); iter.Next(); {
		fmt.Print(iter.Cur())
		fmt.Print(" / ")
	}
	fmt.Println()
}

示例#5

文件： test.go 项目： zhengchun/cwsharp-go

func main() {
	file := "data//cwsharp.dawg"
	tokenizer := cwsharp.NewStopwordFilter(mmseg.New(file))
	w := map[string]bool{"the": true}
	tokenizer.CheckIgnore = func(t cwsharp.Token) bool {
		_, ok := w[t.Text()]
		if t.Kind() == cwsharp.PUNCT || ok {
			return true
		}
		return false
	}
	for _, text := range []string{"长春市长春药店", "The quick brown fox jumps over the lazy dog"} {
		for iter := tokenizer.Traverse(cwsharp.NewStringReader(text)); iter.Next(); {
			fmt.Print(iter.Cur())
			fmt.Print(" / ")
		}
		fmt.Println()
	}
}