示例#1
0
func TestLatticeNewAndFree(t *testing.T) {
	for i := 0; i < 100; i++ {
		la := New(dic.SysDic(), nil)
		if la == nil {
			t.Fatal("unexpected error: cannot new a lattice")
		}
		if la.Input != "" {
			t.Fatalf("unexpected error: lattice input initialize error, %+v", la.Input)
		}
		if len(la.Output) != 0 {
			t.Fatalf("unexpected error: lattice output initialize error, %+v", la.Output)
		}
		if len(la.list) != 0 {
			t.Fatalf("unexpected error: lattice list initialize error, %+v", la.list)
		}
		la.Build("すべては科学する心に宿るのだ")
		la.Free()

		// renew
		la = New(dic.SysDic(), nil)
		if la == nil {
			t.Fatal("unexpected error: cannot new a lattice")
		}
		if la.Input != "" {
			t.Fatalf("unexpected error: lattice input initialize error, %+v", la.Input)
		}
		if len(la.Output) != 0 {
			t.Fatalf("unexpected error: lattice output initialize error, %+v", la.Output)
		}
		if len(la.list) != 0 {
			t.Fatalf("unexpected error: lattice list initialize error, %+v", la.list)
		}
		la.Free()
	}
}
示例#2
0
func TestLatticeBuild01(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Error("cannot new a lattice")
	}
	defer la.Free()

	inp := ""
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}
	boseos := node{ID: -1}
	if len(la.list) != 2 {
		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
	} else if len(la.list[0]) != 1 || *la.list[0][0] != boseos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], boseos)
	} else if len(la.list[1]) != 1 || *la.list[1][0] != boseos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[1][0], boseos)
	}
	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}

	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic != nil {
		t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
	}
}
示例#3
0
func TestLatticeBuild03(t *testing.T) {

	const udicPath = "../../_sample/userdic.txt"

	udic, e := dic.NewUserDic(udicPath)
	if e != nil {
		t.Fatalf("unexpected error: cannot load user dic, %v", e)
	}
	la := New(dic.SysDic(), udic)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "朝青龍"
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}

	if la.list[3][0].Class != USER {
		t.Errorf("%+v", la)
	}

	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}
	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic == nil {
		t.Errorf("lattice initialize error: got %v, expected not empty", la.udic)
	}
}
示例#4
0
func TestLatticeBuild05(t *testing.T) {

	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "ポポピポンポコナーノ"
	var b bytes.Buffer
	for i, step := 0, utf8.RuneCountInString(inp); i < maximumUnknownWordLength; i = i + step {
		if _, e := b.WriteString(inp); e != nil {
			t.Fatalf("unexpected error: create the test input, %v", b.String())
		}
	}
	la.Build(b.String())
	for i := range la.list {
		for j := range la.list[i] {
			l := utf8.RuneCountInString(la.list[i][j].Surface)
			if l > maximumUnknownWordLength {
				t.Errorf("too long unknown word, %v", l)
			}
		}
	}
}
示例#5
0
func TestLatticeBuild04(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "ポポピ"
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}
	bos := node{ID: -1}
	eos := node{ID: -1, Start: 3}
	if len(la.list) != 5 {
		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
	} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], bos)
	} else if len(la.list[len(la.list)-1]) != 1 || *la.list[len(la.list)-1][0] != eos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[len(la.list)-1][0], eos)
	}

	expected := 7
	if len(la.list[1]) != expected {
		t.Errorf("lattice initialize error: got %v, expected %v", len(la.list[1]), expected)
	} else {
		l := la.list[1]
		callAndResponse := []struct {
			in  int
			out node
		}{
			{in: 0, out: node{98477, 0, KNOWN, 0, 1285, 1285, 4279, "ポ", nil}},
			{in: 1, out: node{31, 0, UNKNOWN, 0, 1289, 1289, 13581, "ポ", nil}},
			{in: 2, out: node{32, 0, UNKNOWN, 0, 1285, 1285, 9461, "ポ", nil}},
			{in: 3, out: node{33, 0, UNKNOWN, 0, 1293, 1293, 13661, "ポ", nil}},
			{in: 4, out: node{34, 0, UNKNOWN, 0, 1292, 1292, 10922, "ポ", nil}},
			{in: 5, out: node{35, 0, UNKNOWN, 0, 1288, 1288, 10521, "ポ", nil}},
			{in: 6, out: node{36, 0, UNKNOWN, 0, 3, 3, 14138, "ポ", nil}},
		}
		for _, cr := range callAndResponse {
			if *l[cr.in] != cr.out {
				t.Errorf("lattice initialize error: got %v, expected %v", l[cr.in], cr.out)
			}
		}
	}
	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}
	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic != nil {
		t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
	}
}
示例#6
0
func TestLatticeBuild02(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "あ"
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}
	bos := node{ID: -1}
	eos := node{ID: -1, Start: 1}
	if len(la.list) != 3 {
		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
	} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], bos)
	} else if len(la.list[2]) != 1 || *la.list[2][0] != eos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[2][0], eos)
	}

	expected := 4
	if len(la.list[1]) != expected {
		t.Errorf("lattice initialize error: got %v, expected %v", len(la.list[1]), expected)
	} else {
		l := la.list[1]
		callAndResponse := []struct {
			in  int
			out node
		}{
			{in: 0, out: node{122, 0, KNOWN, 0, 3, 3, 5549, "あ", nil}},
			{in: 1, out: node{123, 0, KNOWN, 0, 776, 776, 6690, "あ", nil}},
			{in: 2, out: node{124, 0, KNOWN, 0, 2, 2, 4262, "あ", nil}},
			{in: 3, out: node{125, 0, KNOWN, 0, 1118, 1118, 9035, "あ", nil}},
		}
		for _, cr := range callAndResponse {
			if *l[cr.in] != cr.out {
				t.Errorf("lattice initialize error: got %v, expected %v", l[cr.in], cr.out)
			}
		}
	}
	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}
	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic != nil {
		t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
	}
}
示例#7
0
func TestForward(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("unexpected error: cannot new a lattice")
	}

	la.Forward(Normal)
	la.Forward(Search)
	la.Forward(Extended)

	for _, m := range []TokenizeMode{Normal, Search, Extended} {
		la.Build("わたしまけましたわ")
		la.Forward(m)
	}
}
示例#8
0
func TestFeatures02(t *testing.T) {
	tok := Token{
		ID:      0,
		Class:   TokenClass(lattice.UNKNOWN),
		Start:   0,
		End:     1,
		Surface: "",
	}
	tok.dic = dic.SysDic()

	f := tok.Features()
	expected := []string{"名詞", "一般", "*", "*", "*", "*", "*"}
	if !reflect.DeepEqual(f, expected) {
		t.Errorf("got %v, expected %v", f, expected)
	}
}
示例#9
0
func TestLatticeString(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	expected := ""
	str := la.String()
	if str != expected {
		t.Errorf("got %v, expected: %v", str, expected)
	}

	la.Build("わたしまけましたわ")
	str = la.String()
	if str == "" {
		t.Errorf("got empty string")
	}
}
示例#10
0
func TestLatticeDot(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	expected := "graph lattice {\n\tdpi=48;\n\tgraph [style=filled, rankdir=LR]\n}\n"
	var b bytes.Buffer
	la.Dot(&b)
	if b.String() != expected {
		t.Errorf("got %v, expected: %v", b.String(), expected)
	}
	b.Reset()
	la.Build("わたしまけましたわ")
	la.Dot(&b)
	if b.String() == "" {
		t.Errorf("got empty string")
	}
}
示例#11
0
func TestFeatures04(t *testing.T) {
	tok := Token{
		ID:      0,
		Class:   DUMMY,
		Start:   0,
		End:     1,
		Surface: "",
	}
	tok.dic = dic.SysDic()
	if udic, e := dic.NewUserDic("../_sample/userdic.txt"); e != nil {
		t.Fatalf("build user dic error: %v", e)
	} else {
		tok.udic = udic
	}

	f := tok.Features()
	if len(f) != 0 {
		t.Errorf("got %v, expected empty", f)
	}
}
示例#12
0
func TestFeatures03(t *testing.T) {
	tok := Token{
		ID:      0,
		Class:   TokenClass(lattice.USER),
		Start:   0,
		End:     1,
		Surface: "",
	}
	tok.dic = dic.SysDic()
	if udic, e := dic.NewUserDic("../_sample/userdic.txt"); e != nil {
		t.Fatalf("build user dic error: %v", e)
	} else {
		tok.udic = udic
	}

	f := tok.Features()
	expected := []string{"カスタム名詞", "日本/経済/新聞", "ニホン/ケイザイ/シンブン"}
	if !reflect.DeepEqual(f, expected) {
		t.Errorf("got %v, expected %v", f, expected)
	}
}
示例#13
0
// SysDic returns the system dictionary.
func SysDic() Dic {
	return Dic{dic.SysDic()}
}