func TestLatticeNewAndFree(t *testing.T) {
for i := 0; i < 100; i++ {
la := New(dic.SysDic(), nil)
if la == nil {
t.Fatal("unexpected error: cannot new a lattice")
}
if la.Input != "" {
t.Fatalf("unexpected error: lattice input initialize error, %+v", la.Input)
}
if len(la.Output) != 0 {
t.Fatalf("unexpected error: lattice output initialize error, %+v", la.Output)
}
if len(la.list) != 0 {
t.Fatalf("unexpected error: lattice list initialize error, %+v", la.list)
}
la.Build("すべては科学する心に宿るのだ")
la.Free()
// renew
la = New(dic.SysDic(), nil)
if la == nil {
t.Fatal("unexpected error: cannot new a lattice")
}
if la.Input != "" {
t.Fatalf("unexpected error: lattice input initialize error, %+v", la.Input)
}
if len(la.Output) != 0 {
t.Fatalf("unexpected error: lattice output initialize error, %+v", la.Output)
}
if len(la.list) != 0 {
t.Fatalf("unexpected error: lattice list initialize error, %+v", la.list)
}
la.Free()
}
}
func TestLatticeBuild01(t *testing.T) {
la := New(dic.SysDic(), nil)
if la == nil {
t.Error("cannot new a lattice")
}
defer la.Free()
inp := ""
la.Build(inp)
if la.Input != inp {
t.Errorf("got %v, expected %v", la.Input, inp)
}
boseos := node{ID: -1}
if len(la.list) != 2 {
t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
} else if len(la.list[0]) != 1 || *la.list[0][0] != boseos {
t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], boseos)
} else if len(la.list[1]) != 1 || *la.list[1][0] != boseos {
t.Errorf("lattice initialize error: got %v, expected %v", *la.list[1][0], boseos)
}
if len(la.Output) != 0 {
t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
}
if la.dic == nil {
t.Errorf("lattice initialize error: dic is nil")
}
if la.udic != nil {
t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
}
}
func TestLatticeBuild03(t *testing.T) {
const udicPath = "../../_sample/userdic.txt"
udic, e := dic.NewUserDic(udicPath)
if e != nil {
t.Fatalf("unexpected error: cannot load user dic, %v", e)
}
la := New(dic.SysDic(), udic)
if la == nil {
t.Fatal("cannot new a lattice")
}
defer la.Free()
inp := "朝青龍"
la.Build(inp)
if la.Input != inp {
t.Errorf("got %v, expected %v", la.Input, inp)
}
if la.list[3][0].Class != USER {
t.Errorf("%+v", la)
}
if len(la.Output) != 0 {
t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
}
if la.dic == nil {
t.Errorf("lattice initialize error: dic is nil")
}
if la.udic == nil {
t.Errorf("lattice initialize error: got %v, expected not empty", la.udic)
}
}
func TestLatticeBuild05(t *testing.T) {
la := New(dic.SysDic(), nil)
if la == nil {
t.Fatal("cannot new a lattice")
}
defer la.Free()
inp := "ポポピポンポコナーノ"
var b bytes.Buffer
for i, step := 0, utf8.RuneCountInString(inp); i < maximumUnknownWordLength; i = i + step {
if _, e := b.WriteString(inp); e != nil {
t.Fatalf("unexpected error: create the test input, %v", b.String())
}
}
la.Build(b.String())
for i := range la.list {
for j := range la.list[i] {
l := utf8.RuneCountInString(la.list[i][j].Surface)
if l > maximumUnknownWordLength {
t.Errorf("too long unknown word, %v", l)
}
}
}
}
func TestLatticeBuild04(t *testing.T) {
la := New(dic.SysDic(), nil)
if la == nil {
t.Fatal("cannot new a lattice")
}
defer la.Free()
inp := "ポポピ"
la.Build(inp)
if la.Input != inp {
t.Errorf("got %v, expected %v", la.Input, inp)
}
bos := node{ID: -1}
eos := node{ID: -1, Start: 3}
if len(la.list) != 5 {
t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], bos)
} else if len(la.list[len(la.list)-1]) != 1 || *la.list[len(la.list)-1][0] != eos {
t.Errorf("lattice initialize error: got %v, expected %v", *la.list[len(la.list)-1][0], eos)
}
expected := 7
if len(la.list[1]) != expected {
t.Errorf("lattice initialize error: got %v, expected %v", len(la.list[1]), expected)
} else {
l := la.list[1]
callAndResponse := []struct {
in int
out node
}{
{in: 0, out: node{98477, 0, KNOWN, 0, 1285, 1285, 4279, "ポ", nil}},
{in: 1, out: node{31, 0, UNKNOWN, 0, 1289, 1289, 13581, "ポ", nil}},
{in: 2, out: node{32, 0, UNKNOWN, 0, 1285, 1285, 9461, "ポ", nil}},
{in: 3, out: node{33, 0, UNKNOWN, 0, 1293, 1293, 13661, "ポ", nil}},
{in: 4, out: node{34, 0, UNKNOWN, 0, 1292, 1292, 10922, "ポ", nil}},
{in: 5, out: node{35, 0, UNKNOWN, 0, 1288, 1288, 10521, "ポ", nil}},
{in: 6, out: node{36, 0, UNKNOWN, 0, 3, 3, 14138, "ポ", nil}},
}
for _, cr := range callAndResponse {
if *l[cr.in] != cr.out {
t.Errorf("lattice initialize error: got %v, expected %v", l[cr.in], cr.out)
}
}
}
if len(la.Output) != 0 {
t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
}
if la.dic == nil {
t.Errorf("lattice initialize error: dic is nil")
}
if la.udic != nil {
t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
}
}
func TestLatticeBuild02(t *testing.T) {
la := New(dic.SysDic(), nil)
if la == nil {
t.Fatal("cannot new a lattice")
}
defer la.Free()
inp := "あ"
la.Build(inp)
if la.Input != inp {
t.Errorf("got %v, expected %v", la.Input, inp)
}
bos := node{ID: -1}
eos := node{ID: -1, Start: 1}
if len(la.list) != 3 {
t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], bos)
} else if len(la.list[2]) != 1 || *la.list[2][0] != eos {
t.Errorf("lattice initialize error: got %v, expected %v", *la.list[2][0], eos)
}
expected := 4
if len(la.list[1]) != expected {
t.Errorf("lattice initialize error: got %v, expected %v", len(la.list[1]), expected)
} else {
l := la.list[1]
callAndResponse := []struct {
in int
out node
}{
{in: 0, out: node{122, 0, KNOWN, 0, 3, 3, 5549, "あ", nil}},
{in: 1, out: node{123, 0, KNOWN, 0, 776, 776, 6690, "あ", nil}},
{in: 2, out: node{124, 0, KNOWN, 0, 2, 2, 4262, "あ", nil}},
{in: 3, out: node{125, 0, KNOWN, 0, 1118, 1118, 9035, "あ", nil}},
}
for _, cr := range callAndResponse {
if *l[cr.in] != cr.out {
t.Errorf("lattice initialize error: got %v, expected %v", l[cr.in], cr.out)
}
}
}
if len(la.Output) != 0 {
t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
}
if la.dic == nil {
t.Errorf("lattice initialize error: dic is nil")
}
if la.udic != nil {
t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
}
}
func TestForward(t *testing.T) {
la := New(dic.SysDic(), nil)
if la == nil {
t.Fatal("unexpected error: cannot new a lattice")
}
la.Forward(Normal)
la.Forward(Search)
la.Forward(Extended)
for _, m := range []TokenizeMode{Normal, Search, Extended} {
la.Build("わたしまけましたわ")
la.Forward(m)
}
}
func TestFeatures02(t *testing.T) {
tok := Token{
ID: 0,
Class: TokenClass(lattice.UNKNOWN),
Start: 0,
End: 1,
Surface: "",
}
tok.dic = dic.SysDic()
f := tok.Features()
expected := []string{"名詞", "一般", "*", "*", "*", "*", "*"}
if !reflect.DeepEqual(f, expected) {
t.Errorf("got %v, expected %v", f, expected)
}
}
func TestLatticeString(t *testing.T) {
la := New(dic.SysDic(), nil)
if la == nil {
t.Fatal("cannot new a lattice")
}
defer la.Free()
expected := ""
str := la.String()
if str != expected {
t.Errorf("got %v, expected: %v", str, expected)
}
la.Build("わたしまけましたわ")
str = la.String()
if str == "" {
t.Errorf("got empty string")
}
}
func TestLatticeDot(t *testing.T) {
la := New(dic.SysDic(), nil)
if la == nil {
t.Fatal("cannot new a lattice")
}
defer la.Free()
expected := "graph lattice {\n\tdpi=48;\n\tgraph [style=filled, rankdir=LR]\n}\n"
var b bytes.Buffer
la.Dot(&b)
if b.String() != expected {
t.Errorf("got %v, expected: %v", b.String(), expected)
}
b.Reset()
la.Build("わたしまけましたわ")
la.Dot(&b)
if b.String() == "" {
t.Errorf("got empty string")
}
}
func TestFeatures04(t *testing.T) {
tok := Token{
ID: 0,
Class: DUMMY,
Start: 0,
End: 1,
Surface: "",
}
tok.dic = dic.SysDic()
if udic, e := dic.NewUserDic("../_sample/userdic.txt"); e != nil {
t.Fatalf("build user dic error: %v", e)
} else {
tok.udic = udic
}
f := tok.Features()
if len(f) != 0 {
t.Errorf("got %v, expected empty", f)
}
}
func TestFeatures03(t *testing.T) {
tok := Token{
ID: 0,
Class: TokenClass(lattice.USER),
Start: 0,
End: 1,
Surface: "",
}
tok.dic = dic.SysDic()
if udic, e := dic.NewUserDic("../_sample/userdic.txt"); e != nil {
t.Fatalf("build user dic error: %v", e)
} else {
tok.udic = udic
}
f := tok.Features()
expected := []string{"カスタム名詞", "日本/経済/新聞", "ニホン/ケイザイ/シンブン"}
if !reflect.DeepEqual(f, expected) {
t.Errorf("got %v, expected %v", f, expected)
}
}
// SysDic returns the system dictionary.
func SysDic() Dic {
return Dic{dic.SysDic()}
}