Ejemplo n.º 1
0
func TestFrenchElision(t *testing.T) {
	tests := []struct {
		input  analysis.TokenStream
		output analysis.TokenStream
	}{
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("l'avion"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("avion"),
				},
			},
		},
	}

	cache := registry.NewCache()
	elisionFilter, err := cache.TokenFilterNamed(ElisionName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := elisionFilter.Filter(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term)
		}
	}
}
Ejemplo n.º 2
0
func CommonBenchmarkIndexBatch(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers, batchSize int) {

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed("standard")
	if err != nil {
		b.Fatal(err)
	}

	b.ResetTimer()
	b.StopTimer()
	for i := 0; i < b.N; i++ {

		s, err := create()
		if err != nil {
			b.Fatal(err)
		}
		analysisQueue := NewAnalysisQueue(analysisWorkers)
		idx := NewUpsideDownCouch(s, analysisQueue)

		err = idx.Open()
		if err != nil {
			b.Fatal(err)
		}

		b.StartTimer()
		batch := index.NewBatch()
		for j := 0; j < 1000; j++ {
			if j%batchSize == 0 {
				if len(batch.IndexOps) > 0 {
					err := idx.Batch(batch)
					if err != nil {
						b.Fatal(err)
					}
				}
				batch = index.NewBatch()
			}
			indexDocument := document.NewDocument("").
				AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer))
			indexDocument.ID = strconv.Itoa(i) + "-" + strconv.Itoa(j)
			batch.Update(indexDocument)
		}
		// close last batch
		if len(batch.IndexOps) > 0 {
			err := idx.Batch(batch)
			if err != nil {
				b.Fatal(err)
			}
		}
		b.StopTimer()
		err = idx.Close()
		if err != nil {
			b.Fatal(err)
		}
		err = destroy()
		if err != nil {
			b.Fatal(err)
		}
		analysisQueue.Close()
	}
}
Ejemplo n.º 3
0
func TestStopWordsFilterLongestMatch(t *testing.T) {

	inputTokenStream := analysis.TokenStream{
		&analysis.Token{
			Term:     []byte("softestball"),
			Start:    0,
			End:      11,
			Position: 1,
		},
	}

	expectedTokenStream := analysis.TokenStream{
		&analysis.Token{
			Term:     []byte("softestball"),
			Start:    0,
			End:      11,
			Position: 1,
		},
		&analysis.Token{
			Term:     []byte("softest"),
			Start:    0,
			End:      7,
			Position: 1,
		},
		&analysis.Token{
			Term:     []byte("ball"),
			Start:    7,
			End:      11,
			Position: 1,
		},
	}

	cache := registry.NewCache()
	dictListConfig := map[string]interface{}{
		"type":   token_map.Name,
		"tokens": []interface{}{"soft", "softest", "ball"},
	}
	_, err := cache.DefineTokenMap("dict_test", dictListConfig)
	if err != nil {
		t.Fatal(err)
	}

	dictConfig := map[string]interface{}{
		"type":               "dict_compound",
		"dict_token_map":     "dict_test",
		"only_longest_match": true,
	}
	dictFilter, err := cache.DefineTokenFilter("dict_test", dictConfig)
	if err != nil {
		t.Fatal(err)
	}

	ouputTokenStream := dictFilter.Filter(inputTokenStream)
	if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) {
		t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream)
	}
}
func TestStopWordsFilter(t *testing.T) {

	inputTokenStream := analysis.TokenStream{
		&analysis.Token{
			Term: []byte("a"),
		},
		&analysis.Token{
			Term: []byte("walk"),
		},
		&analysis.Token{
			Term: []byte("in"),
		},
		&analysis.Token{
			Term: []byte("the"),
		},
		&analysis.Token{
			Term: []byte("park"),
		},
	}

	expectedTokenStream := analysis.TokenStream{
		&analysis.Token{
			Term: []byte("walk"),
		},
		&analysis.Token{
			Term: []byte("park"),
		},
	}

	cache := registry.NewCache()
	stopListConfig := map[string]interface{}{
		"type":   token_map.Name,
		"tokens": []interface{}{"a", "in", "the"},
	}
	_, err := cache.DefineTokenMap("stop_test", stopListConfig)
	if err != nil {
		t.Fatal(err)
	}

	stopConfig := map[string]interface{}{
		"type":           "stop_tokens",
		"stop_token_map": "stop_test",
	}
	stopFilter, err := cache.DefineTokenFilter("stop_test", stopConfig)
	if err != nil {
		t.Fatal(err)
	}

	ouputTokenStream := stopFilter.Filter(inputTokenStream)
	if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) {
		t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream)
	}
}
Ejemplo n.º 5
0
func TestSoraniAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// stop word removal
		{
			input: []byte("ئەم پیاوە"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("پیاو"),
					Position: 2,
					Start:    7,
					End:      17,
				},
			},
		},
		{
			input: []byte("پیاوە"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("پیاو"),
					Position: 1,
					Start:    0,
					End:      10,
				},
			},
		},
		{
			input: []byte("پیاو"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("پیاو"),
					Position: 1,
					Start:    0,
					End:      8,
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %v, got %v", test.output, actual)
		}
	}
}
Ejemplo n.º 6
0
func TestThaiAnalyzerWihtoutOffsets(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// stop words
		{
			input: []byte("บริษัทชื่อ XY&Z - คุยกับ [email protected]"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("บริษัท"),
				},
				&analysis.Token{
					Term: []byte("ชื่อ"),
				},
				&analysis.Token{
					Term: []byte("xy"),
				},
				&analysis.Token{
					Term: []byte("z"),
				},
				&analysis.Token{
					Term: []byte("คุย"),
				},
				&analysis.Token{
					Term: []byte("xyz"),
				},
				&analysis.Token{
					Term: []byte("demo.com"),
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if len(actual) != len(test.output) {
			t.Errorf("expected length: %d, got %d", len(test.output), len(actual))
		}
		for i, tok := range actual {
			if !reflect.DeepEqual(tok.Term, test.output[i].Term) {
				t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)
			}
		}
	}
}
Ejemplo n.º 7
0
func TestEnglishStemmer(t *testing.T) {
	tests := []struct {
		input  analysis.TokenStream
		output analysis.TokenStream
	}{
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("walking"),
				},
				&analysis.Token{
					Term: []byte("talked"),
				},
				&analysis.Token{
					Term: []byte("business"),
				},
				&analysis.Token{
					Term:    []byte("protected"),
					KeyWord: true,
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("walk"),
				},
				&analysis.Token{
					Term: []byte("talk"),
				},
				&analysis.Token{
					Term: []byte("busi"),
				},
				&analysis.Token{
					Term:    []byte("protected"),
					KeyWord: true,
				},
			},
		},
	}

	cache := registry.NewCache()
	stemmerFilter, err := cache.TokenFilterNamed(StemmerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := stemmerFilter.Filter(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %s, got %s", test.output, actual)
		}
	}
}
Ejemplo n.º 8
0
func TestJaAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		{
			input: []byte("こんにちは世界"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("こんにちは"),
					Type:     analysis.Ideographic,
					Position: 1,
					Start:    0,
					End:      15,
				},
				&analysis.Token{
					Term:     []byte("世界"),
					Type:     analysis.Ideographic,
					Position: 2,
					Start:    15,
					End:      21,
				},
			},
		},
		{
			input: []byte("カタカナ"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("カタカナ"),
					Type:     analysis.Ideographic,
					Position: 1,
					Start:    0,
					End:      12,
				},
			},
		},
	}

	cache := registry.NewCache()
	for _, test := range tests {
		analyzer, err := cache.AnalyzerNamed(AnalyzerName)
		if err != nil {
			t.Fatal(err)
		}
		actual := analyzer.Analyze(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %v, got %v", test.output, actual)
		}
	}
}
Ejemplo n.º 9
0
// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules
func NewIndexMapping() *IndexMapping {
	return &IndexMapping{
		TypeMapping:           make(map[string]*DocumentMapping),
		DefaultMapping:        NewDocumentMapping(),
		TypeField:             defaultTypeField,
		DefaultType:           defaultType,
		DefaultAnalyzer:       defaultAnalyzer,
		DefaultDateTimeParser: defaultDateTimeParser,
		DefaultField:          defaultField,
		ByteArrayConverter:    defaultByteArrayConverter,
		CustomAnalysis:        newCustomAnalysis(),
		cache:                 registry.NewCache(),
	}
}
Ejemplo n.º 10
0
func TestElisionFilter(t *testing.T) {

	tests := []struct {
		input  analysis.TokenStream
		output analysis.TokenStream
	}{
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("ar'word"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("word"),
				},
			},
		},
	}

	cache := registry.NewCache()

	articleListConfig := map[string]interface{}{
		"type":   token_map.Name,
		"tokens": []interface{}{"ar"},
	}
	_, err := cache.DefineTokenMap("articles_test", articleListConfig)
	if err != nil {
		t.Fatal(err)
	}

	elisionConfig := map[string]interface{}{
		"type":               "elision",
		"articles_token_map": "articles_test",
	}
	elisionFilter, err := cache.DefineTokenFilter("elision_test", elisionConfig)
	if err != nil {
		t.Fatal(err)
	}

	for _, test := range tests {

		actual := elisionFilter.Filter(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term)
		}
	}
}
Ejemplo n.º 11
0
func TestDanishAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// stemming
		{
			input: []byte("undersøg"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("undersøg"),
					Position: 1,
					Start:    0,
					End:      9,
				},
			},
		},
		{
			input: []byte("undersøgelse"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("undersøg"),
					Position: 1,
					Start:    0,
					End:      13,
				},
			},
		},
		// stop word
		{
			input:  []byte("på"),
			output: analysis.TokenStream{},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %v, got %v", test.output, actual)
		}
	}
}
Ejemplo n.º 12
0
func TestRomanianAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// stemming
		{
			input: []byte("absenţa"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("absenţ"),
				},
			},
		},
		{
			input: []byte("absenţi"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("absenţ"),
				},
			},
		},
		// stop word
		{
			input:  []byte("îl"),
			output: analysis.TokenStream{},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if len(actual) != len(test.output) {
			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
		}
		for i, tok := range actual {
			if !reflect.DeepEqual(tok.Term, test.output[i].Term) {
				t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)
			}
		}
	}
}
Ejemplo n.º 13
0
func CommonBenchmarkIndex(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers int) {

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed("standard")
	if err != nil {
		b.Fatal(err)
	}

	indexDocument := document.NewDocument("").
		AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[0]), analyzer))

	b.ResetTimer()
	b.StopTimer()
	for i := 0; i < b.N; i++ {
		s, err := create()
		if err != nil {
			b.Fatal(err)
		}
		analysisQueue := NewAnalysisQueue(analysisWorkers)
		idx := NewUpsideDownCouch(s, analysisQueue)

		err = idx.Open()
		if err != nil {
			b.Fatal(err)
		}
		indexDocument.ID = strconv.Itoa(i)
		// just time the indexing portion
		b.StartTimer()
		err = idx.Update(indexDocument)
		if err != nil {
			b.Fatal(err)
		}
		b.StopTimer()
		err = idx.Close()
		if err != nil {
			b.Fatal(err)
		}
		err = destroy()
		if err != nil {
			b.Fatal(err)
		}
		analysisQueue.Close()
	}
}
Ejemplo n.º 14
0
func TestThaiAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// stop words
		{
			input: []byte("การที่ได้ต้องแสดงว่างานดี"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("แสดง"),
					Position: 5,
					Start:    39,
					End:      51,
				},
				&analysis.Token{
					Term:     []byte("งาน"),
					Position: 7,
					Start:    60,
					End:      69,
				},
				&analysis.Token{
					Term:     []byte("ดี"),
					Position: 8,
					Start:    69,
					End:      75,
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %v, got %v", test.output, actual)
		}
	}
}
Ejemplo n.º 15
0
func TestItalianLightStemmer(t *testing.T) {
	tests := []struct {
		input  analysis.TokenStream
		output analysis.TokenStream
	}{
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("ragazzo"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("ragazz"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("ragazzi"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("ragazz"),
				},
			},
		},
	}

	cache := registry.NewCache()
	filter, err := cache.TokenFilterNamed(LightStemmerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := filter.Filter(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term)
		}
	}
}
Ejemplo n.º 16
0
func TestHindiAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// two ways to write 'hindi' itself
		{
			input: []byte("हिन्दी"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("हिंद"),
					Position: 1,
					Start:    0,
					End:      18,
				},
			},
		},
		{
			input: []byte("हिंदी"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("हिंद"),
					Position: 1,
					Start:    0,
					End:      15,
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %v, got %v", test.output, actual)
		}
	}
}
Ejemplo n.º 17
0
func TestPersianAnalyzerVerbsDefective(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// active present indicative
		{
			input: []byte("مي خورد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورد"),
				},
			},
		},
		// active preterite indicative
		{
			input: []byte("خورد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورد"),
				},
			},
		},
		// active imperfective preterite indicative
		{
			input: []byte("مي خورد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورد"),
				},
			},
		},
		// active future indicative
		{
			input: []byte("خواهد خورد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورد"),
				},
			},
		},
		// active present progressive indicative
		{
			input: []byte("دارد مي خورد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورد"),
				},
			},
		},
		// active preterite progressive indicative
		{
			input: []byte("داشت مي خورد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورد"),
				},
			},
		},
		// active perfect indicative
		{
			input: []byte("خورده است"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// active imperfective perfect indicative
		{
			input: []byte("مي خورده است"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// active pluperfect indicative
		{
			input: []byte("خورده بود"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// active imperfective pluperfect indicative
		{
			input: []byte("مي خورده بود"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// active preterite subjunctive
		{
			input: []byte("خورده باشد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// active imperfective preterite subjunctive
		{
			input: []byte("مي خورده باشد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// active pluperfect subjunctive
		{
			input: []byte("خورده بوده باشد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// active imperfective pluperfect subjunctive
		{
			input: []byte("مي خورده بوده باشد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive present indicative
		{
			input: []byte("خورده مي شود"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive preterite indicative
		{
			input: []byte("خورده شد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive imperfective preterite indicative
		{
			input: []byte("خورده مي شد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive perfect indicative
		{
			input: []byte("خورده شده است"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive imperfective perfect indicative
		{
			input: []byte("خورده مي شده است"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive pluperfect indicative
		{
			input: []byte("خورده شده بود"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive imperfective pluperfect indicative
		{
			input: []byte("خورده مي شده بود"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive future indicative
		{
			input: []byte("خورده خواهد شد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive present progressive indicative
		{
			input: []byte("دارد خورده مي شود"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive preterite progressive indicative
		{
			input: []byte("داشت خورده مي شد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive present subjunctive
		{
			input: []byte("خورده شود"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive preterite subjunctive
		{
			input: []byte("خورده شده باشد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive imperfective preterite subjunctive
		{
			input: []byte("خورده مي شده باشد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive pluperfect subjunctive
		{
			input: []byte("خورده شده بوده باشد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// passive imperfective pluperfect subjunctive
		{
			input: []byte("خورده مي شده بوده باشد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		// active present subjunctive
		{
			input: []byte("بخورد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("بخورد"),
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if len(actual) != len(test.output) {
			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
		}
		for i, tok := range actual {
			if !reflect.DeepEqual(tok.Term, test.output[i].Term) {
				t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)
			}
		}
	}
}
Ejemplo n.º 18
0
func TestFrenchAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		{
			input:  []byte(""),
			output: analysis.TokenStream{},
		},
		{
			input: []byte("chien chat cheval"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chien"),
				},
				&analysis.Token{
					Term: []byte("chat"),
				},
				&analysis.Token{
					Term: []byte("cheval"),
				},
			},
		},
		{
			input: []byte("chien CHAT CHEVAL"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chien"),
				},
				&analysis.Token{
					Term: []byte("chat"),
				},
				&analysis.Token{
					Term: []byte("cheval"),
				},
			},
		},
		{
			input: []byte("  chien  ,? + = -  CHAT /: > CHEVAL"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chien"),
				},
				&analysis.Token{
					Term: []byte("chat"),
				},
				&analysis.Token{
					Term: []byte("cheval"),
				},
			},
		},
		{
			input: []byte("chien++"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chien"),
				},
			},
		},
		{
			input: []byte("mot \"entreguillemet\""),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("mot"),
				},
				&analysis.Token{
					Term: []byte("entreguilemet"),
				},
			},
		},
		{
			input: []byte("Jean-François"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("jean"),
				},
				&analysis.Token{
					Term: []byte("francoi"),
				},
			},
		},
		// stop words
		{
			input: []byte("le la chien les aux chat du des à cheval"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chien"),
				},
				&analysis.Token{
					Term: []byte("chat"),
				},
				&analysis.Token{
					Term: []byte("cheval"),
				},
			},
		},
		// nouns and adjectives
		{
			input: []byte("lances chismes habitable chiste éléments captifs"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("lanc"),
				},
				&analysis.Token{
					Term: []byte("chism"),
				},
				&analysis.Token{
					Term: []byte("habitabl"),
				},
				&analysis.Token{
					Term: []byte("chist"),
				},
				&analysis.Token{
					Term: []byte("element"),
				},
				&analysis.Token{
					Term: []byte("captif"),
				},
			},
		},
		// verbs
		{
			input: []byte("finissions souffrirent rugissante"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("finision"),
				},
				&analysis.Token{
					Term: []byte("soufrirent"),
				},
				&analysis.Token{
					Term: []byte("rugisant"),
				},
			},
		},
		{
			input: []byte("C3PO aujourd'hui oeuf ïâöûàä anticonstitutionnellement Java++ "),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("c3po"),
				},
				&analysis.Token{
					Term: []byte("aujourd'hui"),
				},
				&analysis.Token{
					Term: []byte("oeuf"),
				},
				&analysis.Token{
					Term: []byte("ïaöuaä"),
				},
				&analysis.Token{
					Term: []byte("anticonstitutionel"),
				},
				&analysis.Token{
					Term: []byte("java"),
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if len(actual) != len(test.output) {
			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
		}
		for i, tok := range actual {
			if !reflect.DeepEqual(tok.Term, test.output[i].Term) {
				t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)
			}
		}
	}
}
Ejemplo n.º 19
0
func TestTurkishAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// stemming
		{
			input: []byte("ağacı"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("ağaç"),
				},
			},
		},
		{
			input: []byte("ağaç"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("ağaç"),
				},
			},
		},
		// stop word
		{
			input:  []byte("dolayı"),
			output: analysis.TokenStream{},
		},
		// apostrophes
		{
			input: []byte("Kıbrıs'ta"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("kıbrıs"),
				},
			},
		},
		{
			input: []byte("Van Gölü'ne"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("van"),
				},
				&analysis.Token{
					Term: []byte("göl"),
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if len(actual) != len(test.output) {
			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
		}
		for i, tok := range actual {
			if !reflect.DeepEqual(tok.Term, test.output[i].Term) {
				t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)
			}
		}
	}
}
Ejemplo n.º 20
0
// UnmarshalJSON deserializes a JSON representation of the IndexMapping
func (im *IndexMapping) UnmarshalJSON(data []byte) error {
	var tmp struct {
		TypeMapping           map[string]*DocumentMapping `json:"types"`
		DefaultMapping        *DocumentMapping            `json:"default_mapping"`
		TypeField             string                      `json:"type_field"`
		DefaultType           string                      `json:"default_type"`
		DefaultAnalyzer       string                      `json:"default_analyzer"`
		DefaultDateTimeParser string                      `json:"default_datetime_parser"`
		DefaultField          string                      `json:"default_field"`
		ByteArrayConverter    string                      `json:"byte_array_converter"`
		CustomAnalysis        *customAnalysis             `json:"analysis"`
	}
	err := json.Unmarshal(data, &tmp)
	if err != nil {
		return err
	}

	im.cache = registry.NewCache()

	im.CustomAnalysis = newCustomAnalysis()
	if tmp.CustomAnalysis != nil {
		if tmp.CustomAnalysis.CharFilters != nil {
			im.CustomAnalysis.CharFilters = tmp.CustomAnalysis.CharFilters
		}
		if tmp.CustomAnalysis.Tokenizers != nil {
			im.CustomAnalysis.Tokenizers = tmp.CustomAnalysis.Tokenizers
		}
		if tmp.CustomAnalysis.TokenMaps != nil {
			im.CustomAnalysis.TokenMaps = tmp.CustomAnalysis.TokenMaps
		}
		if tmp.CustomAnalysis.TokenFilters != nil {
			im.CustomAnalysis.TokenFilters = tmp.CustomAnalysis.TokenFilters
		}
		if tmp.CustomAnalysis.Analyzers != nil {
			im.CustomAnalysis.Analyzers = tmp.CustomAnalysis.Analyzers
		}
		if tmp.CustomAnalysis.DateTimeParsers != nil {
			im.CustomAnalysis.DateTimeParsers = tmp.CustomAnalysis.DateTimeParsers
		}
	}

	im.TypeField = defaultTypeField
	if tmp.TypeField != "" {
		im.TypeField = tmp.TypeField
	}

	im.DefaultType = defaultType
	if tmp.DefaultType != "" {
		im.DefaultType = tmp.DefaultType
	}

	im.DefaultAnalyzer = defaultAnalyzer
	if tmp.DefaultAnalyzer != "" {
		im.DefaultAnalyzer = tmp.DefaultAnalyzer
	}

	im.DefaultDateTimeParser = defaultDateTimeParser
	if tmp.DefaultDateTimeParser != "" {
		im.DefaultDateTimeParser = tmp.DefaultDateTimeParser
	}

	im.DefaultField = defaultField
	if tmp.DefaultField != "" {
		im.DefaultField = tmp.DefaultField
	}

	im.ByteArrayConverter = defaultByteArrayConverter
	if tmp.ByteArrayConverter != "" {
		im.ByteArrayConverter = tmp.ByteArrayConverter
	}

	im.DefaultMapping = NewDocumentMapping()
	if tmp.DefaultMapping != nil {
		im.DefaultMapping = tmp.DefaultMapping
	}

	im.TypeMapping = make(map[string]*DocumentMapping, len(tmp.TypeMapping))
	for typeName, typeDocMapping := range tmp.TypeMapping {
		im.TypeMapping[typeName] = typeDocMapping
	}

	err = im.CustomAnalysis.registerAll(im)
	if err != nil {
		return err
	}

	return nil
}
Ejemplo n.º 21
0
func TestStopWordsFilter(t *testing.T) {

	inputTokenStream := analysis.TokenStream{
		&analysis.Token{
			Term:     []byte("i"),
			Start:    0,
			End:      1,
			Position: 1,
		},
		&analysis.Token{
			Term:     []byte("like"),
			Start:    2,
			End:      6,
			Position: 2,
		},
		&analysis.Token{
			Term:     []byte("to"),
			Start:    7,
			End:      9,
			Position: 3,
		},
		&analysis.Token{
			Term:     []byte("play"),
			Start:    10,
			End:      14,
			Position: 4,
		},
		&analysis.Token{
			Term:     []byte("softball"),
			Start:    15,
			End:      23,
			Position: 5,
		},
	}

	expectedTokenStream := analysis.TokenStream{
		&analysis.Token{
			Term:     []byte("i"),
			Start:    0,
			End:      1,
			Position: 1,
		},
		&analysis.Token{
			Term:     []byte("like"),
			Start:    2,
			End:      6,
			Position: 2,
		},
		&analysis.Token{
			Term:     []byte("to"),
			Start:    7,
			End:      9,
			Position: 3,
		},
		&analysis.Token{
			Term:     []byte("play"),
			Start:    10,
			End:      14,
			Position: 4,
		},
		&analysis.Token{
			Term:     []byte("softball"),
			Start:    15,
			End:      23,
			Position: 5,
		},
		&analysis.Token{
			Term:     []byte("soft"),
			Start:    15,
			End:      19,
			Position: 5,
		},
		&analysis.Token{
			Term:     []byte("ball"),
			Start:    19,
			End:      23,
			Position: 5,
		},
	}

	cache := registry.NewCache()
	dictListConfig := map[string]interface{}{
		"type":   token_map.Name,
		"tokens": []interface{}{"factor", "soft", "ball", "team"},
	}
	_, err := cache.DefineTokenMap("dict_test", dictListConfig)
	if err != nil {
		t.Fatal(err)
	}

	dictConfig := map[string]interface{}{
		"type":           "dict_compound",
		"dict_token_map": "dict_test",
	}
	dictFilter, err := cache.DefineTokenFilter("dict_test", dictConfig)
	if err != nil {
		t.Fatal(err)
	}

	ouputTokenStream := dictFilter.Filter(inputTokenStream)
	if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) {
		t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream)
	}
}
Ejemplo n.º 22
0
func TestGermanAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		{
			input: []byte("Tisch"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("tisch"),
					Position: 1,
					Start:    0,
					End:      5,
				},
			},
		},
		{
			input: []byte("Tische"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("tisch"),
					Position: 1,
					Start:    0,
					End:      6,
				},
			},
		},
		{
			input: []byte("Tischen"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("tisch"),
					Position: 1,
					Start:    0,
					End:      7,
				},
			},
		},
		// german specials
		{
			input: []byte("Schaltflächen"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("schaltflach"),
					Position: 1,
					Start:    0,
					End:      14,
				},
			},
		},
		{
			input: []byte("Schaltflaechen"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("schaltflach"),
					Position: 1,
					Start:    0,
					End:      14,
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %v, got %v", test.output, actual)
		}
	}
}
Ejemplo n.º 23
0
func TestEnglishAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// stemming
		{
			input: []byte("books"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("book"),
					Position: 1,
					Start:    0,
					End:      5,
				},
			},
		},
		{
			input: []byte("book"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("book"),
					Position: 1,
					Start:    0,
					End:      4,
				},
			},
		},
		// stop word removal
		{
			input:  []byte("the"),
			output: analysis.TokenStream{},
		},
		// possessive removal
		{
			input: []byte("steven's"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("steven"),
					Position: 1,
					Start:    0,
					End:      8,
				},
			},
		},
		{
			input: []byte("steven\u2019s"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("steven"),
					Position: 1,
					Start:    0,
					End:      10,
				},
			},
		},
		{
			input: []byte("steven\uFF07s"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("steven"),
					Position: 1,
					Start:    0,
					End:      10,
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %v, got %v", test.output, actual)
		}
	}
}
Ejemplo n.º 24
0
func TestExceptionsTokenizer(t *testing.T) {
	tests := []struct {
		config   map[string]interface{}
		input    []byte
		patterns []string
		result   analysis.TokenStream
	}{
		{
			input: []byte("test http://blevesearch.com/ words"),
			config: map[string]interface{}{
				"type":      "exception",
				"tokenizer": "unicode",
				"exceptions": []interface{}{
					`[hH][tT][tT][pP][sS]?://(\S)*`,
					`[fF][iI][lL][eE]://(\S)*`,
					`[fF][tT][pP]://(\S)*`,
				},
			},
			result: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("test"),
					Position: 1,
					Start:    0,
					End:      4,
				},
				&analysis.Token{
					Term:     []byte("http://blevesearch.com/"),
					Position: 2,
					Start:    5,
					End:      28,
				},
				&analysis.Token{
					Term:     []byte("words"),
					Position: 3,
					Start:    29,
					End:      34,
				},
			},
		},
		{
			input: []byte("what ftp://blevesearch.com/ songs"),
			config: map[string]interface{}{
				"type":      "exception",
				"tokenizer": "unicode",
				"exceptions": []interface{}{
					`[hH][tT][tT][pP][sS]?://(\S)*`,
					`[fF][iI][lL][eE]://(\S)*`,
					`[fF][tT][pP]://(\S)*`,
				},
			},
			result: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("what"),
					Position: 1,
					Start:    0,
					End:      4,
				},
				&analysis.Token{
					Term:     []byte("ftp://blevesearch.com/"),
					Position: 2,
					Start:    5,
					End:      27,
				},
				&analysis.Token{
					Term:     []byte("songs"),
					Position: 3,
					Start:    28,
					End:      33,
				},
			},
		},
		{
			input: []byte("please email [email protected] the URL https://blevesearch.com/"),
			config: map[string]interface{}{
				"type":      "exception",
				"tokenizer": "unicode",
				"exceptions": []interface{}{
					`[hH][tT][tT][pP][sS]?://(\S)*`,
					`[fF][iI][lL][eE]://(\S)*`,
					`[fF][tT][pP]://(\S)*`,
					`\S+@\S+`,
				},
			},
			result: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("please"),
					Position: 1,
					Start:    0,
					End:      6,
				},
				&analysis.Token{
					Term:     []byte("email"),
					Position: 2,
					Start:    7,
					End:      12,
				},
				&analysis.Token{
					Term:     []byte("*****@*****.**"),
					Position: 3,
					Start:    13,
					End:      32,
				},
				&analysis.Token{
					Term:     []byte("the"),
					Position: 4,
					Start:    33,
					End:      36,
				},
				&analysis.Token{
					Term:     []byte("URL"),
					Position: 5,
					Start:    37,
					End:      40,
				},
				&analysis.Token{
					Term:     []byte("https://blevesearch.com/"),
					Position: 6,
					Start:    41,
					End:      65,
				},
			},
		},
	}

	// remaining := unicode.NewUnicodeTokenizer()
	for _, test := range tests {

		// build the requested exception tokenizer
		cache := registry.NewCache()
		tokenizer, err := cache.DefineTokenizer("custom", test.config)
		if err != nil {
			t.Fatal(err)
		}

		// pattern := strings.Join(test.patterns, "|")
		// r, err := regexp.Compile(pattern)
		// if err != nil {
		// 	t.Fatal(err)
		// }
		// tokenizer := NewExceptionsTokenizer(r, remaining)
		actual := tokenizer.Tokenize(test.input)
		if !reflect.DeepEqual(actual, test.result) {
			t.Errorf("expected %v, got %v", test.result, actual)
		}
	}
}
Ejemplo n.º 25
0
func newConfiguration() *configuration {
	return &configuration{
		Cache:         registry.NewCache(),
		analysisQueue: upside_down.NewAnalysisQueue(4),
	}
}
Ejemplo n.º 26
0
func TestFrenchLightStemmer(t *testing.T) {
	tests := []struct {
		input  analysis.TokenStream
		output analysis.TokenStream
	}{
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chevaux"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("cheval"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("cheval"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("cheval"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("hiboux"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("hibou"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("hibou"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("hibou"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chantés"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chant"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chanter"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chant"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chante"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chant"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chant"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("chant"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("baronnes"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("baron"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("barons"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("baron"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("baron"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("baron"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("peaux"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("peau"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("peau"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("peau"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("anneaux"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("aneau"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("anneau"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("aneau"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("neveux"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("neveu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("neveu"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("neveu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("affreux"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("afreu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("affreuse"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("afreu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("investissement"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("investi"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("investir"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("investi"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("assourdissant"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("asourdi"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("assourdir"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("asourdi"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("pratiquement"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("pratiqu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("pratique"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("pratiqu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("administrativement"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("administratif"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("administratif"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("administratif"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("justificatrice"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("justifi"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("justificateur"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("justifi"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("justifier"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("justifi"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("educatrice"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("eduqu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("eduquer"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("eduqu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("communicateur"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("comuniqu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("communiquer"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("comuniqu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("accompagnatrice"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("acompagn"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("accompagnateur"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("acompagn"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("administrateur"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("administr"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("administrer"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("administr"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("productrice"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("product"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("producteur"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("product"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("acheteuse"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("achet"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("acheteur"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("achet"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("planteur"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("plant"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("plante"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("plant"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("poreuse"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("poreu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("poreux"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("poreu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("plieuse"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("plieu"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("bijoutière"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("bijouti"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("bijoutier"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("bijouti"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("caissière"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("caisi"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("caissier"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("caisi"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("abrasive"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("abrasif"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("abrasif"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("abrasif"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("folle"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("fou"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("fou"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("fou"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("personnelle"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("person"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("personne"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("person"),
				},
			},
		},
		// algo bug: too short length
		// {
		// 	input: analysis.TokenStream{
		// 		&analysis.Token{
		// 			Term: []byte("personnel"),
		// 		},
		// 	},
		// 	output: analysis.TokenStream{
		// 		&analysis.Token{
		// 			Term: []byte("person"),
		// 		},
		// 	},
		// },
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("complète"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("complet"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("complet"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("complet"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("aromatique"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("aromat"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("faiblesse"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("faibl"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("faible"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("faibl"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("patinage"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("patin"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("patin"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("patin"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("sonorisation"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("sono"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("ritualisation"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("rituel"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("rituel"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("rituel"),
				},
			},
		},
		// algo bug: masked by rules above
		// {
		// 	input: analysis.TokenStream{
		// 		&analysis.Token{
		// 			Term: []byte("colonisateur"),
		// 		},
		// 	},
		// 	output: analysis.TokenStream{
		// 		&analysis.Token{
		// 			Term: []byte("colon"),
		// 		},
		// 	},
		// },
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("nomination"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("nomin"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("disposition"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("dispos"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("dispose"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("dispos"),
				},
			},
		},
		// SOLR-3463 : abusive compression of repeated characters in numbers
		// Trailing repeated char elision :
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("1234555"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("1234555"),
				},
			},
		},
		// Repeated char within numbers with more than 4 characters :
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("12333345"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("12333345"),
				},
			},
		},
		// Short numbers weren't affected already:
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("1234"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("1234"),
				},
			},
		},
		// Ensure behaviour is preserved for words!
		// Trailing repeated char elision :
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("abcdeff"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("abcdef"),
				},
			},
		},
		// Repeated char within words with more than 4 characters :
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("abcccddeef"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("abcdef"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("créées"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("cre"),
				},
			},
		},
		// Combined letter and digit repetition
		// 10:00pm
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("22hh00"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("22h00"),
				},
			},
		},
	}

	cache := registry.NewCache()
	filter, err := cache.TokenFilterNamed(LightStemmerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := filter.Filter(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term)
		}
	}
}
Ejemplo n.º 27
0
func TestRussianAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// digits safe
		{
			input: []byte("text 1000"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("text"),
				},
				&analysis.Token{
					Term: []byte("1000"),
				},
			},
		},
		{
			input: []byte("Вместе с тем о силе электромагнитной энергии имели представление еще"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("вмест"),
				},
				&analysis.Token{
					Term: []byte("сил"),
				},
				&analysis.Token{
					Term: []byte("электромагнитн"),
				},
				&analysis.Token{
					Term: []byte("энерг"),
				},
				&analysis.Token{
					Term: []byte("имел"),
				},
				&analysis.Token{
					Term: []byte("представлен"),
				},
			},
		},
		{
			input: []byte("Но знание это хранилось в тайне"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("знан"),
				},
				&analysis.Token{
					Term: []byte("эт"),
				},
				&analysis.Token{
					Term: []byte("хран"),
				},
				&analysis.Token{
					Term: []byte("тайн"),
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if len(actual) != len(test.output) {
			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
		}
		for i, tok := range actual {
			if !reflect.DeepEqual(tok.Term, test.output[i].Term) {
				t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)
			}
		}
	}
}
Ejemplo n.º 28
0
func TestPortugueseLightStemmer(t *testing.T) {
	tests := []struct {
		input  analysis.TokenStream
		output analysis.TokenStream
	}{
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("doutores"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("doutor"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("doutor"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("doutor"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("homens"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("homem"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("homem"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("homem"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("papéis"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("papel"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("papel"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("papel"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("normais"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("normal"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("normal"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("normal"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("lencóis"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("lencol"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("lencol"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("lencol"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("barris"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("barril"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("barril"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("barril"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("botões"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("bota"),
				},
			},
		},
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("botão"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("bota"),
				},
			},
		},
		// longer
		{
			input: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("o"),
				},
				&analysis.Token{
					Term: []byte("debate"),
				},
				&analysis.Token{
					Term: []byte("político"),
				},
				&analysis.Token{
					Term: []byte("pelo"),
				},
				&analysis.Token{
					Term: []byte("menos"),
				},
				&analysis.Token{
					Term: []byte("o"),
				},
				&analysis.Token{
					Term: []byte("que"),
				},
				&analysis.Token{
					Term: []byte("vem"),
				},
				&analysis.Token{
					Term: []byte("a"),
				},
				&analysis.Token{
					Term: []byte("público"),
				},
				&analysis.Token{
					Term: []byte("parece"),
				},
				&analysis.Token{
					Term: []byte("de"),
				},
				&analysis.Token{
					Term: []byte("modo"),
				},
				&analysis.Token{
					Term: []byte("nada"),
				},
				&analysis.Token{
					Term: []byte("surpreendente"),
				},
				&analysis.Token{
					Term: []byte("restrito"),
				},
				&analysis.Token{
					Term: []byte("a"),
				},
				&analysis.Token{
					Term: []byte("temas"),
				},
				&analysis.Token{
					Term: []byte("menores"),
				},
				&analysis.Token{
					Term: []byte("mas"),
				},
				&analysis.Token{
					Term: []byte("há"),
				},
				&analysis.Token{
					Term: []byte("evidentemente"),
				},
				&analysis.Token{
					Term: []byte("grandes"),
				},
				&analysis.Token{
					Term: []byte("questões"),
				},
				&analysis.Token{
					Term: []byte("em"),
				},
				&analysis.Token{
					Term: []byte("jogo"),
				},
				&analysis.Token{
					Term: []byte("nas"),
				},
				&analysis.Token{
					Term: []byte("eleições"),
				},
				&analysis.Token{
					Term: []byte("que"),
				},
				&analysis.Token{
					Term: []byte("se"),
				},
				&analysis.Token{
					Term: []byte("aproximam"),
				},
			},
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("o"),
				},
				&analysis.Token{
					Term: []byte("debat"),
				},
				&analysis.Token{
					Term: []byte("politic"),
				},
				&analysis.Token{
					Term: []byte("pelo"),
				},
				&analysis.Token{
					Term: []byte("meno"),
				},
				&analysis.Token{
					Term: []byte("o"),
				},
				&analysis.Token{
					Term: []byte("que"),
				},
				&analysis.Token{
					Term: []byte("vem"),
				},
				&analysis.Token{
					Term: []byte("a"),
				},
				&analysis.Token{
					Term: []byte("public"),
				},
				&analysis.Token{
					Term: []byte("parec"),
				},
				&analysis.Token{
					Term: []byte("de"),
				},
				&analysis.Token{
					Term: []byte("modo"),
				},
				&analysis.Token{
					Term: []byte("nada"),
				},
				&analysis.Token{
					Term: []byte("surpreendent"),
				},
				&analysis.Token{
					Term: []byte("restrit"),
				},
				&analysis.Token{
					Term: []byte("a"),
				},
				&analysis.Token{
					Term: []byte("tema"),
				},
				&analysis.Token{
					Term: []byte("menor"),
				},
				&analysis.Token{
					Term: []byte("mas"),
				},
				&analysis.Token{
					Term: []byte("há"),
				},
				&analysis.Token{
					Term: []byte("evident"),
				},
				&analysis.Token{
					Term: []byte("grand"),
				},
				&analysis.Token{
					Term: []byte("questa"),
				},
				&analysis.Token{
					Term: []byte("em"),
				},
				&analysis.Token{
					Term: []byte("jogo"),
				},
				&analysis.Token{
					Term: []byte("nas"),
				},
				&analysis.Token{
					Term: []byte("eleica"),
				},
				&analysis.Token{
					Term: []byte("que"),
				},
				&analysis.Token{
					Term: []byte("se"),
				},
				&analysis.Token{
					Term: []byte("aproximam"),
				},
			},
		},
	}

	cache := registry.NewCache()
	filter, err := cache.TokenFilterNamed(LightStemmerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := filter.Filter(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term)
		}
	}
}
Ejemplo n.º 29
0
func TestCJKAnalyzer(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		{
			input: []byte("こんにちは世界"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("こん"),
					Type:     analysis.Double,
					Position: 1,
					Start:    0,
					End:      6,
				},
				&analysis.Token{
					Term:     []byte("んに"),
					Type:     analysis.Double,
					Position: 2,
					Start:    3,
					End:      9,
				},
				&analysis.Token{
					Term:     []byte("にち"),
					Type:     analysis.Double,
					Position: 3,
					Start:    6,
					End:      12,
				},
				&analysis.Token{
					Term:     []byte("ちは"),
					Type:     analysis.Double,
					Position: 4,
					Start:    9,
					End:      15,
				},
				&analysis.Token{
					Term:     []byte("は世"),
					Type:     analysis.Double,
					Position: 5,
					Start:    12,
					End:      18,
				},
				&analysis.Token{
					Term:     []byte("世界"),
					Type:     analysis.Double,
					Position: 6,
					Start:    15,
					End:      21,
				},
			},
		},
		{
			input: []byte("一二三四五六七八九十"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("一二"),
					Type:     analysis.Double,
					Position: 1,
					Start:    0,
					End:      6,
				},
				&analysis.Token{
					Term:     []byte("二三"),
					Type:     analysis.Double,
					Position: 2,
					Start:    3,
					End:      9,
				},
				&analysis.Token{
					Term:     []byte("三四"),
					Type:     analysis.Double,
					Position: 3,
					Start:    6,
					End:      12,
				},
				&analysis.Token{
					Term:     []byte("四五"),
					Type:     analysis.Double,
					Position: 4,
					Start:    9,
					End:      15,
				},
				&analysis.Token{
					Term:     []byte("五六"),
					Type:     analysis.Double,
					Position: 5,
					Start:    12,
					End:      18,
				},
				&analysis.Token{
					Term:     []byte("六七"),
					Type:     analysis.Double,
					Position: 6,
					Start:    15,
					End:      21,
				},
				&analysis.Token{
					Term:     []byte("七八"),
					Type:     analysis.Double,
					Position: 7,
					Start:    18,
					End:      24,
				},
				&analysis.Token{
					Term:     []byte("八九"),
					Type:     analysis.Double,
					Position: 8,
					Start:    21,
					End:      27,
				},
				&analysis.Token{
					Term:     []byte("九十"),
					Type:     analysis.Double,
					Position: 9,
					Start:    24,
					End:      30,
				},
			},
		},
		{
			input: []byte("一 二三四 五六七八九 十"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("一"),
					Type:     analysis.Single,
					Position: 1,
					Start:    0,
					End:      3,
				},
				&analysis.Token{
					Term:     []byte("二三"),
					Type:     analysis.Double,
					Position: 2,
					Start:    4,
					End:      10,
				},
				&analysis.Token{
					Term:     []byte("三四"),
					Type:     analysis.Double,
					Position: 3,
					Start:    7,
					End:      13,
				},
				&analysis.Token{
					Term:     []byte("五六"),
					Type:     analysis.Double,
					Position: 5,
					Start:    14,
					End:      20,
				},
				&analysis.Token{
					Term:     []byte("六七"),
					Type:     analysis.Double,
					Position: 6,
					Start:    17,
					End:      23,
				},
				&analysis.Token{
					Term:     []byte("七八"),
					Type:     analysis.Double,
					Position: 7,
					Start:    20,
					End:      26,
				},
				&analysis.Token{
					Term:     []byte("八九"),
					Type:     analysis.Double,
					Position: 8,
					Start:    23,
					End:      29,
				},
				&analysis.Token{
					Term:     []byte("十"),
					Type:     analysis.Single,
					Position: 10,
					Start:    30,
					End:      33,
				},
			},
		},
		{
			input: []byte("abc defgh ijklmn opqrstu vwxy z"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("abc"),
					Type:     analysis.AlphaNumeric,
					Position: 1,
					Start:    0,
					End:      3,
				},
				&analysis.Token{
					Term:     []byte("defgh"),
					Type:     analysis.AlphaNumeric,
					Position: 2,
					Start:    4,
					End:      9,
				},
				&analysis.Token{
					Term:     []byte("ijklmn"),
					Type:     analysis.AlphaNumeric,
					Position: 3,
					Start:    10,
					End:      16,
				},
				&analysis.Token{
					Term:     []byte("opqrstu"),
					Type:     analysis.AlphaNumeric,
					Position: 4,
					Start:    17,
					End:      24,
				},
				&analysis.Token{
					Term:     []byte("vwxy"),
					Type:     analysis.AlphaNumeric,
					Position: 5,
					Start:    25,
					End:      29,
				},
				&analysis.Token{
					Term:     []byte("z"),
					Type:     analysis.AlphaNumeric,
					Position: 6,
					Start:    30,
					End:      31,
				},
			},
		},
		{
			input: []byte("あい"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("あい"),
					Type:     analysis.Double,
					Position: 1,
					Start:    0,
					End:      6,
				},
			},
		},
		{
			input: []byte("あい   "),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("あい"),
					Type:     analysis.Double,
					Position: 1,
					Start:    0,
					End:      6,
				},
			},
		},
		{
			input: []byte("test"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("test"),
					Type:     analysis.AlphaNumeric,
					Position: 1,
					Start:    0,
					End:      4,
				},
			},
		},
		{
			input: []byte("test   "),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("test"),
					Type:     analysis.AlphaNumeric,
					Position: 1,
					Start:    0,
					End:      4,
				},
			},
		},
		{
			input: []byte("あいtest"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("あい"),
					Type:     analysis.Double,
					Position: 1,
					Start:    0,
					End:      6,
				},
				&analysis.Token{
					Term:     []byte("test"),
					Type:     analysis.AlphaNumeric,
					Position: 3,
					Start:    6,
					End:      10,
				},
			},
		},
		{
			input: []byte("testあい    "),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("test"),
					Type:     analysis.AlphaNumeric,
					Position: 1,
					Start:    0,
					End:      4,
				},
				&analysis.Token{
					Term:     []byte("あい"),
					Type:     analysis.Double,
					Position: 2,
					Start:    4,
					End:      10,
				},
			},
		},
		{
			input: []byte("あいうえおabcかきくけこ"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("あい"),
					Type:     analysis.Double,
					Position: 1,
					Start:    0,
					End:      6,
				},
				&analysis.Token{
					Term:     []byte("いう"),
					Type:     analysis.Double,
					Position: 2,
					Start:    3,
					End:      9,
				},
				&analysis.Token{
					Term:     []byte("うえ"),
					Type:     analysis.Double,
					Position: 3,
					Start:    6,
					End:      12,
				},
				&analysis.Token{
					Term:     []byte("えお"),
					Type:     analysis.Double,
					Position: 4,
					Start:    9,
					End:      15,
				},
				&analysis.Token{
					Term:     []byte("abc"),
					Type:     analysis.AlphaNumeric,
					Position: 6,
					Start:    15,
					End:      18,
				},
				&analysis.Token{
					Term:     []byte("かき"),
					Type:     analysis.Double,
					Position: 7,
					Start:    18,
					End:      24,
				},
				&analysis.Token{
					Term:     []byte("きく"),
					Type:     analysis.Double,
					Position: 8,
					Start:    21,
					End:      27,
				},
				&analysis.Token{
					Term:     []byte("くけ"),
					Type:     analysis.Double,
					Position: 9,
					Start:    24,
					End:      30,
				},
				&analysis.Token{
					Term:     []byte("けこ"),
					Type:     analysis.Double,
					Position: 10,
					Start:    27,
					End:      33,
				},
			},
		},
		{
			input: []byte("あいうえおabんcかきくけ こ"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("あい"),
					Type:     analysis.Double,
					Position: 1,
					Start:    0,
					End:      6,
				},
				&analysis.Token{
					Term:     []byte("いう"),
					Type:     analysis.Double,
					Position: 2,
					Start:    3,
					End:      9,
				},
				&analysis.Token{
					Term:     []byte("うえ"),
					Type:     analysis.Double,
					Position: 3,
					Start:    6,
					End:      12,
				},
				&analysis.Token{
					Term:     []byte("えお"),
					Type:     analysis.Double,
					Position: 4,
					Start:    9,
					End:      15,
				},
				&analysis.Token{
					Term:     []byte("ab"),
					Type:     analysis.AlphaNumeric,
					Position: 6,
					Start:    15,
					End:      17,
				},
				&analysis.Token{
					Term:     []byte("ん"),
					Type:     analysis.Single,
					Position: 7,
					Start:    17,
					End:      20,
				},
				&analysis.Token{
					Term:     []byte("c"),
					Type:     analysis.AlphaNumeric,
					Position: 8,
					Start:    20,
					End:      21,
				},
				&analysis.Token{
					Term:     []byte("かき"),
					Type:     analysis.Double,
					Position: 9,
					Start:    21,
					End:      27,
				},
				&analysis.Token{
					Term:     []byte("きく"),
					Type:     analysis.Double,
					Position: 10,
					Start:    24,
					End:      30,
				},
				&analysis.Token{
					Term:     []byte("くけ"),
					Type:     analysis.Double,
					Position: 11,
					Start:    27,
					End:      33,
				},
				&analysis.Token{
					Term:     []byte("こ"),
					Type:     analysis.Single,
					Position: 13,
					Start:    34,
					End:      37,
				},
			},
		},
		{
			input: []byte("一 روبرت موير"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("一"),
					Type:     analysis.Single,
					Position: 1,
					Start:    0,
					End:      3,
				},
				&analysis.Token{
					Term:     []byte("روبرت"),
					Type:     analysis.AlphaNumeric,
					Position: 2,
					Start:    4,
					End:      14,
				},
				&analysis.Token{
					Term:     []byte("موير"),
					Type:     analysis.AlphaNumeric,
					Position: 3,
					Start:    15,
					End:      23,
				},
			},
		},
		{
			input: []byte("一 رُوبرت موير"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("一"),
					Type:     analysis.Single,
					Position: 1,
					Start:    0,
					End:      3,
				},
				&analysis.Token{
					Term:     []byte("رُوبرت"),
					Type:     analysis.AlphaNumeric,
					Position: 2,
					Start:    4,
					End:      16,
				},
				&analysis.Token{
					Term:     []byte("موير"),
					Type:     analysis.AlphaNumeric,
					Position: 3,
					Start:    17,
					End:      25,
				},
			},
		},
		{
			input: []byte("𩬅艱鍟䇹愯瀛"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("𩬅艱"),
					Type:     analysis.Double,
					Position: 1,
					Start:    0,
					End:      7,
				},
				&analysis.Token{
					Term:     []byte("艱鍟"),
					Type:     analysis.Double,
					Position: 2,
					Start:    4,
					End:      10,
				},
				&analysis.Token{
					Term:     []byte("鍟䇹"),
					Type:     analysis.Double,
					Position: 3,
					Start:    7,
					End:      13,
				},
				&analysis.Token{
					Term:     []byte("䇹愯"),
					Type:     analysis.Double,
					Position: 4,
					Start:    10,
					End:      16,
				},
				&analysis.Token{
					Term:     []byte("愯瀛"),
					Type:     analysis.Double,
					Position: 5,
					Start:    13,
					End:      19,
				},
			},
		},
		{
			input: []byte("一"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("一"),
					Type:     analysis.Single,
					Position: 1,
					Start:    0,
					End:      3,
				},
			},
		},
		{
			input: []byte("一丁丂"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term:     []byte("一丁"),
					Type:     analysis.Double,
					Position: 1,
					Start:    0,
					End:      6,
				},
				&analysis.Token{
					Term:     []byte("丁丂"),
					Type:     analysis.Double,
					Position: 2,
					Start:    3,
					End:      9,
				},
			},
		},
	}

	cache := registry.NewCache()
	for _, test := range tests {
		analyzer, err := cache.AnalyzerNamed(AnalyzerName)
		if err != nil {
			t.Fatal(err)
		}
		actual := analyzer.Analyze(test.input)
		if !reflect.DeepEqual(actual, test.output) {
			t.Errorf("expected %v, got %v", test.output, actual)
		}
	}
}
Ejemplo n.º 30
0
func TestPersianAnalyzerOthers(t *testing.T) {
	tests := []struct {
		input  []byte
		output analysis.TokenStream
	}{
		// nouns
		{
			input: []byte("برگ ها"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("برگ"),
				},
			},
		},
		{
			input: []byte("برگ‌ها"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("برگ"),
				},
			},
		},
		// non persian
		{
			input: []byte("English test."),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("english"),
				},
				&analysis.Token{
					Term: []byte("test"),
				},
			},
		},
		// others
		{
			input: []byte("خورده مي شده بوده باشد"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("خورده"),
				},
			},
		},
		{
			input: []byte("برگ‌ها"),
			output: analysis.TokenStream{
				&analysis.Token{
					Term: []byte("برگ"),
				},
			},
		},
	}

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range tests {
		actual := analyzer.Analyze(test.input)
		if len(actual) != len(test.output) {
			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
		}
		for i, tok := range actual {
			if !reflect.DeepEqual(tok.Term, test.output[i].Term) {
				t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)
			}
		}
	}
}