func TestConstantScorerWithQueryNorm(t *testing.T) { scorer := NewConstantScorer(1, 1, true) scorer.SetQueryNorm(2.0) tests := []struct { termMatch *index.TermFieldDoc result *search.DocumentMatch }{ { termMatch: &index.TermFieldDoc{ ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, }, result: &search.DocumentMatch{ IndexInternalID: index.IndexInternalID("one"), Score: 2.0, Sort: []string{}, Expl: &search.Explanation{ Value: 2.0, Message: "weight(^1.000000), product of:", Children: []*search.Explanation{ { Value: 2.0, Message: "ConstantScore()^1.000000, product of:", Children: []*search.Explanation{ { Value: 1, Message: "boost", }, { Value: 2, Message: "queryNorm", }, }, }, { Value: 1.0, Message: "ConstantScore()", }, }, }, }, }, } for _, test := range tests { ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(1, 0), } actual := scorer.Score(ctx, test.termMatch.ID) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) } } }
// Collect goes to the index to find the matching documents func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error { startTime := time.Now() var err error var next *search.DocumentMatch // pre-allocate enough space in the DocumentMatchPool // unless the size + skip is too large, then cap it // everything should still work, just allocates DocumentMatches on demand backingSize := hc.size + hc.skip + 1 if hc.size+hc.skip > PreAllocSizeSkipCap { backingSize = PreAllocSizeSkipCap + 1 } searchContext := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), len(hc.sort)), } select { case <-ctx.Done(): return ctx.Err() default: next, err = searcher.Next(searchContext) } for err == nil && next != nil { if hc.total%CheckDoneEvery == 0 { select { case <-ctx.Done(): return ctx.Err() default: } } if hc.facetsBuilder != nil { err = hc.facetsBuilder.Update(next) if err != nil { break } } err = hc.collectSingle(searchContext, reader, next) if err != nil { break } next, err = searcher.Next(searchContext) } // compute search duration hc.took = time.Since(startTime) if err != nil { return err } // finalize actual results err = hc.finalizeResults(reader) if err != nil { return err } return nil }
func TestConstantScorer(t *testing.T) { scorer := NewConstantScorer(1, 1, true) tests := []struct { termMatch *index.TermFieldDoc result *search.DocumentMatch }{ // test some simple math { termMatch: &index.TermFieldDoc{ ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, Vectors: []*index.TermFieldVector{ { Field: "desc", Pos: 1, Start: 0, End: 4, }, }, }, result: &search.DocumentMatch{ IndexInternalID: index.IndexInternalID("one"), Score: 1.0, Expl: &search.Explanation{ Value: 1.0, Message: "ConstantScore()", }, Sort: []string{}, }, }, } for _, test := range tests { ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(1, 0), } actual := scorer.Score(ctx, test.termMatch.ID) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) } } }
func TestDisjunctionAdvance(t *testing.T) { twoDocIndexReader, err := twoDocIndex.Reader() if err != nil { t.Error(err) } defer func() { err := twoDocIndexReader.Close() if err != nil { t.Fatal(err) } }() martyTermSearcher, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) if err != nil { t.Fatal(err) } dustinTermSearcher, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) if err != nil { t.Fatal(err) } martyOrDustinSearcher, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, true) if err != nil { t.Fatal(err) } ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(martyOrDustinSearcher.DocumentMatchPoolSize(), 0), } match, err := martyOrDustinSearcher.Advance(ctx, index.IndexInternalID("3")) if err != nil { t.Errorf("unexpected error: %v", err) } if match == nil { t.Errorf("expected 3, got nil") } }
func TestPhraseSearch(t *testing.T) { twoDocIndexReader, err := twoDocIndex.Reader() if err != nil { t.Error(err) } defer func() { err := twoDocIndexReader.Close() if err != nil { t.Fatal(err) } }() angstTermSearcher, err := NewTermSearcher(twoDocIndexReader, "angst", "desc", 1.0, true) if err != nil { t.Fatal(err) } beerTermSearcher, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } mustSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{angstTermSearcher, beerTermSearcher}, true) if err != nil { t.Fatal(err) } phraseSearcher, err := NewPhraseSearcher(twoDocIndexReader, mustSearcher, []string{"angst", "beer"}) if err != nil { t.Fatal(err) } tests := []struct { searcher search.Searcher results []*search.DocumentMatch }{ { searcher: phraseSearcher, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("2"), Score: 1.0807601687084403, }, }, }, } for testIndex, test := range tests { defer func() { err := test.searcher.Close() if err != nil { t.Fatal(err) } }() ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), } next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if next.Score != test.results[i].Score { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } ctx.DocumentMatchPool.Put(next) next, err = test.searcher.Next(ctx) i++ } if err != nil { t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) } if len(test.results) != i { t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) } } }
func TestTermSearcher(t *testing.T) { var queryTerm = "beer" var queryField = "desc" var queryBoost = 3.0 var queryExplain = true analysisQueue := index.NewAnalysisQueue(1) i, err := upsidedown.NewUpsideDownCouch( gtreap.Name, map[string]interface{}{ "path": "", }, analysisQueue) if err != nil { t.Fatal(err) } err = i.Open() if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "a", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "b", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "c", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "d", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "e", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "f", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "g", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "h", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "i", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "j", Fields: []document.Field{ document.NewTextField("title", []uint64{}, []byte("cat")), }, }) if err != nil { t.Fatal(err) } indexReader, err := i.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() searcher, err := NewTermSearcher(indexReader, queryTerm, queryField, queryBoost, queryExplain) if err != nil { t.Fatal(err) } defer func() { err := searcher.Close() if err != nil { t.Fatal(err) } }() searcher.SetQueryNorm(2.0) docCount, err := indexReader.DocCount() if err != nil { t.Fatal(err) } idf := 1.0 + math.Log(float64(docCount)/float64(searcher.Count()+1.0)) expectedQueryWeight := 3 * idf * 3 * idf if expectedQueryWeight != searcher.Weight() { t.Errorf("expected weight %v got %v", expectedQueryWeight, searcher.Weight()) } if searcher.Count() != 9 { t.Errorf("expected count of 9, got %d", searcher.Count()) } ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(1, 0), } docMatch, err := searcher.Next(ctx) if err != nil { t.Errorf("expected result, got %v", err) } if !docMatch.IndexInternalID.Equals(index.IndexInternalID("a")) { t.Errorf("expected result ID to be 'a', got '%s", docMatch.IndexInternalID) } ctx.DocumentMatchPool.Put(docMatch) docMatch, err = searcher.Advance(ctx, index.IndexInternalID("c")) if err != nil { t.Errorf("expected result, got %v", err) } if !docMatch.IndexInternalID.Equals(index.IndexInternalID("c")) { t.Errorf("expected result ID to be 'c' got '%s'", docMatch.IndexInternalID) } // try advancing past end ctx.DocumentMatchPool.Put(docMatch) docMatch, err = searcher.Advance(ctx, index.IndexInternalID("z")) if err != nil { t.Fatal(err) } if docMatch != nil { t.Errorf("expected nil, got %v", docMatch) } // try pushing next past end ctx.DocumentMatchPool.Put(docMatch) docMatch, err = searcher.Next(ctx) if err != nil { t.Fatal(err) } if docMatch != nil { t.Errorf("expected nil, got %v", docMatch) } }
func TestTermScorer(t *testing.T) { var docTotal uint64 = 100 var docTerm uint64 = 9 var queryTerm = "beer" var queryField = "desc" var queryBoost = 1.0 scorer := NewTermQueryScorer(queryTerm, queryField, queryBoost, docTotal, docTerm, true) idf := 1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0)) tests := []struct { termMatch *index.TermFieldDoc result *search.DocumentMatch }{ // test some simple math { termMatch: &index.TermFieldDoc{ ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, Vectors: []*index.TermFieldVector{ { Field: "desc", Pos: 1, Start: 0, End: 4, }, }, }, result: &search.DocumentMatch{ IndexInternalID: index.IndexInternalID("one"), Score: math.Sqrt(1.0) * idf, Sort: []string{}, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf, Message: "fieldWeight(desc:beer in one), product of:", Children: []*search.Explanation{ { Value: 1, Message: "tf(termFreq(desc:beer)=1", }, { Value: 1, Message: "fieldNorm(field=desc, doc=one)", }, { Value: idf, Message: "idf(docFreq=9, maxDocs=100)", }, }, }, Locations: search.FieldTermLocationMap{ "desc": search.TermLocationMap{ "beer": []*search.Location{ { Pos: 1, Start: 0, End: 4, }, }, }, }, }, }, // test the same thing again (score should be cached this time) { termMatch: &index.TermFieldDoc{ ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, }, result: &search.DocumentMatch{ IndexInternalID: index.IndexInternalID("one"), Score: math.Sqrt(1.0) * idf, Sort: []string{}, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf, Message: "fieldWeight(desc:beer in one), product of:", Children: []*search.Explanation{ { Value: 1, Message: "tf(termFreq(desc:beer)=1", }, { Value: 1, Message: "fieldNorm(field=desc, doc=one)", }, { Value: idf, Message: "idf(docFreq=9, maxDocs=100)", }, }, }, }, }, // test a case where the sqrt isn't precalculated { termMatch: &index.TermFieldDoc{ ID: index.IndexInternalID("one"), Freq: 65, Norm: 1.0, }, result: &search.DocumentMatch{ IndexInternalID: index.IndexInternalID("one"), Score: math.Sqrt(65) * idf, Sort: []string{}, Expl: &search.Explanation{ Value: math.Sqrt(65) * idf, Message: "fieldWeight(desc:beer in one), product of:", Children: []*search.Explanation{ { Value: math.Sqrt(65), Message: "tf(termFreq(desc:beer)=65", }, { Value: 1, Message: "fieldNorm(field=desc, doc=one)", }, { Value: idf, Message: "idf(docFreq=9, maxDocs=100)", }, }, }, }, }, } for _, test := range tests { ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(1, 0), } actual := scorer.Score(ctx, test.termMatch) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) } } }
func TestTermScorerWithQueryNorm(t *testing.T) { var docTotal uint64 = 100 var docTerm uint64 = 9 var queryTerm = "beer" var queryField = "desc" var queryBoost = 3.0 scorer := NewTermQueryScorer(queryTerm, queryField, queryBoost, docTotal, docTerm, true) idf := 1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0)) scorer.SetQueryNorm(2.0) expectedQueryWeight := 3 * idf * 3 * idf actualQueryWeight := scorer.Weight() if expectedQueryWeight != actualQueryWeight { t.Errorf("expected query weight %f, got %f", expectedQueryWeight, actualQueryWeight) } tests := []struct { termMatch *index.TermFieldDoc result *search.DocumentMatch }{ { termMatch: &index.TermFieldDoc{ ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, }, result: &search.DocumentMatch{ IndexInternalID: index.IndexInternalID("one"), Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, Sort: []string{}, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, Message: "weight(desc:beer^3.000000 in one), product of:", Children: []*search.Explanation{ { Value: 2.0 * idf * 3.0, Message: "queryWeight(desc:beer^3.000000), product of:", Children: []*search.Explanation{ { Value: 3, Message: "boost", }, { Value: idf, Message: "idf(docFreq=9, maxDocs=100)", }, { Value: 2, Message: "queryNorm", }, }, }, { Value: math.Sqrt(1.0) * idf, Message: "fieldWeight(desc:beer in one), product of:", Children: []*search.Explanation{ { Value: 1, Message: "tf(termFreq(desc:beer)=1", }, { Value: 1, Message: "fieldNorm(field=desc, doc=one)", }, { Value: idf, Message: "idf(docFreq=9, maxDocs=100)", }, }, }, }, }, }, }, } for _, test := range tests { ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(1, 0), } actual := scorer.Score(ctx, test.termMatch) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) } } }
func TestMatchNoneSearch(t *testing.T) { twoDocIndexReader, err := twoDocIndex.Reader() if err != nil { t.Error(err) } defer func() { err := twoDocIndexReader.Close() if err != nil { t.Fatal(err) } }() noneSearcher, err := NewMatchNoneSearcher(twoDocIndexReader) if err != nil { t.Fatal(err) } tests := []struct { searcher search.Searcher results []*search.DocumentMatch }{ { searcher: noneSearcher, results: []*search.DocumentMatch{}, }, } for testIndex, test := range tests { defer func() { err := test.searcher.Close() if err != nil { t.Fatal(err) } }() ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), } next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } ctx.DocumentMatchPool.Put(next) next, err = test.searcher.Next(ctx) i++ } if err != nil { t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) } if len(test.results) != i { t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) } } }
func TestRegexpSearch(t *testing.T) { twoDocIndexReader, err := twoDocIndex.Reader() if err != nil { t.Error(err) } defer func() { err := twoDocIndexReader.Close() if err != nil { t.Fatal(err) } }() pattern, err := regexp.Compile("ma.*") if err != nil { t.Fatal(err) } regexpSearcher, err := NewRegexpSearcher(twoDocIndexReader, pattern, "name", 1.0, true) if err != nil { t.Fatal(err) } patternCo, err := regexp.Compile("co.*") if err != nil { t.Fatal(err) } regexpSearcherCo, err := NewRegexpSearcher(twoDocIndexReader, patternCo, "desc", 1.0, true) if err != nil { t.Fatal(err) } tests := []struct { searcher search.Searcher results []*search.DocumentMatch }{ { searcher: regexpSearcher, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 1.916290731874155, }, }, }, { searcher: regexpSearcherCo, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("2"), Score: 0.33875554280828685, }, { IndexInternalID: index.IndexInternalID("3"), Score: 0.33875554280828685, }, }, }, } for testIndex, test := range tests { defer func() { err := test.searcher.Close() if err != nil { t.Fatal(err) } }() ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), } next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if next.Score != test.results[i].Score { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } ctx.DocumentMatchPool.Put(next) next, err = test.searcher.Next(ctx) i++ } if err != nil { t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) } if len(test.results) != i { t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) } } }
func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { analysisQueue := index.NewAnalysisQueue(1) i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue) if err != nil { t.Fatal(err) } err = i.Open() if err != nil { t.Fatal(err) } for _, id := range indexed { err = i.Update(&document.Document{ ID: id, Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } } indexReader, err := i.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() searcher, err := NewDocIDSearcher(indexReader, searched, 1.0, false) if err != nil { t.Fatal(err) } defer func() { err := searcher.Close() if err != nil { t.Fatal(err) } }() ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize(), 0), } // Check the sequence for i, id := range wanted { m, err := searcher.Next(ctx) if err != nil { t.Fatal(err) } if !index.IndexInternalID(id).Equals(m.IndexInternalID) { t.Fatalf("expected %v at position %v, got %v", id, i, m.IndexInternalID) } ctx.DocumentMatchPool.Put(m) } m, err := searcher.Next(ctx) if err != nil { t.Fatal(err) } if m != nil { t.Fatalf("expected nil past the end of the sequence, got %v", m.IndexInternalID) } ctx.DocumentMatchPool.Put(m) // Check seeking for _, id := range wanted { if len(id) != 2 { t.Fatalf("expected identifier must be 2 characters long, got %v", id) } before := id[:1] for _, target := range []string{before, id} { m, err := searcher.Advance(ctx, index.IndexInternalID(target)) if err != nil { t.Fatal(err) } if m == nil || !m.IndexInternalID.Equals(index.IndexInternalID(id)) { t.Fatalf("advancing to %v returned %v instead of %v", before, m, id) } ctx.DocumentMatchPool.Put(m) } } // Seek after the end of the sequence after := "zzz" m, err = searcher.Advance(ctx, index.IndexInternalID(after)) if err != nil { t.Fatal(err) } if m != nil { t.Fatalf("advancing past the end of the sequence should return nil, got %v", m) } ctx.DocumentMatchPool.Put(m) }
func TestConjunctionSearch(t *testing.T) { twoDocIndexReader, err := twoDocIndex.Reader() if err != nil { t.Error(err) } defer func() { err := twoDocIndexReader.Close() if err != nil { t.Fatal(err) } }() // test 0 beerTermSearcher, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } martyTermSearcher, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 5.0, true) if err != nil { t.Fatal(err) } beerAndMartySearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher, martyTermSearcher}, true) if err != nil { t.Fatal(err) } // test 1 angstTermSearcher, err := NewTermSearcher(twoDocIndexReader, "angst", "desc", 1.0, true) if err != nil { t.Fatal(err) } beerTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } angstAndBeerSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{angstTermSearcher, beerTermSearcher2}, true) if err != nil { t.Fatal(err) } // test 2 beerTermSearcher3, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } jackTermSearcher, err := NewTermSearcher(twoDocIndexReader, "jack", "name", 5.0, true) if err != nil { t.Fatal(err) } beerAndJackSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher3, jackTermSearcher}, true) if err != nil { t.Fatal(err) } // test 3 beerTermSearcher4, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } misterTermSearcher, err := NewTermSearcher(twoDocIndexReader, "mister", "title", 5.0, true) if err != nil { t.Fatal(err) } beerAndMisterSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher4, misterTermSearcher}, true) if err != nil { t.Fatal(err) } // test 4 couchbaseTermSearcher, err := NewTermSearcher(twoDocIndexReader, "couchbase", "street", 1.0, true) if err != nil { t.Fatal(err) } misterTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "mister", "title", 5.0, true) if err != nil { t.Fatal(err) } couchbaseAndMisterSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{couchbaseTermSearcher, misterTermSearcher2}, true) if err != nil { t.Fatal(err) } // test 5 beerTermSearcher5, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 5.0, true) if err != nil { t.Fatal(err) } couchbaseTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "couchbase", "street", 1.0, true) if err != nil { t.Fatal(err) } misterTermSearcher3, err := NewTermSearcher(twoDocIndexReader, "mister", "title", 5.0, true) if err != nil { t.Fatal(err) } couchbaseAndMisterSearcher2, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{couchbaseTermSearcher2, misterTermSearcher3}, true) if err != nil { t.Fatal(err) } beerAndCouchbaseAndMisterSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher5, couchbaseAndMisterSearcher2}, true) if err != nil { t.Fatal(err) } tests := []struct { searcher search.Searcher results []*search.DocumentMatch }{ { searcher: beerAndMartySearcher, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 2.0097428702814377, }, }, }, { searcher: angstAndBeerSearcher, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("2"), Score: 1.0807601687084403, }, }, }, { searcher: beerAndJackSearcher, results: []*search.DocumentMatch{}, }, { searcher: beerAndMisterSearcher, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("2"), Score: 1.2877980334016337, }, { IndexInternalID: index.IndexInternalID("3"), Score: 1.2877980334016337, }, }, }, { searcher: couchbaseAndMisterSearcher, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("2"), Score: 1.4436599157093672, }, }, }, { searcher: beerAndCouchbaseAndMisterSearcher, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("2"), Score: 1.441614953806971, }, }, }, } for testIndex, test := range tests { defer func() { err := test.searcher.Close() if err != nil { t.Fatal(err) } }() ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(10, 0), } next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } next, err = test.searcher.Next(ctx) i++ } if err != nil { t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) } if len(test.results) != i { t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) } } }
func TestDisjunctionSearch(t *testing.T) { twoDocIndexReader, err := twoDocIndex.Reader() if err != nil { t.Error(err) } defer func() { err := twoDocIndexReader.Close() if err != nil { t.Fatal(err) } }() martyTermSearcher, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) if err != nil { t.Fatal(err) } dustinTermSearcher, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) if err != nil { t.Fatal(err) } martyOrDustinSearcher, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, true) if err != nil { t.Fatal(err) } martyTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) if err != nil { t.Fatal(err) } dustinTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) if err != nil { t.Fatal(err) } martyOrDustinSearcher2, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher2, dustinTermSearcher2}, 0, true) if err != nil { t.Fatal(err) } raviTermSearcher, err := NewTermSearcher(twoDocIndexReader, "ravi", "name", 1.0, true) if err != nil { t.Fatal(err) } nestedRaviOrMartyOrDustinSearcher, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{raviTermSearcher, martyOrDustinSearcher2}, 0, true) if err != nil { t.Fatal(err) } tests := []struct { searcher search.Searcher results []*search.DocumentMatch }{ { searcher: martyOrDustinSearcher, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 0.6775110856165737, }, { IndexInternalID: index.IndexInternalID("3"), Score: 0.6775110856165737, }, }, }, // test a nested disjunction { searcher: nestedRaviOrMartyOrDustinSearcher, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 0.2765927424732821, }, { IndexInternalID: index.IndexInternalID("3"), Score: 0.2765927424732821, }, { IndexInternalID: index.IndexInternalID("4"), Score: 0.5531854849465642, }, }, }, } for testIndex, test := range tests { defer func() { err := test.searcher.Close() if err != nil { t.Fatal(err) } }() ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), } next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } ctx.DocumentMatchPool.Put(next) next, err = test.searcher.Next(ctx) i++ } if err != nil { t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) } if len(test.results) != i { t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) } } }
func TestFuzzySearch(t *testing.T) { twoDocIndexReader, err := twoDocIndex.Reader() if err != nil { t.Error(err) } defer func() { err := twoDocIndexReader.Close() if err != nil { t.Fatal(err) } }() fuzzySearcherbeet, err := NewFuzzySearcher(twoDocIndexReader, "beet", 0, 1, "desc", 1.0, true) if err != nil { t.Fatal(err) } fuzzySearcherdouches, err := NewFuzzySearcher(twoDocIndexReader, "douches", 0, 2, "desc", 1.0, true) if err != nil { t.Fatal(err) } fuzzySearcheraplee, err := NewFuzzySearcher(twoDocIndexReader, "aplee", 0, 2, "desc", 1.0, true) if err != nil { t.Fatal(err) } fuzzySearcherprefix, err := NewFuzzySearcher(twoDocIndexReader, "water", 3, 2, "desc", 1.0, true) if err != nil { t.Fatal(err) } tests := []struct { searcher search.Searcher results []*search.DocumentMatch }{ { searcher: fuzzySearcherbeet, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 1.0, }, { IndexInternalID: index.IndexInternalID("2"), Score: 0.5, }, { IndexInternalID: index.IndexInternalID("3"), Score: 0.5, }, { IndexInternalID: index.IndexInternalID("4"), Score: 0.9999999838027345, }, }, }, { searcher: fuzzySearcherdouches, results: []*search.DocumentMatch{}, }, { searcher: fuzzySearcheraplee, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("3"), Score: 0.9581453659370776, }, }, }, { searcher: fuzzySearcherprefix, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("5"), Score: 1.916290731874155, }, }, }, } for testIndex, test := range tests { defer func() { err := test.searcher.Close() if err != nil { t.Fatal(err) } }() ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), } next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if next.Score != test.results[i].Score { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } ctx.DocumentMatchPool.Put(next) next, err = test.searcher.Next(ctx) i++ } if err != nil { t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) } if len(test.results) != i { t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) } } }
func TestBooleanSearch(t *testing.T) { if twoDocIndex == nil { t.Fatal("its null") } twoDocIndexReader, err := twoDocIndex.Reader() if err != nil { t.Error(err) } defer func() { err := twoDocIndexReader.Close() if err != nil { t.Fatal(err) } }() // test 0 beerTermSearcher, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } mustSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher}, true) if err != nil { t.Fatal(err) } martyTermSearcher, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) if err != nil { t.Fatal(err) } dustinTermSearcher, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) if err != nil { t.Fatal(err) } shouldSearcher, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, true) if err != nil { t.Fatal(err) } steveTermSearcher, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) if err != nil { t.Fatal(err) } mustNotSearcher, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher}, 0, true) if err != nil { t.Fatal(err) } booleanSearcher, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher, shouldSearcher, mustNotSearcher, true) if err != nil { t.Fatal(err) } // test 1 martyTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) if err != nil { t.Fatal(err) } dustinTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) if err != nil { t.Fatal(err) } shouldSearcher2, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher2, dustinTermSearcher2}, 0, true) if err != nil { t.Fatal(err) } steveTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) if err != nil { t.Fatal(err) } mustNotSearcher2, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher2}, 0, true) if err != nil { t.Fatal(err) } booleanSearcher2, err := NewBooleanSearcher(twoDocIndexReader, nil, shouldSearcher2, mustNotSearcher2, true) if err != nil { t.Fatal(err) } // test 2 steveTermSearcher3, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) if err != nil { t.Fatal(err) } mustNotSearcher3, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher3}, 0, true) if err != nil { t.Fatal(err) } booleanSearcher3, err := NewBooleanSearcher(twoDocIndexReader, nil, nil, mustNotSearcher3, true) if err != nil { t.Fatal(err) } // test 3 beerTermSearcher4, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } mustSearcher4, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher4}, true) if err != nil { t.Fatal(err) } steveTermSearcher4, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) if err != nil { t.Fatal(err) } mustNotSearcher4, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher4}, 0, true) if err != nil { t.Fatal(err) } booleanSearcher4, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher4, nil, mustNotSearcher4, true) if err != nil { t.Fatal(err) } // test 4 beerTermSearcher5, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } mustSearcher5, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher5}, true) if err != nil { t.Fatal(err) } steveTermSearcher5, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) if err != nil { t.Fatal(err) } martyTermSearcher5, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) if err != nil { t.Fatal(err) } mustNotSearcher5, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher5, martyTermSearcher5}, 0, true) if err != nil { t.Fatal(err) } booleanSearcher5, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher5, nil, mustNotSearcher5, true) if err != nil { t.Fatal(err) } // test 5 beerTermSearcher6, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } mustSearcher6, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher6}, true) if err != nil { t.Fatal(err) } martyTermSearcher6, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) if err != nil { t.Fatal(err) } dustinTermSearcher6, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) if err != nil { t.Fatal(err) } shouldSearcher6, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher6, dustinTermSearcher6}, 2, true) if err != nil { t.Fatal(err) } booleanSearcher6, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher6, shouldSearcher6, nil, true) if err != nil { t.Fatal(err) } // test 6 beerTermSearcher7, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } mustSearcher7, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher7}, true) if err != nil { t.Fatal(err) } booleanSearcher7, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher7, nil, nil, true) if err != nil { t.Fatal(err) } martyTermSearcher7, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 5.0, true) if err != nil { t.Fatal(err) } conjunctionSearcher7, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher7, booleanSearcher7}, true) // test 7 beerTermSearcher8, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) if err != nil { t.Fatal(err) } mustSearcher8, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher8}, true) if err != nil { t.Fatal(err) } martyTermSearcher8, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) if err != nil { t.Fatal(err) } dustinTermSearcher8, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) if err != nil { t.Fatal(err) } shouldSearcher8, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher8, dustinTermSearcher8}, 0, true) if err != nil { t.Fatal(err) } steveTermSearcher8, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) if err != nil { t.Fatal(err) } mustNotSearcher8, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher8}, 0, true) if err != nil { t.Fatal(err) } booleanSearcher8, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher8, shouldSearcher8, mustNotSearcher8, true) if err != nil { t.Fatal(err) } dustinTermSearcher8a, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 5.0, true) if err != nil { t.Fatal(err) } conjunctionSearcher8, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{booleanSearcher8, dustinTermSearcher8a}, true) if err != nil { t.Fatal(err) } tests := []struct { searcher search.Searcher results []*search.DocumentMatch }{ { searcher: booleanSearcher, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 0.9818005051949021, }, { IndexInternalID: index.IndexInternalID("3"), Score: 0.808709699395535, }, { IndexInternalID: index.IndexInternalID("4"), Score: 0.34618161159873423, }, }, }, { searcher: booleanSearcher2, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 0.6775110856165737, }, { IndexInternalID: index.IndexInternalID("3"), Score: 0.6775110856165737, }, }, }, // no MUST or SHOULD clauses yields no results { searcher: booleanSearcher3, results: []*search.DocumentMatch{}, }, { searcher: booleanSearcher4, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 1.0, }, { IndexInternalID: index.IndexInternalID("3"), Score: 0.5, }, { IndexInternalID: index.IndexInternalID("4"), Score: 1.0, }, }, }, { searcher: booleanSearcher5, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("3"), Score: 0.5, }, { IndexInternalID: index.IndexInternalID("4"), Score: 1.0, }, }, }, { searcher: booleanSearcher6, results: []*search.DocumentMatch{}, }, // test a conjunction query with a nested boolean { searcher: conjunctionSearcher7, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 2.0097428702814377, }, }, }, { searcher: conjunctionSearcher8, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("3"), Score: 2.0681575785068107, }, }, }, } for testIndex, test := range tests { defer func() { err := test.searcher.Close() if err != nil { t.Fatal(err) } }() ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), } next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } ctx.DocumentMatchPool.Put(next) next, err = test.searcher.Next(ctx) i++ } if err != nil { t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) } if len(test.results) != i { t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) } } }
func TestMatchAllSearch(t *testing.T) { twoDocIndexReader, err := twoDocIndex.Reader() if err != nil { t.Error(err) } defer func() { err := twoDocIndexReader.Close() if err != nil { t.Fatal(err) } }() allSearcher, err := NewMatchAllSearcher(twoDocIndexReader, 1.0, true) if err != nil { t.Fatal(err) } allSearcher2, err := NewMatchAllSearcher(twoDocIndexReader, 1.2, true) if err != nil { t.Fatal(err) } tests := []struct { searcher search.Searcher queryNorm float64 results []*search.DocumentMatch }{ { searcher: allSearcher, queryNorm: 1.0, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 1.0, }, { IndexInternalID: index.IndexInternalID("2"), Score: 1.0, }, { IndexInternalID: index.IndexInternalID("3"), Score: 1.0, }, { IndexInternalID: index.IndexInternalID("4"), Score: 1.0, }, { IndexInternalID: index.IndexInternalID("5"), Score: 1.0, }, }, }, { searcher: allSearcher2, queryNorm: 0.8333333, results: []*search.DocumentMatch{ { IndexInternalID: index.IndexInternalID("1"), Score: 1.0, }, { IndexInternalID: index.IndexInternalID("2"), Score: 1.0, }, { IndexInternalID: index.IndexInternalID("3"), Score: 1.0, }, { IndexInternalID: index.IndexInternalID("4"), Score: 1.0, }, { IndexInternalID: index.IndexInternalID("5"), Score: 1.0, }, }, }, } for testIndex, test := range tests { if test.queryNorm != 1.0 { test.searcher.SetQueryNorm(test.queryNorm) } defer func() { err := test.searcher.Close() if err != nil { t.Fatal(err) } }() ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), } next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } ctx.DocumentMatchPool.Put(next) next, err = test.searcher.Next(ctx) i++ } if err != nil { t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) } if len(test.results) != i { t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) } } }