func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) { tokens := make(analysis.TokenStream, 0) tokens = append(tokens, &analysis.Token{ Start: 0, End: len(n.value), Term: n.value, Position: 1, Type: analysis.DateTime, }) original, err := n.value.Int64() if err == nil { shift := DefaultDateTimePrecisionStep for shift < 64 { shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) if err != nil { break } token := analysis.Token{ Start: 0, End: len(shiftEncoded), Term: shiftEncoded, Position: 1, Type: analysis.DateTime, } tokens = append(tokens, &token) shift += DefaultDateTimePrecisionStep } } fieldLength := len(tokens) tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) return fieldLength, tokenFreqs }
func TestIndexAliasMultipleLayer(t *testing.T) { score1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0) score2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0) score3, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(3.0), 0) score4, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(4.0), 0) ei1 := &stubIndex{ name: "ei1", err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: []*search.DocumentMatch{ { Index: "1", ID: "a", Score: 1.0, Sort: []string{string(score1)}, }, }, MaxScore: 1.0, }} ei2 := &stubIndex{ name: "ei2", checkRequest: func(req *SearchRequest) error { time.Sleep(50 * time.Millisecond) return nil }, err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: []*search.DocumentMatch{ { Index: "2", ID: "b", Score: 2.0, Sort: []string{string(score2)}, }, }, MaxScore: 2.0, }} ei3 := &stubIndex{ name: "ei3", checkRequest: func(req *SearchRequest) error { time.Sleep(50 * time.Millisecond) return nil }, err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: []*search.DocumentMatch{ { Index: "3", ID: "c", Score: 3.0, Sort: []string{string(score3)}, }, }, MaxScore: 3.0, }} ei4 := &stubIndex{ name: "ei4", err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: []*search.DocumentMatch{ { Index: "4", ID: "d", Score: 4.0, Sort: []string{string(score4)}, }, }, MaxScore: 4.0, }} alias1 := NewIndexAlias(ei1, ei2) alias2 := NewIndexAlias(ei3, ei4) aliasTop := NewIndexAlias(alias1, alias2) // ei2 and ei3 have 50ms delay // search across aliasTop should still get results from ei1 and ei4 // total should still be 4 ctx, _ := context.WithTimeout(context.Background(), 25*time.Millisecond) query := NewTermQuery("test") sr := NewSearchRequest(query) expected := &SearchResult{ Status: &SearchStatus{ Total: 4, Successful: 2, Failed: 2, Errors: map[string]error{ "ei2": context.DeadlineExceeded, "ei3": context.DeadlineExceeded, }, }, Request: sr, Total: 2, Hits: search.DocumentMatchCollection{ { Index: "4", ID: "d", Score: 4.0, Sort: []string{string(score4)}, }, { Index: "1", ID: "a", Score: 1.0, Sort: []string{string(score1)}, }, }, MaxScore: 4.0, } res, err := aliasTop.SearchInContext(ctx, sr) if err != nil { t.Fatalf("expected no err, got %v", err) } expected.Took = res.Took if !reflect.DeepEqual(res, expected) { t.Errorf("expected %#v, got %#v", expected, res) } }
// TestMultiSearchTimeoutPartial tests the case where some indexes exceed // the timeout, while others complete successfully func TestMultiSearchTimeoutPartial(t *testing.T) { score1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0) score2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0) score3, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(3.0), 0) ei1 := &stubIndex{ name: "ei1", err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: []*search.DocumentMatch{ { Index: "1", ID: "a", Score: 1.0, Sort: []string{string(score1)}, }, }, MaxScore: 1.0, }} ei2 := &stubIndex{ name: "ei2", err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: []*search.DocumentMatch{ { Index: "2", ID: "b", Score: 2.0, Sort: []string{string(score2)}, }, }, MaxScore: 2.0, }} ei3 := &stubIndex{ name: "ei3", checkRequest: func(req *SearchRequest) error { time.Sleep(50 * time.Millisecond) return nil }, err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: []*search.DocumentMatch{ { Index: "3", ID: "c", Score: 3.0, Sort: []string{string(score3)}, }, }, MaxScore: 3.0, }} // ei3 is set to take >50ms, so run search with timeout less than // this, this should return partial results ctx, _ := context.WithTimeout(context.Background(), 25*time.Millisecond) query := NewTermQuery("test") sr := NewSearchRequest(query) expected := &SearchResult{ Status: &SearchStatus{ Total: 3, Successful: 2, Failed: 1, Errors: map[string]error{ "ei3": context.DeadlineExceeded, }, }, Request: sr, Total: 2, Hits: search.DocumentMatchCollection{ { Index: "2", ID: "b", Score: 2.0, Sort: []string{string(score2)}, }, { Index: "1", ID: "a", Score: 1.0, Sort: []string{string(score1)}, }, }, MaxScore: 2.0, } res, err := MultiSearch(ctx, sr, ei1, ei2, ei3) if err != nil { t.Fatalf("expected no err, got %v", err) } expected.Took = res.Took if !reflect.DeepEqual(res, expected) { t.Errorf("expected %#v, got %#v", expected, res) } }
// TestMultiSearchTimeout tests simple timeout cases // 1. all searches finish successfully before timeout // 2. no searchers finish before the timeout // 3. no searches finish before cancellation func TestMultiSearchTimeout(t *testing.T) { score1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0) score2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0) ei1 := &stubIndex{ name: "ei1", checkRequest: func(req *SearchRequest) error { time.Sleep(50 * time.Millisecond) return nil }, err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: []*search.DocumentMatch{ { Index: "1", ID: "a", Score: 1.0, Sort: []string{string(score1)}, }, }, MaxScore: 1.0, }} ei2 := &stubIndex{ name: "ei2", checkRequest: func(req *SearchRequest) error { time.Sleep(50 * time.Millisecond) return nil }, err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: []*search.DocumentMatch{ { Index: "2", ID: "b", Score: 2.0, Sort: []string{string(score2)}, }, }, MaxScore: 2.0, }} // first run with absurdly long time out, should succeed ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) query := NewTermQuery("test") sr := NewSearchRequest(query) res, err := MultiSearch(ctx, sr, ei1, ei2) if err != nil { t.Errorf("expected no error, got %v", err) } if res.Status.Total != 2 { t.Errorf("expected 2 total, got %d", res.Status.Failed) } if res.Status.Successful != 2 { t.Errorf("expected 0 success, got %d", res.Status.Successful) } if res.Status.Failed != 0 { t.Errorf("expected 2 failed, got %d", res.Status.Failed) } if len(res.Status.Errors) != 0 { t.Errorf("expected 0 errors, got %v", res.Status.Errors) } // now run a search again with an absurdly low timeout (should timeout) ctx, _ = context.WithTimeout(context.Background(), 1*time.Microsecond) res, err = MultiSearch(ctx, sr, ei1, ei2) if err != nil { t.Errorf("expected no error, got %v", err) } if res.Status.Total != 2 { t.Errorf("expected 2 failed, got %d", res.Status.Failed) } if res.Status.Successful != 0 { t.Errorf("expected 0 success, got %d", res.Status.Successful) } if res.Status.Failed != 2 { t.Errorf("expected 2 failed, got %d", res.Status.Failed) } if len(res.Status.Errors) != 2 { t.Errorf("expected 2 errors, got %v", res.Status.Errors) } else { if res.Status.Errors["ei1"].Error() != context.DeadlineExceeded.Error() { t.Errorf("expected err for 'ei1' to be '%s' got '%s'", context.DeadlineExceeded.Error(), res.Status.Errors["ei1"]) } if res.Status.Errors["ei2"].Error() != context.DeadlineExceeded.Error() { t.Errorf("expected err for 'ei2' to be '%s' got '%s'", context.DeadlineExceeded.Error(), res.Status.Errors["ei2"]) } } // now run a search again with a normal timeout, but cancel it first ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) cancel() res, err = MultiSearch(ctx, sr, ei1, ei2) if err != nil { t.Errorf("expected no error, got %v", err) } if res.Status.Total != 2 { t.Errorf("expected 2 failed, got %d", res.Status.Failed) } if res.Status.Successful != 0 { t.Errorf("expected 0 success, got %d", res.Status.Successful) } if res.Status.Failed != 2 { t.Errorf("expected 2 failed, got %d", res.Status.Failed) } if len(res.Status.Errors) != 2 { t.Errorf("expected 2 errors, got %v", res.Status.Errors) } else { if res.Status.Errors["ei1"].Error() != context.Canceled.Error() { t.Errorf("expected err for 'ei1' to be '%s' got '%s'", context.Canceled.Error(), res.Status.Errors["ei1"]) } if res.Status.Errors["ei2"].Error() != context.Canceled.Error() { t.Errorf("expected err for 'ei2' to be '%s' got '%s'", context.Canceled.Error(), res.Status.Errors["ei2"]) } } }
// TestMultiSearchNoError func TestMultiSearchNoError(t *testing.T) { score1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0) score2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0) ei1 := &stubIndex{err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: search.DocumentMatchCollection{ { Index: "1", ID: "a", Score: 1.0, Sort: []string{string(score1)}, }, }, MaxScore: 1.0, }} ei2 := &stubIndex{err: nil, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: search.DocumentMatchCollection{ { Index: "2", ID: "b", Score: 2.0, Sort: []string{string(score2)}, }, }, MaxScore: 2.0, }} sr := NewSearchRequest(NewTermQuery("test")) expected := &SearchResult{ Status: &SearchStatus{ Total: 2, Successful: 2, Errors: make(map[string]error), }, Request: sr, Total: 2, Hits: search.DocumentMatchCollection{ { Index: "2", ID: "b", Score: 2.0, Sort: []string{string(score2)}, }, { Index: "1", ID: "a", Score: 1.0, Sort: []string{string(score1)}, }, }, MaxScore: 2.0, } results, err := MultiSearch(context.Background(), sr, ei1, ei2) if err != nil { t.Error(err) } // cheat and ensure that Took field matches since it invovles time expected.Took = results.Took if !reflect.DeepEqual(results, expected) { t.Errorf("expected %#v, got %#v", expected, results) } }
func TestIndexAliasMulti(t *testing.T) { score1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0) score2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0) ei1Count := uint64(7) ei1 := &stubIndex{ err: nil, docCountResult: &ei1Count, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: search.DocumentMatchCollection{ { ID: "a", Score: 1.0, Sort: []string{string(score1)}, }, }, MaxScore: 1.0, }} ei2Count := uint64(8) ei2 := &stubIndex{ err: nil, docCountResult: &ei2Count, searchResult: &SearchResult{ Status: &SearchStatus{ Total: 1, Successful: 1, Errors: make(map[string]error), }, Total: 1, Hits: search.DocumentMatchCollection{ { ID: "b", Score: 2.0, Sort: []string{string(score2)}, }, }, MaxScore: 2.0, }} alias := NewIndexAlias(ei1, ei2) err := alias.Index("a", "a") if err != ErrorAliasMulti { t.Errorf("expected %v, got %v", ErrorAliasMulti, err) } err = alias.Delete("a") if err != ErrorAliasMulti { t.Errorf("expected %v, got %v", ErrorAliasMulti, err) } batch := alias.NewBatch() err = alias.Batch(batch) if err != ErrorAliasMulti { t.Errorf("expected %v, got %v", ErrorAliasMulti, err) } _, err = alias.Document("a") if err != ErrorAliasMulti { t.Errorf("expected %v, got %v", ErrorAliasMulti, err) } _, err = alias.Fields() if err != ErrorAliasMulti { t.Errorf("expected %v, got %v", ErrorAliasMulti, err) } _, err = alias.GetInternal([]byte("a")) if err != ErrorAliasMulti { t.Errorf("expected %v, got %v", ErrorAliasMulti, err) } err = alias.SetInternal([]byte("a"), []byte("a")) if err != ErrorAliasMulti { t.Errorf("expected %v, got %v", ErrorAliasMulti, err) } err = alias.DeleteInternal([]byte("a")) if err != ErrorAliasMulti { t.Errorf("expected %v, got %v", ErrorAliasMulti, err) } mapping := alias.Mapping() if mapping != nil { t.Errorf("expected nil, got %v", mapping) } indexStat := alias.Stats() if indexStat != nil { t.Errorf("expected nil, got %v", indexStat) } // now a few things that should work sr := NewSearchRequest(NewTermQuery("test")) expected := &SearchResult{ Status: &SearchStatus{ Total: 2, Successful: 2, Errors: make(map[string]error), }, Request: sr, Total: 2, Hits: search.DocumentMatchCollection{ { ID: "b", Score: 2.0, Sort: []string{string(score2)}, }, { ID: "a", Score: 1.0, Sort: []string{string(score1)}, }, }, MaxScore: 2.0, } results, err := alias.Search(sr) if err != nil { t.Error(err) } // cheat and ensure that Took field matches since it invovles time expected.Took = results.Took if !reflect.DeepEqual(results, expected) { t.Errorf("expected %#v, got %#v", expected, results) } count, err := alias.DocCount() if err != nil { t.Errorf("error getting alias doc count: %v", err) } if count != (*ei1.docCountResult + *ei2.docCountResult) { t.Errorf("expected %d, got %d", (*ei1.docCountResult + *ei2.docCountResult), count) } }