func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) {
tokens := make(analysis.TokenStream, 0)
tokens = append(tokens, &analysis.Token{
Start: 0,
End: len(n.value),
Term: n.value,
Position: 1,
Type: analysis.DateTime,
})
original, err := n.value.Int64()
if err == nil {
shift := DefaultDateTimePrecisionStep
for shift < 64 {
shiftEncoded, err := numeric_util.NewPrefixCodedInt64(original, shift)
if err != nil {
break
}
token := analysis.Token{
Start: 0,
End: len(shiftEncoded),
Term: shiftEncoded,
Position: 1,
Type: analysis.DateTime,
}
tokens = append(tokens, &token)
shift += DefaultDateTimePrecisionStep
}
}
fieldLength := len(tokens)
tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions)
return fieldLength, tokenFreqs
}
func TestIndexAliasMultipleLayer(t *testing.T) {
score1, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(1.0), 0)
score2, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(2.0), 0)
score3, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(3.0), 0)
score4, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(4.0), 0)
ei1 := &stubIndex{
name: "ei1",
err: nil,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: []*search.DocumentMatch{
{
Index: "1",
ID: "a",
Score: 1.0,
Sort: []string{string(score1)},
},
},
MaxScore: 1.0,
}}
ei2 := &stubIndex{
name: "ei2",
checkRequest: func(req *SearchRequest) error {
time.Sleep(50 * time.Millisecond)
return nil
},
err: nil,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: []*search.DocumentMatch{
{
Index: "2",
ID: "b",
Score: 2.0,
Sort: []string{string(score2)},
},
},
MaxScore: 2.0,
}}
ei3 := &stubIndex{
name: "ei3",
checkRequest: func(req *SearchRequest) error {
time.Sleep(50 * time.Millisecond)
return nil
},
err: nil,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: []*search.DocumentMatch{
{
Index: "3",
ID: "c",
Score: 3.0,
Sort: []string{string(score3)},
},
},
MaxScore: 3.0,
}}
ei4 := &stubIndex{
name: "ei4",
err: nil,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: []*search.DocumentMatch{
{
Index: "4",
ID: "d",
Score: 4.0,
Sort: []string{string(score4)},
},
},
MaxScore: 4.0,
}}
alias1 := NewIndexAlias(ei1, ei2)
alias2 := NewIndexAlias(ei3, ei4)
aliasTop := NewIndexAlias(alias1, alias2)
// ei2 and ei3 have 50ms delay
// search across aliasTop should still get results from ei1 and ei4
// total should still be 4
ctx, _ := context.WithTimeout(context.Background(), 25*time.Millisecond)
query := NewTermQuery("test")
sr := NewSearchRequest(query)
expected := &SearchResult{
Status: &SearchStatus{
Total: 4,
Successful: 2,
Failed: 2,
Errors: map[string]error{
"ei2": context.DeadlineExceeded,
"ei3": context.DeadlineExceeded,
},
},
Request: sr,
Total: 2,
Hits: search.DocumentMatchCollection{
{
Index: "4",
ID: "d",
Score: 4.0,
Sort: []string{string(score4)},
},
{
Index: "1",
ID: "a",
Score: 1.0,
Sort: []string{string(score1)},
},
},
MaxScore: 4.0,
}
res, err := aliasTop.SearchInContext(ctx, sr)
if err != nil {
t.Fatalf("expected no err, got %v", err)
}
expected.Took = res.Took
if !reflect.DeepEqual(res, expected) {
t.Errorf("expected %#v, got %#v", expected, res)
}
}
// TestMultiSearchTimeoutPartial tests the case where some indexes exceed
// the timeout, while others complete successfully
func TestMultiSearchTimeoutPartial(t *testing.T) {
score1, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(1.0), 0)
score2, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(2.0), 0)
score3, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(3.0), 0)
ei1 := &stubIndex{
name: "ei1",
err: nil,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: []*search.DocumentMatch{
{
Index: "1",
ID: "a",
Score: 1.0,
Sort: []string{string(score1)},
},
},
MaxScore: 1.0,
}}
ei2 := &stubIndex{
name: "ei2",
err: nil,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: []*search.DocumentMatch{
{
Index: "2",
ID: "b",
Score: 2.0,
Sort: []string{string(score2)},
},
},
MaxScore: 2.0,
}}
ei3 := &stubIndex{
name: "ei3",
checkRequest: func(req *SearchRequest) error {
time.Sleep(50 * time.Millisecond)
return nil
},
err: nil,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: []*search.DocumentMatch{
{
Index: "3",
ID: "c",
Score: 3.0,
Sort: []string{string(score3)},
},
},
MaxScore: 3.0,
}}
// ei3 is set to take >50ms, so run search with timeout less than
// this, this should return partial results
ctx, _ := context.WithTimeout(context.Background(), 25*time.Millisecond)
query := NewTermQuery("test")
sr := NewSearchRequest(query)
expected := &SearchResult{
Status: &SearchStatus{
Total: 3,
Successful: 2,
Failed: 1,
Errors: map[string]error{
"ei3": context.DeadlineExceeded,
},
},
Request: sr,
Total: 2,
Hits: search.DocumentMatchCollection{
{
Index: "2",
ID: "b",
Score: 2.0,
Sort: []string{string(score2)},
},
{
Index: "1",
ID: "a",
Score: 1.0,
Sort: []string{string(score1)},
},
},
MaxScore: 2.0,
}
res, err := MultiSearch(ctx, sr, ei1, ei2, ei3)
if err != nil {
t.Fatalf("expected no err, got %v", err)
}
expected.Took = res.Took
if !reflect.DeepEqual(res, expected) {
t.Errorf("expected %#v, got %#v", expected, res)
}
}
// TestMultiSearchTimeout tests simple timeout cases
// 1. all searches finish successfully before timeout
// 2. no searchers finish before the timeout
// 3. no searches finish before cancellation
func TestMultiSearchTimeout(t *testing.T) {
score1, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(1.0), 0)
score2, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(2.0), 0)
ei1 := &stubIndex{
name: "ei1",
checkRequest: func(req *SearchRequest) error {
time.Sleep(50 * time.Millisecond)
return nil
},
err: nil,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: []*search.DocumentMatch{
{
Index: "1",
ID: "a",
Score: 1.0,
Sort: []string{string(score1)},
},
},
MaxScore: 1.0,
}}
ei2 := &stubIndex{
name: "ei2",
checkRequest: func(req *SearchRequest) error {
time.Sleep(50 * time.Millisecond)
return nil
},
err: nil,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: []*search.DocumentMatch{
{
Index: "2",
ID: "b",
Score: 2.0,
Sort: []string{string(score2)},
},
},
MaxScore: 2.0,
}}
// first run with absurdly long time out, should succeed
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
query := NewTermQuery("test")
sr := NewSearchRequest(query)
res, err := MultiSearch(ctx, sr, ei1, ei2)
if err != nil {
t.Errorf("expected no error, got %v", err)
}
if res.Status.Total != 2 {
t.Errorf("expected 2 total, got %d", res.Status.Failed)
}
if res.Status.Successful != 2 {
t.Errorf("expected 0 success, got %d", res.Status.Successful)
}
if res.Status.Failed != 0 {
t.Errorf("expected 2 failed, got %d", res.Status.Failed)
}
if len(res.Status.Errors) != 0 {
t.Errorf("expected 0 errors, got %v", res.Status.Errors)
}
// now run a search again with an absurdly low timeout (should timeout)
ctx, _ = context.WithTimeout(context.Background(), 1*time.Microsecond)
res, err = MultiSearch(ctx, sr, ei1, ei2)
if err != nil {
t.Errorf("expected no error, got %v", err)
}
if res.Status.Total != 2 {
t.Errorf("expected 2 failed, got %d", res.Status.Failed)
}
if res.Status.Successful != 0 {
t.Errorf("expected 0 success, got %d", res.Status.Successful)
}
if res.Status.Failed != 2 {
t.Errorf("expected 2 failed, got %d", res.Status.Failed)
}
if len(res.Status.Errors) != 2 {
t.Errorf("expected 2 errors, got %v", res.Status.Errors)
} else {
if res.Status.Errors["ei1"].Error() != context.DeadlineExceeded.Error() {
t.Errorf("expected err for 'ei1' to be '%s' got '%s'", context.DeadlineExceeded.Error(), res.Status.Errors["ei1"])
}
if res.Status.Errors["ei2"].Error() != context.DeadlineExceeded.Error() {
t.Errorf("expected err for 'ei2' to be '%s' got '%s'", context.DeadlineExceeded.Error(), res.Status.Errors["ei2"])
}
}
// now run a search again with a normal timeout, but cancel it first
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
cancel()
res, err = MultiSearch(ctx, sr, ei1, ei2)
if err != nil {
t.Errorf("expected no error, got %v", err)
}
if res.Status.Total != 2 {
t.Errorf("expected 2 failed, got %d", res.Status.Failed)
}
if res.Status.Successful != 0 {
t.Errorf("expected 0 success, got %d", res.Status.Successful)
}
if res.Status.Failed != 2 {
t.Errorf("expected 2 failed, got %d", res.Status.Failed)
}
if len(res.Status.Errors) != 2 {
t.Errorf("expected 2 errors, got %v", res.Status.Errors)
} else {
if res.Status.Errors["ei1"].Error() != context.Canceled.Error() {
t.Errorf("expected err for 'ei1' to be '%s' got '%s'", context.Canceled.Error(), res.Status.Errors["ei1"])
}
if res.Status.Errors["ei2"].Error() != context.Canceled.Error() {
t.Errorf("expected err for 'ei2' to be '%s' got '%s'", context.Canceled.Error(), res.Status.Errors["ei2"])
}
}
}
// TestMultiSearchNoError
func TestMultiSearchNoError(t *testing.T) {
score1, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(1.0), 0)
score2, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(2.0), 0)
ei1 := &stubIndex{err: nil, searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: search.DocumentMatchCollection{
{
Index: "1",
ID: "a",
Score: 1.0,
Sort: []string{string(score1)},
},
},
MaxScore: 1.0,
}}
ei2 := &stubIndex{err: nil, searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: search.DocumentMatchCollection{
{
Index: "2",
ID: "b",
Score: 2.0,
Sort: []string{string(score2)},
},
},
MaxScore: 2.0,
}}
sr := NewSearchRequest(NewTermQuery("test"))
expected := &SearchResult{
Status: &SearchStatus{
Total: 2,
Successful: 2,
Errors: make(map[string]error),
},
Request: sr,
Total: 2,
Hits: search.DocumentMatchCollection{
{
Index: "2",
ID: "b",
Score: 2.0,
Sort: []string{string(score2)},
},
{
Index: "1",
ID: "a",
Score: 1.0,
Sort: []string{string(score1)},
},
},
MaxScore: 2.0,
}
results, err := MultiSearch(context.Background(), sr, ei1, ei2)
if err != nil {
t.Error(err)
}
// cheat and ensure that Took field matches since it invovles time
expected.Took = results.Took
if !reflect.DeepEqual(results, expected) {
t.Errorf("expected %#v, got %#v", expected, results)
}
}
func TestIndexAliasMulti(t *testing.T) {
score1, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(1.0), 0)
score2, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(2.0), 0)
ei1Count := uint64(7)
ei1 := &stubIndex{
err: nil,
docCountResult: &ei1Count,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: search.DocumentMatchCollection{
{
ID: "a",
Score: 1.0,
Sort: []string{string(score1)},
},
},
MaxScore: 1.0,
}}
ei2Count := uint64(8)
ei2 := &stubIndex{
err: nil,
docCountResult: &ei2Count,
searchResult: &SearchResult{
Status: &SearchStatus{
Total: 1,
Successful: 1,
Errors: make(map[string]error),
},
Total: 1,
Hits: search.DocumentMatchCollection{
{
ID: "b",
Score: 2.0,
Sort: []string{string(score2)},
},
},
MaxScore: 2.0,
}}
alias := NewIndexAlias(ei1, ei2)
err := alias.Index("a", "a")
if err != ErrorAliasMulti {
t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
}
err = alias.Delete("a")
if err != ErrorAliasMulti {
t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
}
batch := alias.NewBatch()
err = alias.Batch(batch)
if err != ErrorAliasMulti {
t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
}
_, err = alias.Document("a")
if err != ErrorAliasMulti {
t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
}
_, err = alias.Fields()
if err != ErrorAliasMulti {
t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
}
_, err = alias.GetInternal([]byte("a"))
if err != ErrorAliasMulti {
t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
}
err = alias.SetInternal([]byte("a"), []byte("a"))
if err != ErrorAliasMulti {
t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
}
err = alias.DeleteInternal([]byte("a"))
if err != ErrorAliasMulti {
t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
}
mapping := alias.Mapping()
if mapping != nil {
t.Errorf("expected nil, got %v", mapping)
}
indexStat := alias.Stats()
if indexStat != nil {
t.Errorf("expected nil, got %v", indexStat)
}
// now a few things that should work
sr := NewSearchRequest(NewTermQuery("test"))
expected := &SearchResult{
Status: &SearchStatus{
Total: 2,
Successful: 2,
Errors: make(map[string]error),
},
Request: sr,
Total: 2,
Hits: search.DocumentMatchCollection{
{
ID: "b",
Score: 2.0,
Sort: []string{string(score2)},
},
{
ID: "a",
Score: 1.0,
Sort: []string{string(score1)},
},
},
MaxScore: 2.0,
}
results, err := alias.Search(sr)
if err != nil {
t.Error(err)
}
// cheat and ensure that Took field matches since it invovles time
expected.Took = results.Took
if !reflect.DeepEqual(results, expected) {
t.Errorf("expected %#v, got %#v", expected, results)
}
count, err := alias.DocCount()
if err != nil {
t.Errorf("error getting alias doc count: %v", err)
}
if count != (*ei1.docCountResult + *ei2.docCountResult) {
t.Errorf("expected %d, got %d", (*ei1.docCountResult + *ei2.docCountResult), count)
}
}