Example #1
0
func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) {
	tokens := make(analysis.TokenStream, 0)
	tokens = append(tokens, &analysis.Token{
		Start:    0,
		End:      len(n.value),
		Term:     n.value,
		Position: 1,
		Type:     analysis.DateTime,
	})

	original, err := n.value.Int64()
	if err == nil {

		shift := DefaultDateTimePrecisionStep
		for shift < 64 {
			shiftEncoded, err := numeric_util.NewPrefixCodedInt64(original, shift)
			if err != nil {
				break
			}
			token := analysis.Token{
				Start:    0,
				End:      len(shiftEncoded),
				Term:     shiftEncoded,
				Position: 1,
				Type:     analysis.DateTime,
			}
			tokens = append(tokens, &token)
			shift += DefaultDateTimePrecisionStep
		}
	}

	fieldLength := len(tokens)
	tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions)
	return fieldLength, tokenFreqs
}
Example #2
0
func TestIndexAliasMultipleLayer(t *testing.T) {
	score1, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(1.0), 0)
	score2, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(2.0), 0)
	score3, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(3.0), 0)
	score4, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(4.0), 0)
	ei1 := &stubIndex{
		name: "ei1",
		err:  nil,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: []*search.DocumentMatch{
				{
					Index: "1",
					ID:    "a",
					Score: 1.0,
					Sort:  []string{string(score1)},
				},
			},
			MaxScore: 1.0,
		}}
	ei2 := &stubIndex{
		name: "ei2",
		checkRequest: func(req *SearchRequest) error {
			time.Sleep(50 * time.Millisecond)
			return nil
		},
		err: nil,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: []*search.DocumentMatch{
				{
					Index: "2",
					ID:    "b",
					Score: 2.0,
					Sort:  []string{string(score2)},
				},
			},
			MaxScore: 2.0,
		}}

	ei3 := &stubIndex{
		name: "ei3",
		checkRequest: func(req *SearchRequest) error {
			time.Sleep(50 * time.Millisecond)
			return nil
		},
		err: nil,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: []*search.DocumentMatch{
				{
					Index: "3",
					ID:    "c",
					Score: 3.0,
					Sort:  []string{string(score3)},
				},
			},
			MaxScore: 3.0,
		}}

	ei4 := &stubIndex{
		name: "ei4",
		err:  nil,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: []*search.DocumentMatch{
				{
					Index: "4",
					ID:    "d",
					Score: 4.0,
					Sort:  []string{string(score4)},
				},
			},
			MaxScore: 4.0,
		}}

	alias1 := NewIndexAlias(ei1, ei2)
	alias2 := NewIndexAlias(ei3, ei4)
	aliasTop := NewIndexAlias(alias1, alias2)

	// ei2 and ei3 have 50ms delay
	// search across aliasTop should still get results from ei1 and ei4
	// total should still be 4

	ctx, _ := context.WithTimeout(context.Background(), 25*time.Millisecond)
	query := NewTermQuery("test")
	sr := NewSearchRequest(query)
	expected := &SearchResult{
		Status: &SearchStatus{
			Total:      4,
			Successful: 2,
			Failed:     2,
			Errors: map[string]error{
				"ei2": context.DeadlineExceeded,
				"ei3": context.DeadlineExceeded,
			},
		},
		Request: sr,
		Total:   2,
		Hits: search.DocumentMatchCollection{
			{
				Index: "4",
				ID:    "d",
				Score: 4.0,
				Sort:  []string{string(score4)},
			},
			{
				Index: "1",
				ID:    "a",
				Score: 1.0,
				Sort:  []string{string(score1)},
			},
		},
		MaxScore: 4.0,
	}

	res, err := aliasTop.SearchInContext(ctx, sr)
	if err != nil {
		t.Fatalf("expected no err, got %v", err)
	}
	expected.Took = res.Took
	if !reflect.DeepEqual(res, expected) {
		t.Errorf("expected %#v, got %#v", expected, res)
	}
}
Example #3
0
// TestMultiSearchTimeoutPartial tests the case where some indexes exceed
// the timeout, while others complete successfully
func TestMultiSearchTimeoutPartial(t *testing.T) {
	score1, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(1.0), 0)
	score2, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(2.0), 0)
	score3, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(3.0), 0)
	ei1 := &stubIndex{
		name: "ei1",
		err:  nil,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: []*search.DocumentMatch{
				{
					Index: "1",
					ID:    "a",
					Score: 1.0,
					Sort:  []string{string(score1)},
				},
			},
			MaxScore: 1.0,
		}}
	ei2 := &stubIndex{
		name: "ei2",
		err:  nil,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: []*search.DocumentMatch{
				{
					Index: "2",
					ID:    "b",
					Score: 2.0,
					Sort:  []string{string(score2)},
				},
			},
			MaxScore: 2.0,
		}}

	ei3 := &stubIndex{
		name: "ei3",
		checkRequest: func(req *SearchRequest) error {
			time.Sleep(50 * time.Millisecond)
			return nil
		},
		err: nil,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: []*search.DocumentMatch{
				{
					Index: "3",
					ID:    "c",
					Score: 3.0,
					Sort:  []string{string(score3)},
				},
			},
			MaxScore: 3.0,
		}}

	// ei3 is set to take >50ms, so run search with timeout less than
	// this, this should return partial results
	ctx, _ := context.WithTimeout(context.Background(), 25*time.Millisecond)
	query := NewTermQuery("test")
	sr := NewSearchRequest(query)
	expected := &SearchResult{
		Status: &SearchStatus{
			Total:      3,
			Successful: 2,
			Failed:     1,
			Errors: map[string]error{
				"ei3": context.DeadlineExceeded,
			},
		},
		Request: sr,
		Total:   2,
		Hits: search.DocumentMatchCollection{
			{
				Index: "2",
				ID:    "b",
				Score: 2.0,
				Sort:  []string{string(score2)},
			},
			{
				Index: "1",
				ID:    "a",
				Score: 1.0,
				Sort:  []string{string(score1)},
			},
		},
		MaxScore: 2.0,
	}

	res, err := MultiSearch(ctx, sr, ei1, ei2, ei3)
	if err != nil {
		t.Fatalf("expected no err, got %v", err)
	}
	expected.Took = res.Took
	if !reflect.DeepEqual(res, expected) {
		t.Errorf("expected %#v, got %#v", expected, res)
	}
}
Example #4
0
// TestMultiSearchTimeout tests simple timeout cases
// 1. all searches finish successfully before timeout
// 2. no searchers finish before the timeout
// 3. no searches finish before cancellation
func TestMultiSearchTimeout(t *testing.T) {
	score1, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(1.0), 0)
	score2, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(2.0), 0)
	ei1 := &stubIndex{
		name: "ei1",
		checkRequest: func(req *SearchRequest) error {
			time.Sleep(50 * time.Millisecond)
			return nil
		},
		err: nil,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: []*search.DocumentMatch{
				{
					Index: "1",
					ID:    "a",
					Score: 1.0,
					Sort:  []string{string(score1)},
				},
			},
			MaxScore: 1.0,
		}}
	ei2 := &stubIndex{
		name: "ei2",
		checkRequest: func(req *SearchRequest) error {
			time.Sleep(50 * time.Millisecond)
			return nil
		},
		err: nil,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: []*search.DocumentMatch{
				{
					Index: "2",
					ID:    "b",
					Score: 2.0,
					Sort:  []string{string(score2)},
				},
			},
			MaxScore: 2.0,
		}}

	// first run with absurdly long time out, should succeed
	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
	query := NewTermQuery("test")
	sr := NewSearchRequest(query)
	res, err := MultiSearch(ctx, sr, ei1, ei2)
	if err != nil {
		t.Errorf("expected no error, got %v", err)
	}
	if res.Status.Total != 2 {
		t.Errorf("expected 2 total, got %d", res.Status.Failed)
	}
	if res.Status.Successful != 2 {
		t.Errorf("expected 0 success, got %d", res.Status.Successful)
	}
	if res.Status.Failed != 0 {
		t.Errorf("expected 2 failed, got %d", res.Status.Failed)
	}
	if len(res.Status.Errors) != 0 {
		t.Errorf("expected 0 errors, got %v", res.Status.Errors)
	}

	// now run a search again with an absurdly low timeout (should timeout)
	ctx, _ = context.WithTimeout(context.Background(), 1*time.Microsecond)
	res, err = MultiSearch(ctx, sr, ei1, ei2)
	if err != nil {
		t.Errorf("expected no error, got %v", err)
	}
	if res.Status.Total != 2 {
		t.Errorf("expected 2 failed, got %d", res.Status.Failed)
	}
	if res.Status.Successful != 0 {
		t.Errorf("expected 0 success, got %d", res.Status.Successful)
	}
	if res.Status.Failed != 2 {
		t.Errorf("expected 2 failed, got %d", res.Status.Failed)
	}
	if len(res.Status.Errors) != 2 {
		t.Errorf("expected 2 errors, got %v", res.Status.Errors)
	} else {
		if res.Status.Errors["ei1"].Error() != context.DeadlineExceeded.Error() {
			t.Errorf("expected err for 'ei1' to be '%s' got '%s'", context.DeadlineExceeded.Error(), res.Status.Errors["ei1"])
		}
		if res.Status.Errors["ei2"].Error() != context.DeadlineExceeded.Error() {
			t.Errorf("expected err for 'ei2' to be '%s' got '%s'", context.DeadlineExceeded.Error(), res.Status.Errors["ei2"])
		}
	}

	// now run a search again with a normal timeout, but cancel it first
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	cancel()
	res, err = MultiSearch(ctx, sr, ei1, ei2)
	if err != nil {
		t.Errorf("expected no error, got %v", err)
	}
	if res.Status.Total != 2 {
		t.Errorf("expected 2 failed, got %d", res.Status.Failed)
	}
	if res.Status.Successful != 0 {
		t.Errorf("expected 0 success, got %d", res.Status.Successful)
	}
	if res.Status.Failed != 2 {
		t.Errorf("expected 2 failed, got %d", res.Status.Failed)
	}
	if len(res.Status.Errors) != 2 {
		t.Errorf("expected 2 errors, got %v", res.Status.Errors)
	} else {
		if res.Status.Errors["ei1"].Error() != context.Canceled.Error() {
			t.Errorf("expected err for 'ei1' to be '%s' got '%s'", context.Canceled.Error(), res.Status.Errors["ei1"])
		}
		if res.Status.Errors["ei2"].Error() != context.Canceled.Error() {
			t.Errorf("expected err for 'ei2' to be '%s' got '%s'", context.Canceled.Error(), res.Status.Errors["ei2"])
		}
	}
}
Example #5
0
// TestMultiSearchNoError
func TestMultiSearchNoError(t *testing.T) {
	score1, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(1.0), 0)
	score2, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(2.0), 0)
	ei1 := &stubIndex{err: nil, searchResult: &SearchResult{
		Status: &SearchStatus{
			Total:      1,
			Successful: 1,
			Errors:     make(map[string]error),
		},
		Total: 1,
		Hits: search.DocumentMatchCollection{
			{
				Index: "1",
				ID:    "a",
				Score: 1.0,
				Sort:  []string{string(score1)},
			},
		},
		MaxScore: 1.0,
	}}
	ei2 := &stubIndex{err: nil, searchResult: &SearchResult{
		Status: &SearchStatus{
			Total:      1,
			Successful: 1,
			Errors:     make(map[string]error),
		},
		Total: 1,
		Hits: search.DocumentMatchCollection{
			{
				Index: "2",
				ID:    "b",
				Score: 2.0,
				Sort:  []string{string(score2)},
			},
		},
		MaxScore: 2.0,
	}}

	sr := NewSearchRequest(NewTermQuery("test"))
	expected := &SearchResult{
		Status: &SearchStatus{
			Total:      2,
			Successful: 2,
			Errors:     make(map[string]error),
		},
		Request: sr,
		Total:   2,
		Hits: search.DocumentMatchCollection{
			{
				Index: "2",
				ID:    "b",
				Score: 2.0,
				Sort:  []string{string(score2)},
			},
			{
				Index: "1",
				ID:    "a",
				Score: 1.0,
				Sort:  []string{string(score1)},
			},
		},
		MaxScore: 2.0,
	}

	results, err := MultiSearch(context.Background(), sr, ei1, ei2)
	if err != nil {
		t.Error(err)
	}
	// cheat and ensure that Took field matches since it invovles time
	expected.Took = results.Took
	if !reflect.DeepEqual(results, expected) {
		t.Errorf("expected %#v, got %#v", expected, results)
	}
}
Example #6
0
func TestIndexAliasMulti(t *testing.T) {
	score1, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(1.0), 0)
	score2, _ := numeric_util.NewPrefixCodedInt64(numeric_util.Float64ToInt64(2.0), 0)
	ei1Count := uint64(7)
	ei1 := &stubIndex{
		err:            nil,
		docCountResult: &ei1Count,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: search.DocumentMatchCollection{
				{
					ID:    "a",
					Score: 1.0,
					Sort:  []string{string(score1)},
				},
			},
			MaxScore: 1.0,
		}}
	ei2Count := uint64(8)
	ei2 := &stubIndex{
		err:            nil,
		docCountResult: &ei2Count,
		searchResult: &SearchResult{
			Status: &SearchStatus{
				Total:      1,
				Successful: 1,
				Errors:     make(map[string]error),
			},
			Total: 1,
			Hits: search.DocumentMatchCollection{
				{
					ID:    "b",
					Score: 2.0,
					Sort:  []string{string(score2)},
				},
			},
			MaxScore: 2.0,
		}}

	alias := NewIndexAlias(ei1, ei2)

	err := alias.Index("a", "a")
	if err != ErrorAliasMulti {
		t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
	}

	err = alias.Delete("a")
	if err != ErrorAliasMulti {
		t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
	}

	batch := alias.NewBatch()
	err = alias.Batch(batch)
	if err != ErrorAliasMulti {
		t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
	}

	_, err = alias.Document("a")
	if err != ErrorAliasMulti {
		t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
	}

	_, err = alias.Fields()
	if err != ErrorAliasMulti {
		t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
	}

	_, err = alias.GetInternal([]byte("a"))
	if err != ErrorAliasMulti {
		t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
	}

	err = alias.SetInternal([]byte("a"), []byte("a"))
	if err != ErrorAliasMulti {
		t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
	}

	err = alias.DeleteInternal([]byte("a"))
	if err != ErrorAliasMulti {
		t.Errorf("expected %v, got %v", ErrorAliasMulti, err)
	}

	mapping := alias.Mapping()
	if mapping != nil {
		t.Errorf("expected nil, got %v", mapping)
	}

	indexStat := alias.Stats()
	if indexStat != nil {
		t.Errorf("expected nil, got %v", indexStat)
	}

	// now a few things that should work
	sr := NewSearchRequest(NewTermQuery("test"))
	expected := &SearchResult{
		Status: &SearchStatus{
			Total:      2,
			Successful: 2,
			Errors:     make(map[string]error),
		},
		Request: sr,
		Total:   2,
		Hits: search.DocumentMatchCollection{
			{
				ID:    "b",
				Score: 2.0,
				Sort:  []string{string(score2)},
			},
			{
				ID:    "a",
				Score: 1.0,
				Sort:  []string{string(score1)},
			},
		},
		MaxScore: 2.0,
	}
	results, err := alias.Search(sr)
	if err != nil {
		t.Error(err)
	}
	// cheat and ensure that Took field matches since it invovles time
	expected.Took = results.Took
	if !reflect.DeepEqual(results, expected) {
		t.Errorf("expected %#v, got %#v", expected, results)
	}

	count, err := alias.DocCount()
	if err != nil {
		t.Errorf("error getting alias doc count: %v", err)
	}
	if count != (*ei1.docCountResult + *ei2.docCountResult) {
		t.Errorf("expected %d, got %d", (*ei1.docCountResult + *ei2.docCountResult), count)
	}
}