Beispiel #1
0
// Periodically removes old entries from index
func (b *Bleve) periodicCleanup(die chan bool) {
	var (
		bvBatchDelete *bv.Batch
		bvRequest     *bv.SearchRequest
		bvResults     *bv.SearchResult
		bvCleaningNow bool
		bvNbCleaned   int
		err           error
		till          string
		limit         int           = 20
		offset        int           = 0
		sleepDuration time.Duration = 3 * time.Second
	)

	defer func() {
		err = b.index.Close()
		if err != nil {
			logger.Instance().
				WithError(err).
				Warning("Unable to close Bleve index")
		}
	}()

	for {
		select {
		case <-die:
			return
		default:
		}

		offset = 0
		bvNbCleaned = 0
		bvCleaningNow = true
		till = time.Now().Add(b.intervalCleanup).Format(time.RFC3339)
		bvQuery := bv.NewDateRangeQuery(nil, &till)
		bvQuery.FieldVal = "timestamp"

		for bvCleaningNow != false {
			bvRequest = bv.NewSearchRequestOptions(bvQuery, limit, offset, false)
			bvResults, err = b.index.Search(bvRequest)

			if err != nil {
				logger.Instance().
					WithError(err).
					Warning("Unable to get obsolete messages from index")

				bvCleaningNow = false
				continue
			}

			if bvResults.Hits.Len() == 0 {
				bvCleaningNow = false
				continue
			}

			// List of documents to be deleted
			bvBatchDelete = b.index.NewBatch()
			for _, hit := range bvResults.Hits {
				bvBatchDelete.Delete(hit.ID)
			}

			// Batch delete them
			err = b.index.Batch(bvBatchDelete)
			if err != nil {
				logger.Instance().
					WithError(err).
					Warning("Unable to delete obsolete messages from index")

				bvCleaningNow = false
				continue
			} else {
				bvNbCleaned += bvBatchDelete.Size()
				offset += limit
			}
		}

		if bvNbCleaned > 0 {
			logger.Instance().
				WithField("nb_messages", bvNbCleaned).
				Infof("Obsolete messages were deleted from index")
		}

		time.Sleep(sleepDuration)
	}
}
Beispiel #2
0
func TestBeerSearchAll(t *testing.T) {
	defer os.RemoveAll("beer-search-test.bleve")

	mapping, err := buildIndexMapping()
	if err != nil {
		t.Fatal(err)
	}
	index, err := bleve.New("beer-search-test.bleve", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	// open the directory
	dirEntries, err := ioutil.ReadDir("data/")
	if err != nil {
		t.Fatal(err)
	}

	indexBatchSize := 100
	batch := index.NewBatch()
	batchCount := 0
	for _, dirEntry := range dirEntries {
		filename := dirEntry.Name()
		// read the bytes
		jsonBytes, err := ioutil.ReadFile("data/" + filename)
		if err != nil {
			t.Fatal(err)
		}
		// // shred them into a document
		ext := filepath.Ext(filename)
		docId := filename[:(len(filename) - len(ext))]
		batch.Index(docId, jsonBytes)
		batchCount++

		if batchCount >= indexBatchSize {
			err = index.Batch(batch)
			if err != nil {
				t.Fatal(err)
			}
			batch = index.NewBatch()
			batchCount = 0
		}
	}
	// flush the last batch
	if batchCount > 0 {
		err = index.Batch(batch)
		if err != nil {
			t.Fatal(err)
		}
	}

	expectedCount := uint64(7303)
	actualCount, err := index.DocCount()
	if err != nil {
		t.Error(err)
	}
	if actualCount != expectedCount {
		t.Errorf("expected %d documents, got %d", expectedCount, actualCount)
	}

	// run a term search
	termQuery := bleve.NewTermQuery("shock").SetField("name")
	termSearchRequest := bleve.NewSearchRequest(termQuery)
	termSearchResult, err := index.Search(termSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount := uint64(1)
	if termSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total)
	} else {
		expectedResultId := "anheuser_busch-shock_top"
		if termSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID)
		}
	}

	// run a match phrase search
	matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food")
	matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery)
	matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if matchPhraseSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total)
	} else {
		expectedResultId := "great_divide_brewing-wild_raspberry_ale"
		if matchPhraseSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID)
		}
	}

	// run a syntax query
	syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley")
	syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery)
	syntaxSearchResult, err := index.Search(syntaxSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if syntaxSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total)
	} else {
		expectedResultId := "iron_city_brewing_co-ic_light"
		if syntaxSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID)
		}
	}

	// run a numeric range search
	queryMin := 50.0
	numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv")
	numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery)
	numericSearchResult, err := index.Search(numericSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if numericSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total)
	} else {
		expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale"
		if numericSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID)
		}
	}

	// run a date range search
	queryStartDate := "2011-10-04"
	dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated")
	dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery)
	dateSearchResult, err := index.Search(dateSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(2)
	if dateSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total)
	} else {
		expectedResultId := "brasserie_du_bouffay-ambr"
		if dateSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID)
		}
	}

	// run a prefix search
	prefixQuery := bleve.NewPrefixQuery("adir").SetField("name")
	prefixSearchRequest := bleve.NewSearchRequest(prefixQuery)
	prefixSearchResult, err := index.Search(prefixSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if prefixSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total)
	} else {
		expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager"
		if prefixSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID)
		}
	}
}
Beispiel #3
0
func TestBeerSearchAll(t *testing.T) {
	defer os.RemoveAll("beer-search-test.bleve")

	mapping, err := buildIndexMapping()
	if err != nil {
		t.Fatal(err)
	}
	index, err := bleve.New("beer-search-test.bleve", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	for jf := range walkDirectory("data/", t) {
		docId := jf.filename[0:strings.LastIndex(jf.filename, ".")]
		err = index.Index(docId, jf.contents)
		if err != nil {
			t.Error(err)
		}
	}

	expectedCount := uint64(7303)
	actualCount := index.DocCount()
	if actualCount != expectedCount {
		t.Errorf("expected %d documents, got %d", expectedCount, actualCount)
	}

	// run a term search
	termQuery := bleve.NewTermQuery("shock").SetField("name")
	termSearchRequest := bleve.NewSearchRequest(termQuery)
	termSearchResult, err := index.Search(termSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount := uint64(1)
	if termSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total)
	} else {
		expectedResultId := "anheuser_busch-shock_top"
		if termSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID)
		}
	}

	// run a match phrase search
	matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food")
	matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery)
	matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if matchPhraseSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total)
	} else {
		expectedResultId := "great_divide_brewing-wild_raspberry_ale"
		if matchPhraseSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID)
		}
	}

	// run a syntax query
	syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley")
	syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery)
	syntaxSearchResult, err := index.Search(syntaxSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if syntaxSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total)
	} else {
		expectedResultId := "iron_city_brewing_co-ic_light"
		if syntaxSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID)
		}
	}

	// run a numeric range search
	queryMin := 50.0
	numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv")
	numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery)
	numericSearchResult, err := index.Search(numericSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if numericSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total)
	} else {
		expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale"
		if numericSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID)
		}
	}

	// run a date range search
	queryStartDate := "2011-10-04"
	dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated")
	dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery)
	dateSearchResult, err := index.Search(dateSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(2)
	if dateSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total)
	} else {
		expectedResultId := "brasserie_du_bouffay-ambr"
		if dateSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID)
		}
	}

	// run a prefix search
	prefixQuery := bleve.NewPrefixQuery("adir").SetField("name")
	prefixSearchRequest := bleve.NewSearchRequest(prefixQuery)
	prefixSearchResult, err := index.Search(prefixSearchRequest)
	if err != nil {
		t.Error(err)
	}
	expectedResultCount = uint64(1)
	if prefixSearchResult.Total != expectedResultCount {
		t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total)
	} else {
		expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager"
		if prefixSearchResult.Hits[0].ID != expectedResultId {
			t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID)
		}
	}
}
Beispiel #4
0
func getJobSearchRequest(r *http.Request) (*bleve.SearchRequest, error) {
	musts := []query.Query{}
	// mustNots := []query.Query{}
	// shoulds := []query.Query{}

	tagShoulds := []query.Query{}
	for _, tag := range r.URL.Query()["tags"] {
		tagShoulds = append(tagShoulds, bleve.NewMatchQuery(tag))
	}
	if len(tagShoulds) > 0 {
		booleanQuery := bleve.NewBooleanQuery()
		booleanQuery.AddShould(tagShoulds...)
		musts = append(musts, booleanQuery)
	}

	value1 := 0.0
	if len(r.URL.Query().Get("price_from")) != 0 {
		intValue1, err := strconv.ParseInt(r.URL.Query().Get("price_from"), 10, 64)
		if err != nil {
			return nil, err
		}
		value1 = float64(intValue1)
	}
	value2 := math.MaxFloat64
	if len(r.URL.Query().Get("price_to")) != 0 {
		intValue2, err := strconv.ParseInt(r.URL.Query().Get("price_to"), 10, 64)
		if err != nil {
			return nil, err
		}
		value2 = float64(intValue2)
	}

	inclusiveValue1 := true
	inclusiveValue2 := false
	numericRangeIncludiveQuery := bleve.NewNumericRangeInclusiveQuery(
		&value1,
		&value2,
		&inclusiveValue1,
		&inclusiveValue2,
	)
	numericRangeIncludiveQuery.SetField("price")
	musts = append(musts, numericRangeIncludiveQuery)

	period := int64(30)
	if len(r.URL.Query().Get("period")) != 0 {
		periodTemp, err := strconv.ParseInt(r.URL.Query().Get("period"), 10, 64)
		if err != nil {
			return nil, err
		}
		if periodTemp > 0 && periodTemp <= 365 {
			period = periodTemp
		}
	}

	now := time.Now()
	dateTo := time.Now().Add(time.Duration(24*period) * time.Hour)
	dateRangeQuery := bleve.NewDateRangeQuery(now, dateTo)
	dateRangeQuery.SetField("startDate")
	musts = append(musts, dateRangeQuery)

	query := bleve.NewBooleanQuery()
	query.AddMust(musts...)
	// query.AddMustNot(mustNots...)
	// query.AddShould(shoulds...)

	searchRequest := bleve.NewSearchRequest(query)
	searchRequest.Fields = []string{"*"}

	return searchRequest, nil
}