// Periodically removes old entries from index
func (b *Bleve) periodicCleanup(die chan bool) {
var (
bvBatchDelete *bv.Batch
bvRequest *bv.SearchRequest
bvResults *bv.SearchResult
bvCleaningNow bool
bvNbCleaned int
err error
till string
limit int = 20
offset int = 0
sleepDuration time.Duration = 3 * time.Second
)
defer func() {
err = b.index.Close()
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to close Bleve index")
}
}()
for {
select {
case <-die:
return
default:
}
offset = 0
bvNbCleaned = 0
bvCleaningNow = true
till = time.Now().Add(b.intervalCleanup).Format(time.RFC3339)
bvQuery := bv.NewDateRangeQuery(nil, &till)
bvQuery.FieldVal = "timestamp"
for bvCleaningNow != false {
bvRequest = bv.NewSearchRequestOptions(bvQuery, limit, offset, false)
bvResults, err = b.index.Search(bvRequest)
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to get obsolete messages from index")
bvCleaningNow = false
continue
}
if bvResults.Hits.Len() == 0 {
bvCleaningNow = false
continue
}
// List of documents to be deleted
bvBatchDelete = b.index.NewBatch()
for _, hit := range bvResults.Hits {
bvBatchDelete.Delete(hit.ID)
}
// Batch delete them
err = b.index.Batch(bvBatchDelete)
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to delete obsolete messages from index")
bvCleaningNow = false
continue
} else {
bvNbCleaned += bvBatchDelete.Size()
offset += limit
}
}
if bvNbCleaned > 0 {
logger.Instance().
WithField("nb_messages", bvNbCleaned).
Infof("Obsolete messages were deleted from index")
}
time.Sleep(sleepDuration)
}
}
func TestBeerSearchAll(t *testing.T) {
defer os.RemoveAll("beer-search-test.bleve")
mapping, err := buildIndexMapping()
if err != nil {
t.Fatal(err)
}
index, err := bleve.New("beer-search-test.bleve", mapping)
if err != nil {
t.Fatal(err)
}
defer index.Close()
// open the directory
dirEntries, err := ioutil.ReadDir("data/")
if err != nil {
t.Fatal(err)
}
indexBatchSize := 100
batch := index.NewBatch()
batchCount := 0
for _, dirEntry := range dirEntries {
filename := dirEntry.Name()
// read the bytes
jsonBytes, err := ioutil.ReadFile("data/" + filename)
if err != nil {
t.Fatal(err)
}
// // shred them into a document
ext := filepath.Ext(filename)
docId := filename[:(len(filename) - len(ext))]
batch.Index(docId, jsonBytes)
batchCount++
if batchCount >= indexBatchSize {
err = index.Batch(batch)
if err != nil {
t.Fatal(err)
}
batch = index.NewBatch()
batchCount = 0
}
}
// flush the last batch
if batchCount > 0 {
err = index.Batch(batch)
if err != nil {
t.Fatal(err)
}
}
expectedCount := uint64(7303)
actualCount, err := index.DocCount()
if err != nil {
t.Error(err)
}
if actualCount != expectedCount {
t.Errorf("expected %d documents, got %d", expectedCount, actualCount)
}
// run a term search
termQuery := bleve.NewTermQuery("shock").SetField("name")
termSearchRequest := bleve.NewSearchRequest(termQuery)
termSearchResult, err := index.Search(termSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount := uint64(1)
if termSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total)
} else {
expectedResultId := "anheuser_busch-shock_top"
if termSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID)
}
}
// run a match phrase search
matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food")
matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery)
matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if matchPhraseSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total)
} else {
expectedResultId := "great_divide_brewing-wild_raspberry_ale"
if matchPhraseSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID)
}
}
// run a syntax query
syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley")
syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery)
syntaxSearchResult, err := index.Search(syntaxSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if syntaxSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total)
} else {
expectedResultId := "iron_city_brewing_co-ic_light"
if syntaxSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID)
}
}
// run a numeric range search
queryMin := 50.0
numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv")
numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery)
numericSearchResult, err := index.Search(numericSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if numericSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total)
} else {
expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale"
if numericSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID)
}
}
// run a date range search
queryStartDate := "2011-10-04"
dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated")
dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery)
dateSearchResult, err := index.Search(dateSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(2)
if dateSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total)
} else {
expectedResultId := "brasserie_du_bouffay-ambr"
if dateSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID)
}
}
// run a prefix search
prefixQuery := bleve.NewPrefixQuery("adir").SetField("name")
prefixSearchRequest := bleve.NewSearchRequest(prefixQuery)
prefixSearchResult, err := index.Search(prefixSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if prefixSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total)
} else {
expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager"
if prefixSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID)
}
}
}
func TestBeerSearchAll(t *testing.T) {
defer os.RemoveAll("beer-search-test.bleve")
mapping, err := buildIndexMapping()
if err != nil {
t.Fatal(err)
}
index, err := bleve.New("beer-search-test.bleve", mapping)
if err != nil {
t.Fatal(err)
}
defer index.Close()
for jf := range walkDirectory("data/", t) {
docId := jf.filename[0:strings.LastIndex(jf.filename, ".")]
err = index.Index(docId, jf.contents)
if err != nil {
t.Error(err)
}
}
expectedCount := uint64(7303)
actualCount := index.DocCount()
if actualCount != expectedCount {
t.Errorf("expected %d documents, got %d", expectedCount, actualCount)
}
// run a term search
termQuery := bleve.NewTermQuery("shock").SetField("name")
termSearchRequest := bleve.NewSearchRequest(termQuery)
termSearchResult, err := index.Search(termSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount := uint64(1)
if termSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, termSearchResult.Total)
} else {
expectedResultId := "anheuser_busch-shock_top"
if termSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, termSearchResult.Hits[0].ID)
}
}
// run a match phrase search
matchPhraseQuery := bleve.NewMatchPhraseQuery("spicy mexican food")
matchPhraseSearchRequest := bleve.NewSearchRequest(matchPhraseQuery)
matchPhraseSearchResult, err := index.Search(matchPhraseSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if matchPhraseSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, matchPhraseSearchResult.Total)
} else {
expectedResultId := "great_divide_brewing-wild_raspberry_ale"
if matchPhraseSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, matchPhraseSearchResult.Hits[0].ID)
}
}
// run a syntax query
syntaxQuery := bleve.NewQueryStringQuery("+name:light +description:water -description:barley")
syntaxSearchRequest := bleve.NewSearchRequest(syntaxQuery)
syntaxSearchResult, err := index.Search(syntaxSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if syntaxSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, syntaxSearchResult.Total)
} else {
expectedResultId := "iron_city_brewing_co-ic_light"
if syntaxSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, syntaxSearchResult.Hits[0].ID)
}
}
// run a numeric range search
queryMin := 50.0
numericRangeQuery := bleve.NewNumericRangeQuery(&queryMin, nil).SetField("abv")
numericSearchRequest := bleve.NewSearchRequest(numericRangeQuery)
numericSearchResult, err := index.Search(numericSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if numericSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, numericSearchResult.Total)
} else {
expectedResultId := "woodforde_s_norfolk_ales-norfolk_nog_old_dark_ale"
if numericSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, numericSearchResult.Hits[0].ID)
}
}
// run a date range search
queryStartDate := "2011-10-04"
dateRangeQuery := bleve.NewDateRangeQuery(&queryStartDate, nil).SetField("updated")
dateSearchRequest := bleve.NewSearchRequest(dateRangeQuery)
dateSearchResult, err := index.Search(dateSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(2)
if dateSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, dateSearchResult.Total)
} else {
expectedResultId := "brasserie_du_bouffay-ambr"
if dateSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, dateSearchResult.Hits[0].ID)
}
}
// run a prefix search
prefixQuery := bleve.NewPrefixQuery("adir").SetField("name")
prefixSearchRequest := bleve.NewSearchRequest(prefixQuery)
prefixSearchResult, err := index.Search(prefixSearchRequest)
if err != nil {
t.Error(err)
}
expectedResultCount = uint64(1)
if prefixSearchResult.Total != expectedResultCount {
t.Errorf("expected %d hits, got %d", expectedResultCount, prefixSearchResult.Total)
} else {
expectedResultId := "f_x_matt_brewing-saranac_adirondack_lager"
if prefixSearchResult.Hits[0].ID != expectedResultId {
t.Errorf("expected top hit ID: %s, got %s", expectedResultId, prefixSearchResult.Hits[0].ID)
}
}
}
func getJobSearchRequest(r *http.Request) (*bleve.SearchRequest, error) {
musts := []query.Query{}
// mustNots := []query.Query{}
// shoulds := []query.Query{}
tagShoulds := []query.Query{}
for _, tag := range r.URL.Query()["tags"] {
tagShoulds = append(tagShoulds, bleve.NewMatchQuery(tag))
}
if len(tagShoulds) > 0 {
booleanQuery := bleve.NewBooleanQuery()
booleanQuery.AddShould(tagShoulds...)
musts = append(musts, booleanQuery)
}
value1 := 0.0
if len(r.URL.Query().Get("price_from")) != 0 {
intValue1, err := strconv.ParseInt(r.URL.Query().Get("price_from"), 10, 64)
if err != nil {
return nil, err
}
value1 = float64(intValue1)
}
value2 := math.MaxFloat64
if len(r.URL.Query().Get("price_to")) != 0 {
intValue2, err := strconv.ParseInt(r.URL.Query().Get("price_to"), 10, 64)
if err != nil {
return nil, err
}
value2 = float64(intValue2)
}
inclusiveValue1 := true
inclusiveValue2 := false
numericRangeIncludiveQuery := bleve.NewNumericRangeInclusiveQuery(
&value1,
&value2,
&inclusiveValue1,
&inclusiveValue2,
)
numericRangeIncludiveQuery.SetField("price")
musts = append(musts, numericRangeIncludiveQuery)
period := int64(30)
if len(r.URL.Query().Get("period")) != 0 {
periodTemp, err := strconv.ParseInt(r.URL.Query().Get("period"), 10, 64)
if err != nil {
return nil, err
}
if periodTemp > 0 && periodTemp <= 365 {
period = periodTemp
}
}
now := time.Now()
dateTo := time.Now().Add(time.Duration(24*period) * time.Hour)
dateRangeQuery := bleve.NewDateRangeQuery(now, dateTo)
dateRangeQuery.SetField("startDate")
musts = append(musts, dateRangeQuery)
query := bleve.NewBooleanQuery()
query.AddMust(musts...)
// query.AddMustNot(mustNots...)
// query.AddShould(shoulds...)
searchRequest := bleve.NewSearchRequest(query)
searchRequest.Fields = []string{"*"}
return searchRequest, nil
}