// Periodically flushes messages to bleve index
func (b *Bleve) PeriodicFlush(die chan bool) {
var (
bvBatchIndex *bv.Batch
err error
nbMessages int
sleepDuration time.Duration = 3 * time.Second
)
// Run periodic cleanup task
go b.periodicCleanup(die)
for {
select {
case <-die:
return
default:
}
nbMessages = len(b.messages)
if nbMessages > 0 && (nbMessages >= b.batchSize || time.Now().Sub(b.lastFlush) > b.intervalFlush) {
b.mutexFlushMessages.Lock()
bvBatchIndex = b.index.NewBatch()
for _, message := range b.messages {
err = bvBatchIndex.Index(uuid.NewV4().String(), message)
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to add message to batch")
}
}
if bvBatchIndex.Size() > 0 {
err = b.index.Batch(bvBatchIndex)
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to batch index messages")
} else {
logger.Instance().
WithField("nb_messages", bvBatchIndex.Size()).
Info("Messages successfully indexed")
b.lastFlush = time.Now()
b.messages = []*storage.Message{}
}
}
b.mutexFlushMessages.Unlock()
}
time.Sleep(sleepDuration)
}
}
// Periodically removes old entries from index
func (b *Bleve) periodicCleanup(die chan bool) {
var (
bvBatchDelete *bv.Batch
bvRequest *bv.SearchRequest
bvResults *bv.SearchResult
bvCleaningNow bool
bvNbCleaned int
err error
till string
limit int = 20
offset int = 0
sleepDuration time.Duration = 3 * time.Second
)
defer func() {
err = b.index.Close()
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to close Bleve index")
}
}()
for {
select {
case <-die:
return
default:
}
offset = 0
bvNbCleaned = 0
bvCleaningNow = true
till = time.Now().Add(b.intervalCleanup).Format(time.RFC3339)
bvQuery := bv.NewDateRangeQuery(nil, &till)
bvQuery.FieldVal = "timestamp"
for bvCleaningNow != false {
bvRequest = bv.NewSearchRequestOptions(bvQuery, limit, offset, false)
bvResults, err = b.index.Search(bvRequest)
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to get obsolete messages from index")
bvCleaningNow = false
continue
}
if bvResults.Hits.Len() == 0 {
bvCleaningNow = false
continue
}
// List of documents to be deleted
bvBatchDelete = b.index.NewBatch()
for _, hit := range bvResults.Hits {
bvBatchDelete.Delete(hit.ID)
}
// Batch delete them
err = b.index.Batch(bvBatchDelete)
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to delete obsolete messages from index")
bvCleaningNow = false
continue
} else {
bvNbCleaned += bvBatchDelete.Size()
offset += limit
}
}
if bvNbCleaned > 0 {
logger.Instance().
WithField("nb_messages", bvNbCleaned).
Infof("Obsolete messages were deleted from index")
}
time.Sleep(sleepDuration)
}
}