Example #1
0
// Periodically flushes messages to bleve index
func (b *Bleve) PeriodicFlush(die chan bool) {
	var (
		bvBatchIndex  *bv.Batch
		err           error
		nbMessages    int
		sleepDuration time.Duration = 3 * time.Second
	)

	// Run periodic cleanup task
	go b.periodicCleanup(die)

	for {
		select {
		case <-die:
			return
		default:
		}

		nbMessages = len(b.messages)

		if nbMessages > 0 && (nbMessages >= b.batchSize || time.Now().Sub(b.lastFlush) > b.intervalFlush) {
			b.mutexFlushMessages.Lock()

			bvBatchIndex = b.index.NewBatch()

			for _, message := range b.messages {
				err = bvBatchIndex.Index(uuid.NewV4().String(), message)
				if err != nil {
					logger.Instance().
						WithError(err).
						Warning("Unable to add message to batch")
				}
			}

			if bvBatchIndex.Size() > 0 {
				err = b.index.Batch(bvBatchIndex)

				if err != nil {
					logger.Instance().
						WithError(err).
						Warning("Unable to batch index messages")
				} else {
					logger.Instance().
						WithField("nb_messages", bvBatchIndex.Size()).
						Info("Messages successfully indexed")

					b.lastFlush = time.Now()
					b.messages = []*storage.Message{}
				}
			}

			b.mutexFlushMessages.Unlock()
		}

		time.Sleep(sleepDuration)
	}
}
Example #2
0
// Periodically removes old entries from index
func (b *Bleve) periodicCleanup(die chan bool) {
	var (
		bvBatchDelete *bv.Batch
		bvRequest     *bv.SearchRequest
		bvResults     *bv.SearchResult
		bvCleaningNow bool
		bvNbCleaned   int
		err           error
		till          string
		limit         int           = 20
		offset        int           = 0
		sleepDuration time.Duration = 3 * time.Second
	)

	defer func() {
		err = b.index.Close()
		if err != nil {
			logger.Instance().
				WithError(err).
				Warning("Unable to close Bleve index")
		}
	}()

	for {
		select {
		case <-die:
			return
		default:
		}

		offset = 0
		bvNbCleaned = 0
		bvCleaningNow = true
		till = time.Now().Add(b.intervalCleanup).Format(time.RFC3339)
		bvQuery := bv.NewDateRangeQuery(nil, &till)
		bvQuery.FieldVal = "timestamp"

		for bvCleaningNow != false {
			bvRequest = bv.NewSearchRequestOptions(bvQuery, limit, offset, false)
			bvResults, err = b.index.Search(bvRequest)

			if err != nil {
				logger.Instance().
					WithError(err).
					Warning("Unable to get obsolete messages from index")

				bvCleaningNow = false
				continue
			}

			if bvResults.Hits.Len() == 0 {
				bvCleaningNow = false
				continue
			}

			// List of documents to be deleted
			bvBatchDelete = b.index.NewBatch()
			for _, hit := range bvResults.Hits {
				bvBatchDelete.Delete(hit.ID)
			}

			// Batch delete them
			err = b.index.Batch(bvBatchDelete)
			if err != nil {
				logger.Instance().
					WithError(err).
					Warning("Unable to delete obsolete messages from index")

				bvCleaningNow = false
				continue
			} else {
				bvNbCleaned += bvBatchDelete.Size()
				offset += limit
			}
		}

		if bvNbCleaned > 0 {
			logger.Instance().
				WithField("nb_messages", bvNbCleaned).
				Infof("Obsolete messages were deleted from index")
		}

		time.Sleep(sleepDuration)
	}
}