func (e *ElasticWReport) PutToBulk(bulk *el.BulkService, p []*models.Problem) {
for _, problem := range p {
problem.ReportId = e.rid
bulk.Add(el.NewBulkIndexRequest().Index(e.indexName).Type(TypoProblemName).Doc(problem))
}
}
// Periodically flushes messages to elastic
func (e *Elastic) PeriodicFlush(die chan bool) {
var (
esBulk *es.BulkService
esResponse *es.BulkResponse
err error
nbMessages int
sleepDuration time.Duration = 3 * time.Second
)
for {
select {
case <-die:
return
default:
}
nbMessages = len(e.messages)
if nbMessages > 0 && (nbMessages >= e.batchSize || time.Now().Sub(e.lastFlush) > e.intervalFlush) {
e.mutexFlushMessages.Lock()
esBulk = e.client.Bulk()
for _, message := range e.messages {
esBulk.Add(es.NewBulkIndexRequest().
Index(e.indexName).
Type(e.typeName).
Id(uuid.NewV4().String()).
Ttl(e.ttl).
Doc(message))
}
if esBulk.NumberOfActions() > 0 {
esResponse, err = esBulk.Do()
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to batch index messages")
} else {
nbCreated := len(esResponse.Indexed())
if nbCreated != nbMessages {
logger.Instance().
WithField("nb_messages", nbMessages).
WithField("nb_created", nbCreated).
Warning("Not all messages were indexed")
} else {
logger.Instance().
WithField("nb_messages", nbMessages).
Info("Messages successfully indexed")
}
e.lastFlush = time.Now()
e.messages = []*storage.Message{}
}
}
e.mutexFlushMessages.Unlock()
}
time.Sleep(sleepDuration)
}
}