Beispiel #1
0
func (e *ElasticWReport) PutToBulk(bulk *el.BulkService, p []*models.Problem) {
	for _, problem := range p {
		problem.ReportId = e.rid
		bulk.Add(el.NewBulkIndexRequest().Index(e.indexName).Type(TypoProblemName).Doc(problem))
	}
}
Beispiel #2
0
// Periodically flushes messages to elastic
func (e *Elastic) PeriodicFlush(die chan bool) {
	var (
		esBulk        *es.BulkService
		esResponse    *es.BulkResponse
		err           error
		nbMessages    int
		sleepDuration time.Duration = 3 * time.Second
	)

	for {
		select {
		case <-die:
			return
		default:
		}

		nbMessages = len(e.messages)

		if nbMessages > 0 && (nbMessages >= e.batchSize || time.Now().Sub(e.lastFlush) > e.intervalFlush) {
			e.mutexFlushMessages.Lock()

			esBulk = e.client.Bulk()

			for _, message := range e.messages {
				esBulk.Add(es.NewBulkIndexRequest().
					Index(e.indexName).
					Type(e.typeName).
					Id(uuid.NewV4().String()).
					Ttl(e.ttl).
					Doc(message))
			}

			if esBulk.NumberOfActions() > 0 {
				esResponse, err = esBulk.Do()

				if err != nil {
					logger.Instance().
						WithError(err).
						Warning("Unable to batch index messages")
				} else {
					nbCreated := len(esResponse.Indexed())
					if nbCreated != nbMessages {
						logger.Instance().
							WithField("nb_messages", nbMessages).
							WithField("nb_created", nbCreated).
							Warning("Not all messages were indexed")
					} else {
						logger.Instance().
							WithField("nb_messages", nbMessages).
							Info("Messages successfully indexed")
					}

					e.lastFlush = time.Now()
					e.messages = []*storage.Message{}
				}
			}

			e.mutexFlushMessages.Unlock()
		}

		time.Sleep(sleepDuration)
	}
}