func ghToEs(in <-chan *github.Issue, out chan<- es.BulkableRequest) {
for source := range in {
id := *source.HTMLURL
issue := esIssue{
Repo: *source.Repository.FullName,
Number: *source.Number,
Title: *source.Title,
State: *source.State,
Comments: *source.Comments,
Body: source.Body,
CreatedAt: source.CreatedAt,
UpdatedAt: source.UpdatedAt,
ClosedAt: source.ClosedAt,
Reporter: *source.User.Login,
Labels: []string{},
Tags: make(map[string]string),
}
for _, label := range source.Labels {
name := *label.Name
issue.Labels = append(issue.Labels, name)
priorityMatch := priorityRE.FindSubmatch([]byte(name))
if priorityMatch != nil {
prio, _ := strconv.ParseInt(string(priorityMatch[1]), 10, 8)
prio8 := int8(prio)
issue.Priority = &prio8
}
splitMatch := splitRE.FindSubmatch([]byte(name))
if splitMatch != nil {
issue.Tags[string(splitMatch[1])] = string(splitMatch[2])
}
}
if source.Assignee != nil {
issue.Assignee = source.Assignee.Login
}
if source.Milestone != nil {
issue.Milestone = source.Milestone.Title
}
out <- es.NewBulkIndexRequest().Index(*esIndex).Type("issue").Id(id).Doc(issue)
}
}
func (e *ElasticWReport) PutToBulk(bulk *el.BulkService, p []*models.Problem) {
for _, problem := range p {
problem.ReportId = e.rid
bulk.Add(el.NewBulkIndexRequest().Index(e.indexName).Type(TypoProblemName).Doc(problem))
}
}
// Periodically flushes messages to elastic
func (e *Elastic) PeriodicFlush(die chan bool) {
var (
esBulk *es.BulkService
esResponse *es.BulkResponse
err error
nbMessages int
sleepDuration time.Duration = 3 * time.Second
)
for {
select {
case <-die:
return
default:
}
nbMessages = len(e.messages)
if nbMessages > 0 && (nbMessages >= e.batchSize || time.Now().Sub(e.lastFlush) > e.intervalFlush) {
e.mutexFlushMessages.Lock()
esBulk = e.client.Bulk()
for _, message := range e.messages {
esBulk.Add(es.NewBulkIndexRequest().
Index(e.indexName).
Type(e.typeName).
Id(uuid.NewV4().String()).
Ttl(e.ttl).
Doc(message))
}
if esBulk.NumberOfActions() > 0 {
esResponse, err = esBulk.Do()
if err != nil {
logger.Instance().
WithError(err).
Warning("Unable to batch index messages")
} else {
nbCreated := len(esResponse.Indexed())
if nbCreated != nbMessages {
logger.Instance().
WithField("nb_messages", nbMessages).
WithField("nb_created", nbCreated).
Warning("Not all messages were indexed")
} else {
logger.Instance().
WithField("nb_messages", nbMessages).
Info("Messages successfully indexed")
}
e.lastFlush = time.Now()
e.messages = []*storage.Message{}
}
}
e.mutexFlushMessages.Unlock()
}
time.Sleep(sleepDuration)
}
}