// The inspecting the response
func ExampleBulkIndexer_responses() {
indexer := core.NewBulkIndexer(10)
// Create a custom Sender Func, to allow inspection of response/error
indexer.BulkSender = func(buf *bytes.Buffer) error {
// @buf is the buffer of docs about to be written
respJson, err := api.DoCommand("POST", "/_bulk", nil, buf)
if err != nil {
// handle it better than this
fmt.Println(string(respJson))
}
return err
}
done := make(chan bool)
indexer.Run(done)
for i := 0; i < 20; i++ {
indexer.Index("twitter", "user", strconv.Itoa(i), "", nil, `{"name":"bob"}`, true)
}
done <- true // send shutdown signal
}
func (this *EsOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
var (
pack *engine.PipelinePack
reloadChan = make(chan interface{})
ok = true
globals = engine.Globals()
inChan = r.InChan()
reportTicker = time.NewTicker(this.reportInterval)
)
this.indexer = core.NewBulkIndexer(this.bulkMaxConn)
this.indexer.BulkMaxDocs = this.bulkMaxDocs
this.indexer.BulkMaxBuffer = this.bulkMaxBuffer
// start the bulk indexer
this.indexer.Run(this.stopChan)
defer reportTicker.Stop()
observer.Subscribe(engine.RELOAD, reloadChan)
LOOP:
for ok {
select {
case <-this.stopChan:
ok = false
case <-reportTicker.C:
this.showPeriodicalStats()
case <-reloadChan:
// TODO
case <-time.After(this.flushInterval):
this.indexer.Flush()
case pack, ok = <-inChan:
if !ok {
break LOOP
}
if globals.Debug {
globals.Println(*pack)
}
this.feedEs(h.Project(pack.Project), pack)
pack.Recycle()
}
}
engine.Globals().Printf("[%s]Total output to ES: %d", r.Name(), this.totalN)
// before shutdown, flush again
if globals.Verbose {
engine.Globals().Println("Waiting for ES flush...")
}
this.indexer.Flush()
if globals.Verbose {
engine.Globals().Println("ES flushed")
}
// let indexer stop
this.stopChan <- true
return nil
}
