func (lsi *LogstreamInput) Run(ir p.InputRunner, h p.PluginHelper, stopChan chan chan bool,
deliverer p.Deliverer, sRunner p.SplitterRunner) {
if !sRunner.UseMsgBytes() {
sRunner.SetPackDecorator(lsi.packDecorator)
}
lsi.ir = ir
lsi.stopChan = stopChan
lsi.deliverer = deliverer
lsi.sRunner = sRunner
var err error
// Check for more data interval
interval, _ := time.ParseDuration("250ms")
tick := time.Tick(interval)
ok := true
for ok {
// Clear our error
err = nil
// Attempt to read and deliver as many as we can.
err = lsi.deliverRecords()
// Save our position if the stream hasn't done so for us.
if err != io.EOF {
lsi.stream.SavePosition()
}
lsi.recordCount = 0
if err != nil && err != io.EOF {
ir.LogError(err)
}
// Did our parser func get stopped?
if lsi.stopped != nil {
ok = false
continue
}
// Wait for our next interval, stop if needed
select {
case lsi.stopped = <-stopChan:
ok = false
case <-tick:
continue
}
}
close(lsi.stopped)
deliverer.Done()
}
func splitStream(ir p.InputRunner, sRunner p.SplitterRunner, r io.ReadCloser) error {
var (
record []byte
longRecord []byte
err error
deliver bool
nullSplitter bool
)
// If we're using a NullSplitter we want to make sure we capture the
// entire HTTP request or response body and not be subject to what we get
// from a single Read() call.
if _, ok := sRunner.Splitter().(*p.NullSplitter); ok {
nullSplitter = true
}
for err == nil {
deliver = true
_, record, err = sRunner.GetRecordFromStream(r)
if err == io.ErrShortBuffer {
if sRunner.KeepTruncated() {
err = fmt.Errorf("record exceeded MAX_RECORD_SIZE %d and was truncated",
message.MAX_RECORD_SIZE)
} else {
deliver = false
err = fmt.Errorf("record exceeded MAX_RECORD_SIZE %d and was dropped",
message.MAX_RECORD_SIZE)
}
ir.LogError(err)
err = nil // non-fatal, keep going
} else if sRunner.IncompleteFinal() && err == io.EOF && len(record) == 0 {
record = sRunner.GetRemainingData()
}
if len(record) > 0 && deliver {
if nullSplitter {
// Concatenate all the records until EOF. This should be safe
// b/c NullSplitter means FindRecord will always return the
// full buffer contents, we don't have to worry about
// GetRecordFromStream trying to append multiple reads to a
// single record and triggering an io.ErrShortBuffer error.
longRecord = append(longRecord, record...)
} else {
sRunner.DeliverRecord(record, nil)
}
}
}
r.Close()
if err == io.EOF && nullSplitter && len(longRecord) > 0 {
sRunner.DeliverRecord(longRecord, nil)
}
return err
}
