// Creates deliverer and stop channel and starts the provided LogstreamInput.
func (li *LogstreamerInput) startLogstreamInput(logstream *LogstreamInput, i int,
ir p.InputRunner, h p.PluginHelper) {
stop := make(chan chan bool, 1)
token := strconv.Itoa(i)
deliverer := ir.NewDeliverer(token)
sRunner := ir.NewSplitterRunner(token)
li.stopLogstreamChans = append(li.stopLogstreamChans, stop)
go logstream.Run(ir, h, stop, deliverer, sRunner)
}
func (input *S3SplitFileInput) fetcher(runner pipeline.InputRunner, wg *sync.WaitGroup, workerId uint32) {
var (
s3Key string
startTime time.Time
duration float64
)
fetcherName := fmt.Sprintf("S3Reader%d", workerId)
deliverer := runner.NewDeliverer(fetcherName)
defer deliverer.Done()
splitterRunner := runner.NewSplitterRunner(fetcherName)
ok := true
for ok {
select {
case s3Key, ok = <-input.listChan:
if !ok {
// Channel is closed => we're shutting down, exit cleanly.
// runner.LogMessage("Fetcher all done! shutting down.")
break
}
startTime = time.Now().UTC()
err := input.readS3File(runner, &deliverer, &splitterRunner, s3Key)
atomic.AddInt64(&input.processFileCount, 1)
leftovers := splitterRunner.GetRemainingData()
lenLeftovers := len(leftovers)
if lenLeftovers > 0 {
atomic.AddInt64(&input.processFileDiscardedBytes, int64(lenLeftovers))
runner.LogError(fmt.Errorf("Trailing data, possible corruption: %d bytes left in stream at EOF: %s", lenLeftovers, s3Key))
}
if err != nil && err != io.EOF {
runner.LogError(fmt.Errorf("Error reading %s: %s", s3Key, err))
atomic.AddInt64(&input.processFileFailures, 1)
continue
}
duration = time.Now().UTC().Sub(startTime).Seconds()
runner.LogMessage(fmt.Sprintf("Successfully fetched %s in %.2fs ", s3Key, duration))
case <-input.stop:
for _ = range input.listChan {
// Drain the channel without processing the files.
// Technically the S3Iterator can still add one back on to the
// channel but this ensures there is room so it won't block.
}
ok = false
}
}
wg.Done()
}
func (input *S3OffsetInput) fetcher(runner pipeline.InputRunner, wg *sync.WaitGroup, workerId uint32) {
var (
loc MessageLocation
startTime time.Time
duration float64
headers map[string][]string
record []byte
err error
)
headers = map[string][]string{
"Range": []string{""},
}
fetcherName := fmt.Sprintf("S3Reader%d", workerId)
deliverer := runner.NewDeliverer(fetcherName)
defer deliverer.Done()
splitterRunner := runner.NewSplitterRunner(fetcherName)
ok := true
for ok {
select {
case loc, ok = <-input.offsetChan:
if !ok {
// Channel is closed => we're shutting down, exit cleanly.
runner.LogMessage("Fetcher all done! shutting down.")
break
}
startTime = time.Now().UTC()
// Read one message from the given location
headers["Range"][0] = fmt.Sprintf("bytes=%d-%d", loc.Offset, loc.Offset+loc.Length-1)
atomic.AddInt64(&input.processMessageCount, 1)
atomic.AddInt64(&input.processMessageBytes, int64(loc.Length))
for attempt := uint32(1); attempt <= input.S3Retries; attempt++ {
record, err = getClientRecord(input.bucket, &loc, headers)
if err != nil {
runner.LogMessage(fmt.Sprintf("Error #%d fetching %s @ %d+%d: %s\n", attempt, loc.Key, loc.Offset, loc.Length, err))
} else {
break
}
}
if err != nil {
atomic.AddInt64(&input.processMessageFailures, 1)
continue
}
splitterRunner.DeliverRecord(record, deliverer)
duration = time.Now().UTC().Sub(startTime).Seconds()
runner.LogMessage(fmt.Sprintf("Successfully fetched %s in %.2fs ", loc.Key, duration))
case <-input.stop:
runner.LogMessage("Stopping fetcher...")
for _ = range input.offsetChan {
// Drain the channel without processing anything.
}
ok = false
}
}
wg.Done()
}