// Creates deliverer and stop channel and starts the provided LogstreamInput. func (li *LogstreamerInput) startLogstreamInput(logstream *LogstreamInput, i int, ir p.InputRunner, h p.PluginHelper) { stop := make(chan chan bool, 1) token := strconv.Itoa(i) deliverer := ir.NewDeliverer(token) sRunner := ir.NewSplitterRunner(token) li.stopLogstreamChans = append(li.stopLogstreamChans, stop) go logstream.Run(ir, h, stop, deliverer, sRunner) }
func (input *S3SplitFileInput) fetcher(runner pipeline.InputRunner, wg *sync.WaitGroup, workerId uint32) { var ( s3Key string startTime time.Time duration float64 ) fetcherName := fmt.Sprintf("S3Reader%d", workerId) deliverer := runner.NewDeliverer(fetcherName) defer deliverer.Done() splitterRunner := runner.NewSplitterRunner(fetcherName) ok := true for ok { select { case s3Key, ok = <-input.listChan: if !ok { // Channel is closed => we're shutting down, exit cleanly. // runner.LogMessage("Fetcher all done! shutting down.") break } startTime = time.Now().UTC() err := input.readS3File(runner, &deliverer, &splitterRunner, s3Key) atomic.AddInt64(&input.processFileCount, 1) leftovers := splitterRunner.GetRemainingData() lenLeftovers := len(leftovers) if lenLeftovers > 0 { atomic.AddInt64(&input.processFileDiscardedBytes, int64(lenLeftovers)) runner.LogError(fmt.Errorf("Trailing data, possible corruption: %d bytes left in stream at EOF: %s", lenLeftovers, s3Key)) } if err != nil && err != io.EOF { runner.LogError(fmt.Errorf("Error reading %s: %s", s3Key, err)) atomic.AddInt64(&input.processFileFailures, 1) continue } duration = time.Now().UTC().Sub(startTime).Seconds() runner.LogMessage(fmt.Sprintf("Successfully fetched %s in %.2fs ", s3Key, duration)) case <-input.stop: for _ = range input.listChan { // Drain the channel without processing the files. // Technically the S3Iterator can still add one back on to the // channel but this ensures there is room so it won't block. } ok = false } } wg.Done() }
func (input *S3OffsetInput) fetcher(runner pipeline.InputRunner, wg *sync.WaitGroup, workerId uint32) { var ( loc MessageLocation startTime time.Time duration float64 headers map[string][]string record []byte err error ) headers = map[string][]string{ "Range": []string{""}, } fetcherName := fmt.Sprintf("S3Reader%d", workerId) deliverer := runner.NewDeliverer(fetcherName) defer deliverer.Done() splitterRunner := runner.NewSplitterRunner(fetcherName) ok := true for ok { select { case loc, ok = <-input.offsetChan: if !ok { // Channel is closed => we're shutting down, exit cleanly. runner.LogMessage("Fetcher all done! shutting down.") break } startTime = time.Now().UTC() // Read one message from the given location headers["Range"][0] = fmt.Sprintf("bytes=%d-%d", loc.Offset, loc.Offset+loc.Length-1) atomic.AddInt64(&input.processMessageCount, 1) atomic.AddInt64(&input.processMessageBytes, int64(loc.Length)) for attempt := uint32(1); attempt <= input.S3Retries; attempt++ { record, err = getClientRecord(input.bucket, &loc, headers) if err != nil { runner.LogMessage(fmt.Sprintf("Error #%d fetching %s @ %d+%d: %s\n", attempt, loc.Key, loc.Offset, loc.Length, err)) } else { break } } if err != nil { atomic.AddInt64(&input.processMessageFailures, 1) continue } splitterRunner.DeliverRecord(record, deliverer) duration = time.Now().UTC().Sub(startTime).Seconds() runner.LogMessage(fmt.Sprintf("Successfully fetched %s in %.2fs ", loc.Key, duration)) case <-input.stop: runner.LogMessage("Stopping fetcher...") for _ = range input.offsetChan { // Drain the channel without processing anything. } ok = false } } wg.Done() }