示例#1
0
func (lsi *LogstreamInput) Run(ir p.InputRunner, h p.PluginHelper, stopChan chan chan bool,
	deliverer p.Deliverer, sRunner p.SplitterRunner) {

	if !sRunner.UseMsgBytes() {
		sRunner.SetPackDecorator(lsi.packDecorator)
	}

	lsi.ir = ir
	lsi.stopChan = stopChan
	lsi.deliverer = deliverer
	lsi.sRunner = sRunner
	var err error

	// Check for more data interval
	interval, _ := time.ParseDuration("250ms")
	tick := time.Tick(interval)

	ok := true
	for ok {
		// Clear our error
		err = nil

		// Attempt to read and deliver as many as we can.
		err = lsi.deliverRecords()
		// Save our position if the stream hasn't done so for us.
		if err != io.EOF {
			lsi.stream.SavePosition()
		}
		lsi.recordCount = 0

		if err != nil && err != io.EOF {
			ir.LogError(err)
		}

		// Did our parser func get stopped?
		if lsi.stopped != nil {
			ok = false
			continue
		}

		// Wait for our next interval, stop if needed
		select {
		case lsi.stopped = <-stopChan:
			ok = false
		case <-tick:
			continue
		}
	}
	close(lsi.stopped)
	deliverer.Done()
}
示例#2
0
func splitStream(ir p.InputRunner, sRunner p.SplitterRunner, r io.ReadCloser) error {
	var (
		record       []byte
		longRecord   []byte
		err          error
		deliver      bool
		nullSplitter bool
	)
	// If we're using a NullSplitter we want to make sure we capture the
	// entire HTTP request or response body and not be subject to what we get
	// from a single Read() call.
	if _, ok := sRunner.Splitter().(*p.NullSplitter); ok {
		nullSplitter = true
	}
	for err == nil {
		deliver = true
		_, record, err = sRunner.GetRecordFromStream(r)
		if err == io.ErrShortBuffer {
			if sRunner.KeepTruncated() {
				err = fmt.Errorf("record exceeded MAX_RECORD_SIZE %d and was truncated",
					message.MAX_RECORD_SIZE)
			} else {
				deliver = false
				err = fmt.Errorf("record exceeded MAX_RECORD_SIZE %d and was dropped",
					message.MAX_RECORD_SIZE)
			}
			ir.LogError(err)
			err = nil // non-fatal, keep going
		} else if sRunner.IncompleteFinal() && err == io.EOF && len(record) == 0 {
			record = sRunner.GetRemainingData()
		}
		if len(record) > 0 && deliver {
			if nullSplitter {
				// Concatenate all the records until EOF. This should be safe
				// b/c NullSplitter means FindRecord will always return the
				// full buffer contents, we don't have to worry about
				// GetRecordFromStream trying to append multiple reads to a
				// single record and triggering an io.ErrShortBuffer error.
				longRecord = append(longRecord, record...)
			} else {
				sRunner.DeliverRecord(record, nil)
			}
		}
	}
	r.Close()
	if err == io.EOF && nullSplitter && len(longRecord) > 0 {
		sRunner.DeliverRecord(longRecord, nil)
	}
	return err
}