Example #1
0
// NewLineSource creates a new LineSource from input reader by applying
// the given codec.
func NewLineSource(
	in io.Reader,
	codec encoding.Encoding,
	bufferSize int,
) (LineSource, error) {
	r, err := encoding.NewLineReader(in, codec, bufferSize)
	return LineSource{r}, err
}
Example #2
0
func TestReadLine(t *testing.T) {

	absPath, err := filepath.Abs("../tests/files/logs/")
	// All files starting with tmp are ignored
	logFile := absPath + "/tmp" + strconv.Itoa(rand.Int()) + ".log"

	assert.NotNil(t, absPath)
	assert.Nil(t, err)

	if err != nil {
		t.Fatalf("Error creating the absolute path: %s", absPath)
	}

	file, err := os.Create(logFile)
	defer file.Close()
	defer os.Remove(logFile)

	assert.Nil(t, err)
	assert.NotNil(t, file)

	firstLineString := "9Characte\n"
	secondLineString := "This is line 2\n"

	length, err := file.WriteString(firstLineString)
	assert.Nil(t, err)
	assert.NotNil(t, length)

	length, err = file.WriteString(secondLineString)
	assert.Nil(t, err)
	assert.NotNil(t, length)

	file.Sync()

	// Open file for reading
	readFile, err := os.Open(logFile)
	defer readFile.Close()
	assert.Nil(t, err)

	h := Harvester{}
	assert.NotNil(t, h)

	// Read only 10 bytes which is not the end of the file
	timedIn := newTimedReader(readFile)
	codec, _ := encoding.Plain(file)
	reader, _ := encoding.NewLineReader(timedIn, codec, 100)

	// Read third line
	text, bytesread, err := readLine(reader, &timedIn.lastReadTime)

	assert.Nil(t, err)
	assert.Equal(t, text, firstLineString[0:len(firstLineString)-1])
	assert.Equal(t, bytesread, len(firstLineString))

	// read second line
	text, bytesread, err = readLine(reader, &timedIn.lastReadTime)

	assert.Equal(t, text, secondLineString[0:len(secondLineString)-1])
	assert.Equal(t, bytesread, len(secondLineString))
	assert.Nil(t, err)

	// Read third line, which doesn't exist
	text, bytesread, err = readLine(reader, &timedIn.lastReadTime)
	assert.Equal(t, "", text)
	assert.Equal(t, bytesread, 0)
	assert.Equal(t, err, io.EOF)
}
Example #3
0
// Log harvester reads files line by line and sends events to the defined output
func (h *Harvester) Harvest() {

	defer func() {
		// On completion, push offset so we can continue where we left off if we relaunch on the same file
		h.Stat.Return <- h.Offset

		// Make sure file is closed as soon as harvester exits
		// If file was never properly opened, it can't be closed
		if h.file != nil {
			h.file.Close()
			logp.Debug("harvester", "Closing file: %s", h.Path)
		}
	}()

	enc, err := h.open()
	if err != nil {
		logp.Err("Stop Harvesting. Unexpected file opening error: %s", err)
		return
	}

	info, err := h.file.Stat()
	if err != nil {
		logp.Err("Stop Harvesting. Unexpected file stat rror: %s", err)
		return
	}

	logp.Info("Harvester started for file: %s", h.Path)

	// TODO: NewLineReader uses additional buffering to deal with encoding and testing
	//       for new lines in input stream. Simple 8-bit based encodings, or plain
	//       don't require 'complicated' logic.
	timedIn := newTimedReader(h.file)
	reader, err := encoding.NewLineReader(timedIn, enc, h.Config.BufferSize)
	if err != nil {
		logp.Err("Stop Harvesting. Unexpected encoding line reader error: %s", err)
		return
	}

	// XXX: lastReadTime handling last time a full line was read only?
	//      timedReader provides timestamp some bytes have actually been read from file
	lastReadTime := time.Now()

	for {
		// Partial lines return error and are only read on completion
		text, bytesRead, err := readLine(reader, &timedIn.lastReadTime)

		if err != nil {

			// In case of err = io.EOF returns nil
			err = h.handleReadlineError(lastReadTime, err)

			// Return in case of error which leads to stopping harvester and closing file
			if err != nil {
				logp.Info("Read line error: %s", err)
				return
			}

			continue
		}

		lastReadTime = time.Now()

		// Reset Backoff
		h.backoff = h.Config.BackoffDuration

		if h.shouldExportLine(text) {

			// Sends text to spooler
			event := &input.FileEvent{
				ReadTime:     lastReadTime,
				Source:       &h.Path,
				InputType:    h.Config.InputType,
				DocumentType: h.Config.DocumentType,
				Offset:       h.Offset,
				Bytes:        bytesRead,
				Text:         &text,
				Fields:       &h.Config.Fields,
				Fileinfo:     &info,
			}

			event.SetFieldsUnderRoot(h.Config.FieldsUnderRoot)
			h.SpoolerChan <- event // ship the new event downstream
		}

		// Set Offset
		h.Offset += int64(bytesRead) // Update offset if complete line has been processed
	}
}