// Log harvester reads files line by line and sends events to the defined output
func (h *Harvester) Harvest() {
enc, err := h.open()
if err != nil {
logp.Err("Stop Harvesting. Unexpected Error: %s", err)
}
defer func() {
// On completion, push offset so we can continue where we left off if we relaunch on the same file
h.Stat.Return <- h.Offset
// Make sure file is closed as soon as harvester exits
h.file.Close()
}()
if err != nil {
logp.Err("Stop Harvesting. Unexpected Error: %s", err)
return
}
info, err := h.file.Stat()
if err != nil {
logp.Err("Stop Harvesting. Unexpected Error: %s", err)
return
}
logp.Info("Harvester started for file: %s", h.Path)
// TODO: NewLineReader uses additional buffering to deal with encoding and testing
// for new lines in input stream. Simple 8-bit based encodings, or plain
// don't require 'complicated' logic.
timedIn := newTimedReader(h.file)
reader, err := encoding.NewLineReader(timedIn, enc, h.Config.BufferSize)
if err != nil {
logp.Err("Stop Harvesting. Unexpected Error: %s", err)
return
}
// XXX: lastReadTime handling last time a full line was read only?
// timedReader provides timestamp some bytes have actually been read from file
lastReadTime := time.Now()
for {
// Partial lines return error and are only read on completion
text, bytesRead, err := readLine(reader, &timedIn.lastReadTime)
if err != nil {
// In case of err = io.EOF returns nil
err = h.handleReadlineError(lastReadTime, err)
// Return in case of error which leads to stopping harvester and closing file
if err != nil {
logp.Info("Read line error: %s", err)
return
}
continue
}
lastReadTime = time.Now()
// Reset Backoff
h.backoff = h.Config.BackoffDuration
// Sends text to spooler
event := &input.FileEvent{
ReadTime: lastReadTime,
Source: &h.Path,
InputType: h.Config.InputType,
DocumentType: h.Config.DocumentType,
Offset: h.Offset,
Bytes: bytesRead,
Text: &text,
Fields: &h.Config.Fields,
Fileinfo: &info,
}
h.Offset += int64(bytesRead) // Update offset if complete line has been processed
event.SetFieldsUnderRoot(h.Config.FieldsUnderRoot)
h.SpoolerChan <- event // ship the new event downstream
}
}
func TestReadLine(t *testing.T) {
absPath, err := filepath.Abs("../tests/files/logs/")
// All files starting with tmp are ignored
logFile := absPath + "/tmp" + strconv.Itoa(rand.Int()) + ".log"
assert.NotNil(t, absPath)
assert.Nil(t, err)
if err != nil {
t.Fatalf("Error creating the absolute path: %s", absPath)
}
file, err := os.Create(logFile)
defer file.Close()
defer os.Remove(logFile)
assert.Nil(t, err)
assert.NotNil(t, file)
firstLineString := "9Characte\n"
secondLineString := "This is line 2\n"
length, err := file.WriteString(firstLineString)
assert.Nil(t, err)
assert.NotNil(t, length)
length, err = file.WriteString(secondLineString)
assert.Nil(t, err)
assert.NotNil(t, length)
file.Sync()
// Open file for reading
readFile, err := os.Open(logFile)
defer readFile.Close()
assert.Nil(t, err)
h := Harvester{}
assert.NotNil(t, h)
// Read only 10 bytes which is not the end of the file
timedIn := newTimedReader(readFile)
codec, _ := encoding.Plain(file)
reader, _ := encoding.NewLineReader(timedIn, codec, 100)
// Read third line
text, bytesread, err := readLine(reader, &timedIn.lastReadTime)
assert.Nil(t, err)
assert.Equal(t, text, firstLineString[0:len(firstLineString)-1])
assert.Equal(t, bytesread, len(firstLineString))
// read second line
text, bytesread, err = readLine(reader, &timedIn.lastReadTime)
assert.Equal(t, text, secondLineString[0:len(secondLineString)-1])
assert.Equal(t, bytesread, len(secondLineString))
assert.Nil(t, err)
// Read third line, which doesn't exist
text, bytesread, err = readLine(reader, &timedIn.lastReadTime)
assert.Equal(t, "", text)
assert.Equal(t, bytesread, 0)
assert.Equal(t, err, io.EOF)
}
