func (r *Registrar) fetchState(filePath string, fileInfo os.FileInfo) (int64, bool) {
// Check if there is a state for this file
lastState, isFound := r.GetFileState(filePath)
if isFound && input.IsSameFile(filePath, fileInfo) {
logp.Debug("registar", "Same file as before found. Fetch the state and persist it.")
// We're resuming - throw the last state back downstream so we resave it
// And return the offset - also force harvest in case the file is old and we're about to skip it
r.Persist <- lastState
return lastState.Offset, true
}
if previous, err := r.getPreviousFile(filePath, fileInfo); err == nil {
// File has rotated between shutdown and startup
// We return last state downstream, with a modified event source with the new file name
// And return the offset - also force harvest in case the file is old and we're about to skip it
logp.Info("Detected rename of a previously harvested file: %s -> %s", previous, filePath)
lastState, _ := r.GetFileState(previous)
lastState.Source = &filePath
r.Persist <- lastState
return lastState.Offset, true
}
if isFound {
logp.Info("Not resuming rotated file: %s", filePath)
}
logp.Info("prospector", "New file. Start reading from the beginning: %s", filePath)
// New file so just start from the beginning
return 0, false
}
// handleReadlineError handles error which are raised during reading file.
//
// If error is EOF, it will check for:
// * File truncated
// * Older then ignore_older
// * General file error
//
// If none of the above cases match, no error will be returned and file is kept open
//
// In case of a general error, the error itself is returned
func (h *Harvester) handleReadlineError(lastTimeRead time.Time, err error) error {
if err != io.EOF || !h.file.Continuable() {
logp.Err("Unexpected state reading from %s; error: %s", h.Path, err)
return err
}
// Refetch fileinfo to check if the file was truncated or disappeared.
// Errors if the file was removed/rotated after reading and before
// calling the stat function
info, statErr := h.file.Stat()
if statErr != nil {
logp.Err("Unexpected error reading from %s; error: %s", h.Path, statErr)
return statErr
}
// Handle fails if file was truncated
if info.Size() < h.Offset {
seeker, ok := h.file.(io.Seeker)
if !ok {
logp.Err("Can not seek source")
return err
}
logp.Debug("harvester", "File was truncated as offset (%s) > size (%s). Begin reading file from offset 0: %s", h.Offset, info.Size(), h.Path)
h.Offset = 0
seeker.Seek(h.Offset, os.SEEK_SET)
return nil
}
age := time.Since(lastTimeRead)
if age > h.ProspectorConfig.IgnoreOlderDuration {
// If the file hasn't change for longer the ignore_older, harvester stops
// and file handle will be closed.
return fmt.Errorf("Stop harvesting as file is older then ignore_older: %s; Last change was: %s ", h.Path, age)
}
if h.Config.ForceCloseFiles {
// Check if the file name exists (see #93)
_, statErr := os.Stat(h.file.Name())
// Error means file does not exist. If no error, check if same file. If not close as rotated.
if statErr != nil || !input.IsSameFile(h.file.Name(), info) {
logp.Info("Force close file: %s; error: %s", h.Path, statErr)
// Return directly on windows -> file is closing
return fmt.Errorf("Force closing file: %s", h.Path)
}
}
if err != io.EOF {
logp.Err("Unexpected state reading from %s; error: %s", h.Path, err)
}
logp.Debug("harvester", "End of file reached: %s; Backoff now.", h.Path)
// Do nothing in case it is just EOF, keep reading the file after backing off
h.backOff()
return nil
}
func (r *logFileReader) Read(buf []byte) (int, error) {
if r.truncated {
var offset int64
if seeker, ok := r.fs.(io.Seeker); ok {
var err error
offset, err = seeker.Seek(0, os.SEEK_CUR)
if err != nil {
return 0, err
}
}
r.offset = offset
r.truncated = false
}
for {
n, err := r.fs.Read(buf)
if n > 0 {
r.offset += int64(n)
r.lastTimeRead = time.Now()
}
if err == nil {
// reset backoff
r.backoff = r.config.backoffDuration
return n, nil
}
continuable := r.fs.Continuable()
if err == io.EOF && !continuable {
logp.Info("Reached end of file: %s", r.fs.Name())
return n, err
}
if err != io.EOF || !continuable {
logp.Err("Unexpected state reading from %s; error: %s", r.fs.Name(), err)
return n, err
}
// Refetch fileinfo to check if the file was truncated or disappeared.
// Errors if the file was removed/rotated after reading and before
// calling the stat function
info, statErr := r.fs.Stat()
if statErr != nil {
logp.Err("Unexpected error reading from %s; error: %s", r.fs.Name(), statErr)
return n, statErr
}
// handle fails if file was truncated
if info.Size() < r.offset {
logp.Debug("harvester",
"File was truncated as offset (%s) > size (%s). Begin reading file from offset 0: %s",
r.offset, info.Size(), r.fs.Name())
r.truncated = true
return n, errFileTruncate
}
age := time.Since(r.lastTimeRead)
if age > r.config.maxInactive {
// If the file hasn't change for longer then maxInactive, harvester stops
// and file handle will be closed.
return n, errInactive
}
if r.config.forceClose {
// Check if the file name exists (see #93)
_, statErr := os.Stat(r.fs.Name())
// Error means file does not exist. If no error, check if same file. If
// not close as rotated.
if statErr != nil || !input.IsSameFile(r.fs.Name(), info) {
logp.Info("Force close file: %s; error: %s", r.fs.Name(), statErr)
// Return directly on windows -> file is closing
return n, errForceClose
}
}
if err != io.EOF {
logp.Err("Unexpected state reading from %s; error: %s", r.fs.Name(), err)
}
logp.Debug("harvester", "End of file reached: %s; Backoff now.", r.fs.Name())
buf = buf[n:]
if len(buf) == 0 {
return n, nil
}
r.wait()
}
}