Exemple #1
0
func (r *Registrar) fetchState(filePath string, fileInfo os.FileInfo) (int64, bool) {

	// Check if there is a state for this file
	lastState, isFound := r.GetFileState(filePath)

	if isFound && input.IsSameFile(filePath, fileInfo) {
		logp.Debug("registar", "Same file as before found. Fetch the state and persist it.")
		// We're resuming - throw the last state back downstream so we resave it
		// And return the offset - also force harvest in case the file is old and we're about to skip it
		r.Persist <- lastState
		return lastState.Offset, true
	}

	if previous, err := r.getPreviousFile(filePath, fileInfo); err == nil {
		// File has rotated between shutdown and startup
		// We return last state downstream, with a modified event source with the new file name
		// And return the offset - also force harvest in case the file is old and we're about to skip it
		logp.Info("Detected rename of a previously harvested file: %s -> %s", previous, filePath)

		lastState, _ := r.GetFileState(previous)
		lastState.Source = &filePath
		r.Persist <- lastState
		return lastState.Offset, true
	}

	if isFound {
		logp.Info("Not resuming rotated file: %s", filePath)
	}

	logp.Info("prospector", "New file. Start reading from the beginning: %s", filePath)

	// New file so just start from the beginning
	return 0, false
}
Exemple #2
0
// handleReadlineError handles error which are raised during reading file.
//
// If error is EOF, it will check for:
// * File truncated
// * Older then ignore_older
// * General file error
//
// If none of the above cases match, no error will be returned and file is kept open
//
// In case of a general error, the error itself is returned
func (h *Harvester) handleReadlineError(lastTimeRead time.Time, err error) error {
	if err != io.EOF || !h.file.Continuable() {
		logp.Err("Unexpected state reading from %s; error: %s", h.Path, err)
		return err
	}

	// Refetch fileinfo to check if the file was truncated or disappeared.
	// Errors if the file was removed/rotated after reading and before
	// calling the stat function
	info, statErr := h.file.Stat()
	if statErr != nil {
		logp.Err("Unexpected error reading from %s; error: %s", h.Path, statErr)
		return statErr
	}

	// Handle fails if file was truncated
	if info.Size() < h.Offset {
		seeker, ok := h.file.(io.Seeker)
		if !ok {
			logp.Err("Can not seek source")
			return err
		}

		logp.Debug("harvester", "File was truncated as offset (%s) > size (%s). Begin reading file from offset 0: %s", h.Offset, info.Size(), h.Path)

		h.Offset = 0
		seeker.Seek(h.Offset, os.SEEK_SET)
		return nil
	}

	age := time.Since(lastTimeRead)
	if age > h.ProspectorConfig.IgnoreOlderDuration {
		// If the file hasn't change for longer the ignore_older, harvester stops
		// and file handle will be closed.
		return fmt.Errorf("Stop harvesting as file is older then ignore_older: %s; Last change was: %s ", h.Path, age)
	}

	if h.Config.ForceCloseFiles {
		// Check if the file name exists (see #93)
		_, statErr := os.Stat(h.file.Name())

		// Error means file does not exist. If no error, check if same file. If not close as rotated.
		if statErr != nil || !input.IsSameFile(h.file.Name(), info) {
			logp.Info("Force close file: %s; error: %s", h.Path, statErr)
			// Return directly on windows -> file is closing
			return fmt.Errorf("Force closing file: %s", h.Path)
		}
	}

	if err != io.EOF {
		logp.Err("Unexpected state reading from %s; error: %s", h.Path, err)
	}

	logp.Debug("harvester", "End of file reached: %s; Backoff now.", h.Path)

	// Do nothing in case it is just EOF, keep reading the file after backing off
	h.backOff()
	return nil
}
Exemple #3
0
func (r *logFileReader) Read(buf []byte) (int, error) {
	if r.truncated {
		var offset int64
		if seeker, ok := r.fs.(io.Seeker); ok {
			var err error
			offset, err = seeker.Seek(0, os.SEEK_CUR)
			if err != nil {
				return 0, err
			}
		}
		r.offset = offset
		r.truncated = false
	}

	for {
		n, err := r.fs.Read(buf)
		if n > 0 {
			r.offset += int64(n)
			r.lastTimeRead = time.Now()
		}
		if err == nil {
			// reset backoff
			r.backoff = r.config.backoffDuration
			return n, nil
		}

		continuable := r.fs.Continuable()
		if err == io.EOF && !continuable {
			logp.Info("Reached end of file: %s", r.fs.Name())
			return n, err
		}

		if err != io.EOF || !continuable {
			logp.Err("Unexpected state reading from %s; error: %s", r.fs.Name(), err)
			return n, err
		}

		// Refetch fileinfo to check if the file was truncated or disappeared.
		// Errors if the file was removed/rotated after reading and before
		// calling the stat function
		info, statErr := r.fs.Stat()
		if statErr != nil {
			logp.Err("Unexpected error reading from %s; error: %s", r.fs.Name(), statErr)
			return n, statErr
		}

		// handle fails if file was truncated
		if info.Size() < r.offset {
			logp.Debug("harvester",
				"File was truncated as offset (%s) > size (%s). Begin reading file from offset 0: %s",
				r.offset, info.Size(), r.fs.Name())
			r.truncated = true
			return n, errFileTruncate
		}

		age := time.Since(r.lastTimeRead)
		if age > r.config.maxInactive {
			// If the file hasn't change for longer then maxInactive, harvester stops
			// and file handle will be closed.
			return n, errInactive
		}

		if r.config.forceClose {
			// Check if the file name exists (see #93)
			_, statErr := os.Stat(r.fs.Name())

			// Error means file does not exist. If no error, check if same file. If
			// not close as rotated.
			if statErr != nil || !input.IsSameFile(r.fs.Name(), info) {
				logp.Info("Force close file: %s; error: %s", r.fs.Name(), statErr)
				// Return directly on windows -> file is closing
				return n, errForceClose
			}
		}

		if err != io.EOF {
			logp.Err("Unexpected state reading from %s; error: %s", r.fs.Name(), err)
		}

		logp.Debug("harvester", "End of file reached: %s; Backoff now.", r.fs.Name())
		buf = buf[n:]
		if len(buf) == 0 {
			return n, nil
		}
		r.wait()
	}
}