Esempio n. 1
0
// Scan starts a scanGlob for each provided path/glob
func (p *ProspectorLog) scan() {

	newlastscan := time.Now()

	p.missingFiles = map[string]os.FileInfo{}

	// Now let's do one quick scan to pick up new files
	for file, fileinfo := range p.getFiles() {

		logp.Debug("prospector", "Check file for harvesting: %s", file)

		newFile := input.NewFile(fileinfo)

		// Check the current info against p.prospectorinfo[file]
		lastinfo, isKnown := p.harvesterStats[file]

		oldFile := input.NewFile(lastinfo.Fileinfo)

		// Create a new prospector info with the stat info for comparison
		newInfo := input.NewFileStat(newFile.FileInfo, p.iteration)

		// Init harvester with info
		h, err := p.Prospector.createHarvester(file, newInfo)
		if err != nil {
			logp.Err("Error initializing harvester: %v", err)
			continue
		}

		// Conditions for starting a new harvester:
		// - file path hasn't been seen before
		// - the file's inode or device changed
		if !isKnown {
			p.checkNewFile(h)
		} else {
			h.Stat.Continue(&lastinfo)
			p.checkExistingFile(h, &newFile, &oldFile)
		}

		// Track the stat data for this file for later comparison to check for
		// rotation/etc
		p.harvesterStats[h.Path] = *h.Stat
	}

	p.lastscan = newlastscan
}
Esempio n. 2
0
func (p *ProspectorLog) Init() {
	logp.Debug("prospector", "exclude_files: %s", p.config.ExcludeFiles)

	logp.Info("Load previous states from registry into memory")

	for path, fileinfo := range p.getFiles() {

		// Check for each path found, if there is a previous state
		offset := p.Prospector.registrar.fetchState(path, fileinfo)

		// Offset found -> skip to previous state
		if offset > 0 {
			fs := input.NewFileStat(fileinfo, 0)
			fs.Skip(offset)
			p.harvesterStats[path] = *fs
		}
	}

	logp.Info("Previous states loaded: %v", len(p.harvesterStats))
}
Esempio n. 3
0
// Scans the specific path which can be a glob (/**/**/*.log)
// For all found files it is checked if a harvester should be started
func (p *ProspectorLog) scanGlob(glob string) {

	logp.Debug("prospector", "scan path %s", glob)

	// Evaluate the path as a wildcards/shell glob
	matches, err := filepath.Glob(glob)
	if err != nil {
		logp.Debug("prospector", "glob(%s) failed: %v", glob, err)
		return
	}

	p.missingFiles = map[string]os.FileInfo{}

	// Check any matched files to see if we need to start a harvester
	for _, file := range matches {
		logp.Debug("prospector", "Check file for harvesting: %s", file)

		// check if the file is in the exclude_files list
		if p.isFileExcluded(file) {
			logp.Debug("prospector", "Exclude file: %s", file)
			continue
		}

		// Stat the file, following any symlinks.
		fileinfo, err := os.Stat(file)
		if err != nil {
			logp.Debug("prospector", "stat(%s) failed: %s", file, err)
			continue
		}

		newFile := input.NewFile(fileinfo)

		if newFile.FileInfo.IsDir() {
			logp.Debug("prospector", "Skipping directory: %s", file)
			continue
		}

		// Check the current info against p.prospectorinfo[file]
		lastinfo, isKnown := p.harvesterStats[file]

		oldFile := input.NewFile(lastinfo.Fileinfo)

		// Create a new prospector info with the stat info for comparison
		newInfo := input.NewFileStat(newFile.FileInfo, p.iteration)

		// Init harvester with info
		h, err := p.Prospector.AddHarvester(file, newInfo)

		if err != nil {
			logp.Err("Error initializing harvester: %v", err)
			continue
		}

		// Conditions for starting a new harvester:
		// - file path hasn't been seen before
		// - the file's inode or device changed
		if !isKnown {
			p.checkNewFile(h)
		} else {
			h.Stat.Continue(&lastinfo)
			p.checkExistingFile(h, &newFile, &oldFile)
		}

		// Track the stat data for this file for later comparison to check for
		// rotation/etc
		p.harvesterStats[h.Path] = *h.Stat
	}
}