// Scan starts a scanGlob for each provided path/glob
func (p *ProspectorLog) scan() {
newlastscan := time.Now()
p.missingFiles = map[string]os.FileInfo{}
// Now let's do one quick scan to pick up new files
for file, fileinfo := range p.getFiles() {
logp.Debug("prospector", "Check file for harvesting: %s", file)
newFile := input.NewFile(fileinfo)
// Check the current info against p.prospectorinfo[file]
lastinfo, isKnown := p.harvesterStats[file]
oldFile := input.NewFile(lastinfo.Fileinfo)
// Create a new prospector info with the stat info for comparison
newInfo := input.NewFileStat(newFile.FileInfo, p.iteration)
// Init harvester with info
h, err := p.Prospector.createHarvester(file, newInfo)
if err != nil {
logp.Err("Error initializing harvester: %v", err)
continue
}
// Conditions for starting a new harvester:
// - file path hasn't been seen before
// - the file's inode or device changed
if !isKnown {
p.checkNewFile(h)
} else {
h.Stat.Continue(&lastinfo)
p.checkExistingFile(h, &newFile, &oldFile)
}
// Track the stat data for this file for later comparison to check for
// rotation/etc
p.harvesterStats[h.Path] = *h.Stat
}
p.lastscan = newlastscan
}
func (p *ProspectorLog) Init() {
logp.Debug("prospector", "exclude_files: %s", p.config.ExcludeFiles)
logp.Info("Load previous states from registry into memory")
for path, fileinfo := range p.getFiles() {
// Check for each path found, if there is a previous state
offset := p.Prospector.registrar.fetchState(path, fileinfo)
// Offset found -> skip to previous state
if offset > 0 {
fs := input.NewFileStat(fileinfo, 0)
fs.Skip(offset)
p.harvesterStats[path] = *fs
}
}
logp.Info("Previous states loaded: %v", len(p.harvesterStats))
}
// Scans the specific path which can be a glob (/**/**/*.log)
// For all found files it is checked if a harvester should be started
func (p *ProspectorLog) scanGlob(glob string) {
logp.Debug("prospector", "scan path %s", glob)
// Evaluate the path as a wildcards/shell glob
matches, err := filepath.Glob(glob)
if err != nil {
logp.Debug("prospector", "glob(%s) failed: %v", glob, err)
return
}
p.missingFiles = map[string]os.FileInfo{}
// Check any matched files to see if we need to start a harvester
for _, file := range matches {
logp.Debug("prospector", "Check file for harvesting: %s", file)
// check if the file is in the exclude_files list
if p.isFileExcluded(file) {
logp.Debug("prospector", "Exclude file: %s", file)
continue
}
// Stat the file, following any symlinks.
fileinfo, err := os.Stat(file)
if err != nil {
logp.Debug("prospector", "stat(%s) failed: %s", file, err)
continue
}
newFile := input.NewFile(fileinfo)
if newFile.FileInfo.IsDir() {
logp.Debug("prospector", "Skipping directory: %s", file)
continue
}
// Check the current info against p.prospectorinfo[file]
lastinfo, isKnown := p.harvesterStats[file]
oldFile := input.NewFile(lastinfo.Fileinfo)
// Create a new prospector info with the stat info for comparison
newInfo := input.NewFileStat(newFile.FileInfo, p.iteration)
// Init harvester with info
h, err := p.Prospector.AddHarvester(file, newInfo)
if err != nil {
logp.Err("Error initializing harvester: %v", err)
continue
}
// Conditions for starting a new harvester:
// - file path hasn't been seen before
// - the file's inode or device changed
if !isKnown {
p.checkNewFile(h)
} else {
h.Stat.Continue(&lastinfo)
p.checkExistingFile(h, &newFile, &oldFile)
}
// Track the stat data for this file for later comparison to check for
// rotation/etc
p.harvesterStats[h.Path] = *h.Stat
}
}