// Scans the specific path which can be a glob (/**/**/*.log)
// For all found files it is checked if a harvester should be started
func (p *Prospector) scan(path string, output chan *input.FileEvent) {
logp.Debug("prospector", "scan path %s", path)
// Evaluate the path as a wildcards/shell glob
matches, err := filepath.Glob(path)
if err != nil {
logp.Debug("prospector", "glob(%s) failed: %v", path, err)
return
}
p.missingFiles = map[string]os.FileInfo{}
// Check any matched files to see if we need to start a harvester
for _, file := range matches {
logp.Debug("prospector", "Check file for harvesting: %s", file)
// Stat the file, following any symlinks.
fileinfo, err := os.Stat(file)
// TODO(sissel): check err
if err != nil {
logp.Debug("prospector", "stat(%s) failed: %s", file, err)
continue
}
newFile := input.File{
FileInfo: fileinfo,
}
if newFile.FileInfo.IsDir() {
logp.Debug("prospector", "Skipping directory: %s", file)
continue
}
// Check the current info against p.prospectorinfo[file]
lastinfo, isKnown := p.prospectorList[file]
oldFile := input.File{
FileInfo: lastinfo.Fileinfo,
}
// Create a new prospector info with the stat info for comparison
newInfo := harvester.NewFileStat(newFile.FileInfo, p.iteration)
// Conditions for starting a new harvester:
// - file path hasn't been seen before
// - the file's inode or device changed
if !isKnown {
p.checkNewFile(newInfo, file, output)
} else {
newInfo.Continue(&lastinfo)
p.checkExistingFile(newInfo, &newFile, &oldFile, file, output)
}
// Track the stat data for this file for later comparison to check for
// rotation/etc
p.prospectorList[file] = *newInfo
} // for each file matched by the glob
}
// Scans the specific path which can be a glob (/**/**/*.log)
// For all found files it is checked if a harvester should be started
func (p *Prospector) scan(path string, output chan *input.FileEvent) {
logp.Debug("prospector", "scan path %s", path)
// Evaluate the path as a wildcards/shell glob
matches, err := filepath.Glob(path)
if err != nil {
logp.Debug("prospector", "glob(%s) failed: %v", path, err)
return
}
p.missingFiles = map[string]os.FileInfo{}
p.oldStates = map[string]oldState{}
// Check any matched files to see if we need to start a harvester
for _, file := range matches {
logp.Debug("prospector", "Check file for harvesting: %s", file)
// check if the file is in the exclude_files list
if p.isFileExcluded(file) {
logp.Debug("prospector", "Exclude file: %s", file)
continue
}
// Stat the file, following any symlinks.
fileinfo, err := os.Stat(file)
// TODO(sissel): check err
if err != nil {
logp.Debug("prospector", "stat(%s) failed: %s", file, err)
continue
}
if fileinfo.IsDir() {
logp.Debug("prospector", "Skipping directory: %s", file)
continue
}
// Check the current info against p.prospectorinfo[file]
lastinfo, isKnown := p.prospectorList[file]
// Create a new prospector info with the stat info for comparison
newInfo := harvester.NewFileStat(fileinfo, p.iteration)
// Call crawler if there if there exists a state for the given file
offset, resuming := p.registrar.fetchState(file, newInfo.Fileinfo)
p.oldStates[file] = oldState{
fileinfo: fileinfo,
lastinfo: lastinfo,
isKnown: isKnown,
newInfo: newInfo,
offset: offset,
resuming: resuming,
}
}
for file, oldState := range p.oldStates {
newFile := input.File{
FileInfo: oldState.fileinfo,
}
oldFile := input.File{
FileInfo: oldState.lastinfo.Fileinfo,
}
// Conditions for starting a new harvester:
// - file path hasn't been seen before
// - the file's inode or device changed
if !oldState.isKnown {
p.checkNewFile(oldState.newInfo, file, output, oldState)
} else {
oldState.newInfo.Continue(&oldState.lastinfo)
p.checkExistingFile(oldState.newInfo, &newFile, &oldFile, file, output, oldState)
}
// Track the stat data for this file for later comparison to check for
// rotation/etc
p.prospectorList[file] = *oldState.newInfo
} // for each file matched by the glob
}
// Scans the specific path which can be a glob (/**/**/*.log)
// For all found files it is checked if a harvester should be started
func (p ProspectorLog) scanGlob(glob string) {
logp.Debug("prospector", "scan path %s", glob)
// Evaluate the path as a wildcards/shell glob
matches, err := filepath.Glob(glob)
if err != nil {
logp.Debug("prospector", "glob(%s) failed: %v", glob, err)
return
}
p.missingFiles = map[string]os.FileInfo{}
// Check any matched files to see if we need to start a harvester
for _, file := range matches {
logp.Debug("prospector", "Check file for harvesting: %s", file)
// check if the file is in the exclude_files list
if p.isFileExcluded(file) {
logp.Debug("prospector", "Exclude file: %s", file)
continue
}
// Stat the file, following any symlinks.
fileinfo, err := os.Stat(file)
if err != nil {
logp.Debug("prospector", "stat(%s) failed: %s", file, err)
continue
}
newFile := input.NewFile(fileinfo)
if newFile.FileInfo.IsDir() {
logp.Debug("prospector", "Skipping directory: %s", file)
continue
}
// Check the current info against p.prospectorinfo[file]
lastinfo, isKnown := p.prospectorList[file]
oldFile := input.NewFile(lastinfo.Fileinfo)
// Create a new prospector info with the stat info for comparison
newInfo := harvester.NewFileStat(newFile.FileInfo, p.iteration)
// Init harvester with info
h, err := p.Prospector.AddHarvester(file, newInfo)
if err != nil {
logp.Err("Error initializing harvester: %v", err)
continue
}
// Conditions for starting a new harvester:
// - file path hasn't been seen before
// - the file's inode or device changed
if !isKnown {
p.checkNewFile(h)
} else {
h.Stat.Continue(&lastinfo)
p.checkExistingFile(h, &newFile, &oldFile)
}
// Track the stat data for this file for later comparison to check for
// rotation/etc
p.prospectorList[h.Path] = *h.Stat
}
}
