// Scans the specific path which can be a glob (/**/**/*.log) // For all found files it is checked if a harvester should be started func (p *Prospector) scan(path string, output chan *input.FileEvent) { logp.Debug("prospector", "scan path %s", path) // Evaluate the path as a wildcards/shell glob matches, err := filepath.Glob(path) if err != nil { logp.Debug("prospector", "glob(%s) failed: %v", path, err) return } p.missingFiles = map[string]os.FileInfo{} // Check any matched files to see if we need to start a harvester for _, file := range matches { logp.Debug("prospector", "Check file for harvesting: %s", file) // Stat the file, following any symlinks. fileinfo, err := os.Stat(file) // TODO(sissel): check err if err != nil { logp.Debug("prospector", "stat(%s) failed: %s", file, err) continue } newFile := input.File{ FileInfo: fileinfo, } if newFile.FileInfo.IsDir() { logp.Debug("prospector", "Skipping directory: %s", file) continue } // Check the current info against p.prospectorinfo[file] lastinfo, isKnown := p.prospectorList[file] oldFile := input.File{ FileInfo: lastinfo.Fileinfo, } // Create a new prospector info with the stat info for comparison newInfo := harvester.NewFileStat(newFile.FileInfo, p.iteration) // Conditions for starting a new harvester: // - file path hasn't been seen before // - the file's inode or device changed if !isKnown { p.checkNewFile(newInfo, file, output) } else { newInfo.Continue(&lastinfo) p.checkExistingFile(newInfo, &newFile, &oldFile, file, output) } // Track the stat data for this file for later comparison to check for // rotation/etc p.prospectorList[file] = *newInfo } // for each file matched by the glob }
// Scans the specific path which can be a glob (/**/**/*.log) // For all found files it is checked if a harvester should be started func (p *Prospector) scan(path string, output chan *input.FileEvent) { logp.Debug("prospector", "scan path %s", path) // Evaluate the path as a wildcards/shell glob matches, err := filepath.Glob(path) if err != nil { logp.Debug("prospector", "glob(%s) failed: %v", path, err) return } p.missingFiles = map[string]os.FileInfo{} p.oldStates = map[string]oldState{} // Check any matched files to see if we need to start a harvester for _, file := range matches { logp.Debug("prospector", "Check file for harvesting: %s", file) // check if the file is in the exclude_files list if p.isFileExcluded(file) { logp.Debug("prospector", "Exclude file: %s", file) continue } // Stat the file, following any symlinks. fileinfo, err := os.Stat(file) // TODO(sissel): check err if err != nil { logp.Debug("prospector", "stat(%s) failed: %s", file, err) continue } if fileinfo.IsDir() { logp.Debug("prospector", "Skipping directory: %s", file) continue } // Check the current info against p.prospectorinfo[file] lastinfo, isKnown := p.prospectorList[file] // Create a new prospector info with the stat info for comparison newInfo := harvester.NewFileStat(fileinfo, p.iteration) // Call crawler if there if there exists a state for the given file offset, resuming := p.registrar.fetchState(file, newInfo.Fileinfo) p.oldStates[file] = oldState{ fileinfo: fileinfo, lastinfo: lastinfo, isKnown: isKnown, newInfo: newInfo, offset: offset, resuming: resuming, } } for file, oldState := range p.oldStates { newFile := input.File{ FileInfo: oldState.fileinfo, } oldFile := input.File{ FileInfo: oldState.lastinfo.Fileinfo, } // Conditions for starting a new harvester: // - file path hasn't been seen before // - the file's inode or device changed if !oldState.isKnown { p.checkNewFile(oldState.newInfo, file, output, oldState) } else { oldState.newInfo.Continue(&oldState.lastinfo) p.checkExistingFile(oldState.newInfo, &newFile, &oldFile, file, output, oldState) } // Track the stat data for this file for later comparison to check for // rotation/etc p.prospectorList[file] = *oldState.newInfo } // for each file matched by the glob }
// Scans the specific path which can be a glob (/**/**/*.log) // For all found files it is checked if a harvester should be started func (p ProspectorLog) scanGlob(glob string) { logp.Debug("prospector", "scan path %s", glob) // Evaluate the path as a wildcards/shell glob matches, err := filepath.Glob(glob) if err != nil { logp.Debug("prospector", "glob(%s) failed: %v", glob, err) return } p.missingFiles = map[string]os.FileInfo{} // Check any matched files to see if we need to start a harvester for _, file := range matches { logp.Debug("prospector", "Check file for harvesting: %s", file) // check if the file is in the exclude_files list if p.isFileExcluded(file) { logp.Debug("prospector", "Exclude file: %s", file) continue } // Stat the file, following any symlinks. fileinfo, err := os.Stat(file) if err != nil { logp.Debug("prospector", "stat(%s) failed: %s", file, err) continue } newFile := input.NewFile(fileinfo) if newFile.FileInfo.IsDir() { logp.Debug("prospector", "Skipping directory: %s", file) continue } // Check the current info against p.prospectorinfo[file] lastinfo, isKnown := p.prospectorList[file] oldFile := input.NewFile(lastinfo.Fileinfo) // Create a new prospector info with the stat info for comparison newInfo := harvester.NewFileStat(newFile.FileInfo, p.iteration) // Init harvester with info h, err := p.Prospector.AddHarvester(file, newInfo) if err != nil { logp.Err("Error initializing harvester: %v", err) continue } // Conditions for starting a new harvester: // - file path hasn't been seen before // - the file's inode or device changed if !isKnown { p.checkNewFile(h) } else { h.Stat.Continue(&lastinfo) p.checkExistingFile(h, &newFile, &oldFile) } // Track the stat data for this file for later comparison to check for // rotation/etc p.prospectorList[h.Path] = *h.Stat } }