// Scan starts a scanGlob for each provided path/glob
func (p *ProspectorLog) scan() {
newlastscan := time.Now()
// TODO: Track harvesters to prevent any file from being harvested twice. Finished state could be delayed?
// Now let's do one quick scan to pick up new files
for file, fileinfo := range p.getFiles() {
logp.Debug("prospector", "Check file for harvesting: %s", file)
// Create new state for comparison
newState := input.NewFileState(fileinfo, file)
// TODO: This currently blocks writing updates every time state is fetched. Should be improved for performance
p.Prospector.stateMutex.Lock()
// Load last state
index, lastState := p.Prospector.findPreviousState(newState)
p.Prospector.stateMutex.Unlock()
// Decides if previous state exists
if index == -1 {
p.harvestNewFile(newState)
} else {
p.harvestExistingFile(newState, lastState)
}
}
p.lastscan = newlastscan
}
func (p *ProspectorLog) Init() {
logp.Debug("prospector", "exclude_files: %s", p.config.ExcludeFiles)
logp.Info("Load previous states from registry into memory")
// Load the initial state from the registry
for path, fileinfo := range p.getFiles() {
// Check for each path found, if there is a previous state
offset := p.Prospector.registrar.fetchState(path, fileinfo)
// Offset found -> skip to previous state
if offset > 0 {
state := input.NewFileState(fileinfo, path)
state.Offset = offset
// Make sure new harvester is started for all states
state.Finished = true
// Prospector must update all states as it has to detect also file rotation
p.Prospector.updateState(state)
}
}
logp.Info("Previous states loaded: %v", len(p.Prospector.harvesterStates))
}