func (p *Prospector) stdinRun(spoolChan chan *input.FileEvent) {
h, err := harvester.NewHarvester(
p.ProspectorConfig,
&p.ProspectorConfig.Harvester,
"-",
nil,
spoolChan,
)
if err != nil {
logp.Err("Error initializing stdin harvester: %v", err)
return
}
// This signals we finished considering the previous state
event := &input.FileState{
Source: nil,
}
p.registrar.Persist <- event
h.Start()
for {
if !p.running {
break
}
// Wait time during endless loop
oneSecond, _ := time.ParseDuration("1s")
time.Sleep(oneSecond)
}
}
// Check if harvester for new file has to be started
// For a new file the following options exist:
func (p *Prospector) checkNewFile(newinfo *harvester.FileStat, file string, output chan *input.FileEvent) {
logp.Debug("prospector", "Start harvesting unknown file: %s", file)
// Init harvester with info
h, err := harvester.NewHarvester(
p.ProspectorConfig, &p.ProspectorConfig.Harvester, file, newinfo, output)
if err != nil {
logp.Err("Error initializing harvester: %v", err)
return
}
// Check for unmodified time, but only if the file modification time is before the last scan started
// This ensures we don't skip genuine creations with dead times less than 10s
if newinfo.Fileinfo.ModTime().Before(p.lastscan) &&
time.Since(newinfo.Fileinfo.ModTime()) > p.ProspectorConfig.IgnoreOlderDuration {
logp.Debug("prospector", "Fetching old state of file to resume: %s", file)
// Call crawler if there if there exists a state for the given file
offset, resuming := p.registrar.fetchState(file, newinfo.Fileinfo)
// Are we resuming a dead file? We have to resume even if dead so we catch any old updates to the file
// This is safe as the harvester, once it hits the EOF and a timeout, will stop harvesting
// Once we detect changes again we can resume another harvester again - this keeps number of go routines to a minimum
if resuming {
logp.Debug("prospector", "Resuming harvester on a previously harvested file: %s", file)
h.Offset = offset
h.Start()
} else {
// Old file, skip it, but push offset of file size so we start from the end if this file changes and needs picking up
logp.Debug("prospector", "Skipping file (older than ignore older of %v, %v): %s",
p.ProspectorConfig.IgnoreOlderDuration,
time.Since(newinfo.Fileinfo.ModTime()),
file)
newinfo.Skip(newinfo.Fileinfo.Size())
}
} else if previousFile, err := p.getPreviousFile(file, newinfo.Fileinfo); err == nil {
// This file was simply renamed (known inode+dev) - link the same harvester channel as the old file
logp.Debug("prospector", "File rename was detected: %s -> %s", previousFile, file)
lastinfo := p.prospectorList[previousFile]
newinfo.Continue(&lastinfo)
} else {
// Call crawler if there if there exists a state for the given file
offset, resuming := p.registrar.fetchState(file, newinfo.Fileinfo)
// Are we resuming a file or is this a completely new file?
if resuming {
logp.Debug("prospector", "Resuming harvester on a previously harvested file: %s", file)
} else {
logp.Debug("prospector", "Launching harvester on new file: %s", file)
}
// Launch the harvester
h.Offset = offset
h.Start()
}
}
// checkExistingFile checks if a harvester has to be started for a already known file
// For existing files the following options exist:
// * Last reading position is 0, no harvester has to be started as old harvester probably still busy
// * The old known modification time is older then the current one. Start at last known position
// * The new file is not the same as the old file, means file was renamed
// ** New file is actually really a new file, start a new harvester
// ** Renamed file has a state, continue there
func (p *Prospector) checkExistingFile(newinfo *harvester.FileStat, newFile *input.File, oldFile *input.File, file string, output chan *input.FileEvent) {
logp.Debug("prospector", "Update existing file for harvesting: %s", file)
h, err := harvester.NewHarvester(
p.ProspectorConfig, &p.ProspectorConfig.Harvester,
file, newinfo, output)
if err != nil {
logp.Err("Error initializing harvester: %v", err)
return
}
if !oldFile.IsSameFile(newFile) {
if previousFile, err := p.getPreviousFile(file, newinfo.Fileinfo); err == nil {
// This file was renamed from another file we know - link the same harvester channel as the old file
logp.Debug("prospector", "File rename was detected: %s -> %s", previousFile, file)
logp.Debug("prospector", "Launching harvester on renamed file: %s", file)
lastinfo := p.prospectorList[previousFile]
newinfo.Continue(&lastinfo)
} else {
// File is not the same file we saw previously, it must have rotated and is a new file
logp.Debug("prospector", "Launching harvester on rotated file: %s", file)
// Forget about the previous harvester and let it continue on the old file - so start a new channel to use with the new harvester
newinfo.Ignore()
// Start a new harvester on the path
h.Start()
}
// Keep the old file in missingFiles so we don't rescan it if it was renamed and we've not yet reached the new filename
// We only need to keep it for the remainder of this iteration then we can assume it was deleted and forget about it
p.missingFiles[file] = oldFile.FileInfo
} else if newinfo.Finished() && oldFile.FileInfo.ModTime() != newinfo.Fileinfo.ModTime() {
// Resume harvesting of an old file we've stopped harvesting from
logp.Debug("prospector", "Resuming harvester on an old file that was just modified: %s", file)
// Start a harvester on the path; an old file was just modified and it doesn't have a harvester
// The offset to continue from will be stored in the harvester channel - so take that to use and also clear the channel
h.Offset = <-newinfo.Return
h.Start()
} else {
logp.Debug("prospector", "Not harvesting, file didn't change: %s", file)
}
}
// Starts scanning through all the file paths and fetch the related files. Start a harvester for each file
func (p *Prospector) Run(spoolChan chan *input.FileEvent) {
p.running = true
// Handle any "-" (stdin) paths
for i, path := range p.ProspectorConfig.Paths {
logp.Debug("prospector", "Harvest path: %s", path)
if path == "-" {
// Offset and Initial never get used when path is "-"
h, err := harvester.NewHarvester(
p.ProspectorConfig, &p.ProspectorConfig.Harvester,
path, nil, spoolChan)
if err != nil {
logp.Err("Error initializing harvester: %v", err)
return
}
h.Start()
// Remove it from the file list
p.ProspectorConfig.Paths = append(p.ProspectorConfig.Paths[:i], p.ProspectorConfig.Paths[i+1:]...)
}
}
// Seed last scan time
p.lastscan = time.Now()
// Now let's do one quick scan to pick up new files
for _, path := range p.ProspectorConfig.Paths {
p.scan(path, spoolChan)
}
// This signals we finished considering the previous state
event := &input.FileState{
Source: nil,
}
p.registrar.Persist <- event
for {
newlastscan := time.Now()
for _, path := range p.ProspectorConfig.Paths {
// Scan - flag false so new files always start at beginning TODO: is this still working as expected?
p.scan(path, spoolChan)
}
p.lastscan = newlastscan
// Defer next scan for the defined scanFrequency
time.Sleep(p.ProspectorConfig.ScanFrequencyDuration)
logp.Debug("prospector", "Start next scan")
// Clear out files that disappeared and we've stopped harvesting
for file, lastinfo := range p.prospectorList {
if len(lastinfo.Harvester) != 0 && lastinfo.LastIteration < p.iteration {
delete(p.prospectorList, file)
}
}
p.iteration++ // Overflow is allowed
if !p.running {
break
}
}
}