func (c *Crawler) Start(states file.States) error {
logp.Info("Loading Prospectors: %v", len(c.prospectorConfigs))
// Prospect the globs/paths given on the command line and launch harvesters
for _, prospectorConfig := range c.prospectorConfigs {
prospector, err := prospector.NewProspector(prospectorConfig, states, c.spooler.Channel)
if err != nil {
return fmt.Errorf("Error in initing prospector: %s", err)
}
c.prospectors = append(c.prospectors, prospector)
}
logp.Info("Loading Prospectors completed. Number of prospectors: %v", len(c.prospectors))
for i, p := range c.prospectors {
c.wg.Add(1)
go func(id int, prospector *prospector.Prospector) {
defer func() {
c.wg.Done()
logp.Debug("crawler", "Prospector %v stopped", id)
}()
logp.Debug("crawler", "Starting prospector %v", id)
prospector.Run()
}(i, p)
}
logp.Info("All prospectors are initialised and running with %d states to persist", states.Count())
return nil
}
func NewProspector(cfg *common.Config, states file.States, spoolerChan chan *input.FileEvent) (*Prospector, error) {
prospector := &Prospector{
cfg: cfg,
config: defaultConfig,
spoolerChan: spoolerChan,
harvesterChan: make(chan *input.FileEvent),
done: make(chan struct{}),
states: states.Copy(),
wg: sync.WaitGroup{},
}
if err := cfg.Unpack(&prospector.config); err != nil {
return nil, err
}
if err := prospector.config.Validate(); err != nil {
return nil, err
}
err := prospector.Init()
if err != nil {
return nil, err
}
logp.Debug("prospector", "File Configs: %v", prospector.config.Paths)
return prospector, nil
}
func NewProspector(cfg *common.Config, states file.States, outlet Outlet) (*Prospector, error) {
prospector := &Prospector{
cfg: cfg,
config: defaultConfig,
outlet: outlet,
harvesterChan: make(chan *input.Event),
done: make(chan struct{}),
wg: sync.WaitGroup{},
states: &file.States{},
channelWg: sync.WaitGroup{},
}
if err := cfg.Unpack(&prospector.config); err != nil {
return nil, err
}
if err := prospector.config.Validate(); err != nil {
return nil, err
}
err := prospector.Init(states.GetStates())
if err != nil {
return nil, err
}
logp.Debug("prospector", "File Configs: %v", prospector.config.Paths)
return prospector, nil
}
// Init sets up the prospector
// It goes through all states coming from the registry. Only the states which match the glob patterns of
// the prospector will be loaded and updated. All other states will not be touched.
func (p *ProspectorLog) Init(states file.States) error {
logp.Debug("prospector", "exclude_files: %s", p.config.ExcludeFiles)
for _, state := range states.GetStates() {
// Check if state source belongs to this prospector. If yes, update the state.
if p.matchesFile(state.Source) {
state.TTL = -1
// Update prospector states and send new states to registry
err := p.Prospector.updateState(input.NewEvent(state))
if err != nil {
logp.Err("Problem putting initial state: %+v", err)
return err
}
}
}
logp.Info("Prospector with previous states loaded: %v", p.Prospector.states.Count())
return nil
}