// handleIgnoreOlder handles states which fall under ignore older // Based on the state information it is decided if the state information has to be updated or not func (p *ProspectorLog) handleIgnoreOlder(lastState, newState file.State) error { logp.Debug("prospector", "Ignore file because ignore_older reached: %s", newState.Source) if !lastState.IsEmpty() { if !lastState.Finished { logp.Info("File is falling under ignore_older before harvesting is finished. Adjust your close_* settings: %s", newState.Source) } // Old state exist, no need to update it return nil } // Make sure file is not falling under clean_inactive yet if p.isCleanInactive(newState) { logp.Debug("prospector", "Do not write state for ignore_older because clean_inactive reached") return nil } // Write state for ignore_older file as none exists yet newState.Finished = true err := p.Prospector.updateState(input.NewEvent(newState)) if err != nil { return err } return nil }
// handleIgnoreOlder handles states which fall under ignore older // Based on the state information it is decided if the state information has to be updated or not func (p *ProspectorLog) handleIgnoreOlder(lastState, newState file.State) error { logp.Debug("prospector", "Ignore file because ignore_older reached: %s", newState.Source) if !lastState.IsEmpty() { if !lastState.Finished { logp.Info("File is falling under ignore_older before harvesting is finished. Adjust your close_* settings: %s", newState.Source) } // Old state exist, no need to update it return nil } // Make sure file is not falling under clean_inactive yet if p.isCleanInactive(newState) { logp.Debug("prospector", "Do not write state for ignore_older because clean_inactive reached") return nil } // Set offset to end of file to be consistent with files which were harvested before // See https://github.com/elastic/beats/pull/2907 newState.Offset = newState.Fileinfo.Size() // Write state for ignore_older file as none exists yet newState.Finished = true err := p.Prospector.updateState(input.NewEvent(newState)) if err != nil { return err } return nil }
// startHarvester starts a new harvester with the given offset // In case the HarvesterLimit is reached, an error is returned func (p *Prospector) startHarvester(state file.State, offset int64) error { if p.config.HarvesterLimit > 0 && atomic.LoadUint64(&p.harvesterCounter) >= p.config.HarvesterLimit { harvesterSkipped.Add(1) return fmt.Errorf("Harvester limit reached.") } state.Offset = offset // Set state to "not" finished to indicate that a harvester is running state.Finished = false // Create harvester with state h, err := p.createHarvester(state) if err != nil { return err } reader, err := h.Setup() if err != nil { return fmt.Errorf("Error setting up harvester: %s", err) } // State is directly updated and not through channel to make state update immediate // State is only updated after setup is completed successfully err = p.updateState(input.NewEvent(state)) if err != nil { return err } p.wg.Add(1) // startHarvester is not run concurrently, but atomic operations are need for the decrementing of the counter // inside the following go routine atomic.AddUint64(&p.harvesterCounter, 1) go func() { defer func() { atomic.AddUint64(&p.harvesterCounter, ^uint64(0)) p.wg.Done() }() // Starts harvester and picks the right type. In case type is not set, set it to defeault (log) h.Harvest(reader) }() return nil }