func (p *Publisher) updateStatistics(status int, err error) { p.Lock() p.status = status p.line_speed = core.CalculateSpeed(time.Since(p.last_measurement), p.line_speed, float64(p.line_count-p.last_line_count), &p.seconds_no_ack) p.last_line_count = p.line_count p.last_retry_count = p.retry_count p.last_measurement = time.Now() if err == ErrNetworkTimeout || err == ErrNetworkPing { p.timeout_count++ } p.Unlock() }
func (h *Harvester) harvest(output chan<- *core.EventDescriptor) (int64, error) { if err := h.prepareHarvester(); err != nil { return h.offset, err } defer h.file.Close() h.output = output if h.path == "-" { log.Info("Started stdin harvester") h.offset = 0 } else { // Get current offset in file offset, err := h.file.Seek(0, os.SEEK_CUR) if err != nil { log.Warning("Failed to determine start offset for %s: %s", h.path, err) return h.offset, err } if h.offset != offset { log.Warning("Started harvester at position %d (requested %d): %s", offset, h.offset, h.path) } else { log.Info("Started harvester at position %d (requested %d): %s", offset, h.offset, h.path) } h.offset = offset } // The buffer size limits the maximum line length we can read, including terminator reader := NewLineReader(h.file, int(h.config.General.LineBufferBytes), int(h.config.General.MaxLineBytes)) // TODO: Make configurable? read_timeout := 10 * time.Second last_read_time := time.Now() last_line_count := uint64(0) last_byte_count := uint64(0) last_measurement := last_read_time seconds_without_events := 0 ReadLoop: for { text, bytesread, err := h.readline(reader) if duration := time.Since(last_measurement); duration >= time.Second { h.Lock() h.line_speed = core.CalculateSpeed(duration, h.line_speed, float64(h.line_count-last_line_count), &seconds_without_events) h.byte_speed = core.CalculateSpeed(duration, h.byte_speed, float64(h.byte_count-last_byte_count), &seconds_without_events) last_byte_count = h.byte_count last_line_count = h.line_count last_measurement = time.Now() h.codec.Meter() h.last_eof = nil h.Unlock() // Check shutdown select { case <-h.stop_chan: break ReadLoop default: } } if err == nil { line_offset := h.offset h.offset += int64(bytesread) // Codec is last - it forwards harvester state for us such as offset for resume h.codec.Event(line_offset, h.offset, text) last_read_time = time.Now() h.line_count++ h.byte_count += uint64(bytesread) continue } if err != io.EOF { if h.path == "-" { log.Error("Unexpected error reading from stdin: %s", err) } else { log.Error("Unexpected error reading from %s: %s", h.path, err) } return h.codec.Teardown(), err } if h.path == "-" { // Stdin has finished - stdin blocks permanently until the stream ends // Once the stream ends, finish the harvester log.Info("Stopping harvest of stdin; EOF reached") return h.codec.Teardown(), nil } // Check shutdown select { case <-h.stop_chan: break ReadLoop default: } h.Lock() if h.last_eof_off == nil { h.last_eof_off = new(int64) } *h.last_eof_off = h.offset if h.last_eof == nil { h.last_eof = new(time.Time) } *h.last_eof = last_read_time h.Unlock() // Don't check for truncation until we hit the full read_timeout if time.Since(last_read_time) < read_timeout { continue } info, err := h.file.Stat() if err != nil { log.Error("Unexpected error checking status of %s: %s", h.path, err) return h.codec.Teardown(), err } if info.Size() < h.offset { log.Warning("Unexpected file truncation, seeking to beginning: %s", h.path) h.file.Seek(0, os.SEEK_SET) h.offset = 0 // TODO: Should we be allowing truncation to lose buffer data? Or should // we be flushing what we have? // Reset line buffer and codec buffers reader.Reset() h.codec.Reset() continue } // If last_read_time was more than dead time, this file is probably dead. // Stop only if the mtime did not change since last check - this stops a // race where we hit EOF but as we Stat() the mtime is updated - this mtime // is the one we monitor in order to resume checking, so we need to check it // didn't already update if age := time.Since(last_read_time); age > h.stream_config.DeadTime && h.fileinfo.ModTime() == info.ModTime() { log.Info("Stopping harvest of %s; last change was %v ago", h.path, age-(age%time.Second)) return h.codec.Teardown(), nil } // Store latest stat() h.fileinfo = info } log.Info("Harvester for %s exiting", h.path) return h.codec.Teardown(), nil }