func (p *Publisher) updateStatistics(status int, err error) {
p.Lock()
p.status = status
p.line_speed = core.CalculateSpeed(time.Since(p.last_measurement), p.line_speed, float64(p.line_count-p.last_line_count), &p.seconds_no_ack)
p.last_line_count = p.line_count
p.last_retry_count = p.retry_count
p.last_measurement = time.Now()
if err == ErrNetworkTimeout || err == ErrNetworkPing {
p.timeout_count++
}
p.Unlock()
}
func (h *Harvester) harvest(output chan<- *core.EventDescriptor) (int64, error) {
if err := h.prepareHarvester(); err != nil {
return h.offset, err
}
defer h.file.Close()
h.output = output
if h.path == "-" {
log.Info("Started stdin harvester")
h.offset = 0
} else {
// Get current offset in file
offset, err := h.file.Seek(0, os.SEEK_CUR)
if err != nil {
log.Warning("Failed to determine start offset for %s: %s", h.path, err)
return h.offset, err
}
if h.offset != offset {
log.Warning("Started harvester at position %d (requested %d): %s", offset, h.offset, h.path)
} else {
log.Info("Started harvester at position %d (requested %d): %s", offset, h.offset, h.path)
}
h.offset = offset
}
// The buffer size limits the maximum line length we can read, including terminator
reader := NewLineReader(h.file, int(h.config.General.LineBufferBytes), int(h.config.General.MaxLineBytes))
// TODO: Make configurable?
read_timeout := 10 * time.Second
last_read_time := time.Now()
last_line_count := uint64(0)
last_byte_count := uint64(0)
last_measurement := last_read_time
seconds_without_events := 0
ReadLoop:
for {
text, bytesread, err := h.readline(reader)
if duration := time.Since(last_measurement); duration >= time.Second {
h.Lock()
h.line_speed = core.CalculateSpeed(duration, h.line_speed, float64(h.line_count-last_line_count), &seconds_without_events)
h.byte_speed = core.CalculateSpeed(duration, h.byte_speed, float64(h.byte_count-last_byte_count), &seconds_without_events)
last_byte_count = h.byte_count
last_line_count = h.line_count
last_measurement = time.Now()
h.codec.Meter()
h.last_eof = nil
h.Unlock()
// Check shutdown
select {
case <-h.stop_chan:
break ReadLoop
default:
}
}
if err == nil {
line_offset := h.offset
h.offset += int64(bytesread)
// Codec is last - it forwards harvester state for us such as offset for resume
h.codec.Event(line_offset, h.offset, text)
last_read_time = time.Now()
h.line_count++
h.byte_count += uint64(bytesread)
continue
}
if err != io.EOF {
if h.path == "-" {
log.Error("Unexpected error reading from stdin: %s", err)
} else {
log.Error("Unexpected error reading from %s: %s", h.path, err)
}
return h.codec.Teardown(), err
}
if h.path == "-" {
// Stdin has finished - stdin blocks permanently until the stream ends
// Once the stream ends, finish the harvester
log.Info("Stopping harvest of stdin; EOF reached")
return h.codec.Teardown(), nil
}
// Check shutdown
select {
case <-h.stop_chan:
break ReadLoop
default:
}
h.Lock()
if h.last_eof_off == nil {
h.last_eof_off = new(int64)
}
*h.last_eof_off = h.offset
if h.last_eof == nil {
h.last_eof = new(time.Time)
}
*h.last_eof = last_read_time
h.Unlock()
// Don't check for truncation until we hit the full read_timeout
if time.Since(last_read_time) < read_timeout {
continue
}
info, err := h.file.Stat()
if err != nil {
log.Error("Unexpected error checking status of %s: %s", h.path, err)
return h.codec.Teardown(), err
}
if info.Size() < h.offset {
log.Warning("Unexpected file truncation, seeking to beginning: %s", h.path)
h.file.Seek(0, os.SEEK_SET)
h.offset = 0
// TODO: Should we be allowing truncation to lose buffer data? Or should
// we be flushing what we have?
// Reset line buffer and codec buffers
reader.Reset()
h.codec.Reset()
continue
}
// If last_read_time was more than dead time, this file is probably dead.
// Stop only if the mtime did not change since last check - this stops a
// race where we hit EOF but as we Stat() the mtime is updated - this mtime
// is the one we monitor in order to resume checking, so we need to check it
// didn't already update
if age := time.Since(last_read_time); age > h.stream_config.DeadTime && h.fileinfo.ModTime() == info.ModTime() {
log.Info("Stopping harvest of %s; last change was %v ago", h.path, age-(age%time.Second))
return h.codec.Teardown(), nil
}
// Store latest stat()
h.fileinfo = info
}
log.Info("Harvester for %s exiting", h.path)
return h.codec.Teardown(), nil
}