예제 #1
0
func (p *Publisher) updateStatistics(status int, err error) {
	p.Lock()

	p.status = status

	p.line_speed = core.CalculateSpeed(time.Since(p.last_measurement), p.line_speed, float64(p.line_count-p.last_line_count), &p.seconds_no_ack)

	p.last_line_count = p.line_count
	p.last_retry_count = p.retry_count
	p.last_measurement = time.Now()

	if err == ErrNetworkTimeout || err == ErrNetworkPing {
		p.timeout_count++
	}

	p.Unlock()
}
예제 #2
0
func (h *Harvester) harvest(output chan<- *core.EventDescriptor) (int64, error) {
	if err := h.prepareHarvester(); err != nil {
		return h.offset, err
	}

	defer h.file.Close()

	h.output = output

	if h.path == "-" {
		log.Info("Started stdin harvester")
		h.offset = 0
	} else {
		// Get current offset in file
		offset, err := h.file.Seek(0, os.SEEK_CUR)
		if err != nil {
			log.Warning("Failed to determine start offset for %s: %s", h.path, err)
			return h.offset, err
		}

		if h.offset != offset {
			log.Warning("Started harvester at position %d (requested %d): %s", offset, h.offset, h.path)
		} else {
			log.Info("Started harvester at position %d (requested %d): %s", offset, h.offset, h.path)
		}

		h.offset = offset
	}

	// The buffer size limits the maximum line length we can read, including terminator
	reader := NewLineReader(h.file, int(h.config.General.LineBufferBytes), int(h.config.General.MaxLineBytes))

	// TODO: Make configurable?
	read_timeout := 10 * time.Second

	last_read_time := time.Now()
	last_line_count := uint64(0)
	last_byte_count := uint64(0)
	last_measurement := last_read_time
	seconds_without_events := 0

ReadLoop:
	for {
		text, bytesread, err := h.readline(reader)

		if duration := time.Since(last_measurement); duration >= time.Second {
			h.Lock()

			h.line_speed = core.CalculateSpeed(duration, h.line_speed, float64(h.line_count-last_line_count), &seconds_without_events)
			h.byte_speed = core.CalculateSpeed(duration, h.byte_speed, float64(h.byte_count-last_byte_count), &seconds_without_events)

			last_byte_count = h.byte_count
			last_line_count = h.line_count
			last_measurement = time.Now()

			h.codec.Meter()

			h.last_eof = nil

			h.Unlock()

			// Check shutdown
			select {
			case <-h.stop_chan:
				break ReadLoop
			default:
			}
		}

		if err == nil {
			line_offset := h.offset
			h.offset += int64(bytesread)

			// Codec is last - it forwards harvester state for us such as offset for resume
			h.codec.Event(line_offset, h.offset, text)

			last_read_time = time.Now()
			h.line_count++
			h.byte_count += uint64(bytesread)

			continue
		}

		if err != io.EOF {
			if h.path == "-" {
				log.Error("Unexpected error reading from stdin: %s", err)
			} else {
				log.Error("Unexpected error reading from %s: %s", h.path, err)
			}
			return h.codec.Teardown(), err
		}

		if h.path == "-" {
			// Stdin has finished - stdin blocks permanently until the stream ends
			// Once the stream ends, finish the harvester
			log.Info("Stopping harvest of stdin; EOF reached")
			return h.codec.Teardown(), nil
		}

		// Check shutdown
		select {
		case <-h.stop_chan:
			break ReadLoop
		default:
		}

		h.Lock()
		if h.last_eof_off == nil {
			h.last_eof_off = new(int64)
		}
		*h.last_eof_off = h.offset

		if h.last_eof == nil {
			h.last_eof = new(time.Time)
		}
		*h.last_eof = last_read_time
		h.Unlock()

		// Don't check for truncation until we hit the full read_timeout
		if time.Since(last_read_time) < read_timeout {
			continue
		}

		info, err := h.file.Stat()
		if err != nil {
			log.Error("Unexpected error checking status of %s: %s", h.path, err)
			return h.codec.Teardown(), err
		}

		if info.Size() < h.offset {
			log.Warning("Unexpected file truncation, seeking to beginning: %s", h.path)
			h.file.Seek(0, os.SEEK_SET)
			h.offset = 0

			// TODO: Should we be allowing truncation to lose buffer data? Or should
			//       we be flushing what we have?
			// Reset line buffer and codec buffers
			reader.Reset()
			h.codec.Reset()
			continue
		}

		// If last_read_time was more than dead time, this file is probably dead.
		// Stop only if the mtime did not change since last check - this stops a
		// race where we hit EOF but as we Stat() the mtime is updated - this mtime
		// is the one we monitor in order to resume checking, so we need to check it
		// didn't already update
		if age := time.Since(last_read_time); age > h.stream_config.DeadTime && h.fileinfo.ModTime() == info.ModTime() {
			log.Info("Stopping harvest of %s; last change was %v ago", h.path, age-(age%time.Second))
			return h.codec.Teardown(), nil
		}

		// Store latest stat()
		h.fileinfo = info
	}

	log.Info("Harvester for %s exiting", h.path)
	return h.codec.Teardown(), nil
}