Example #1
0
File: sync.go Project: dckit/dccli
func (s *Syncer) syncFile(file string) error {
	f, err := position.Open(file)
	if err != nil {
		if err == position.ErrNoUpdate {
			err = nil
		}
		return err
	}
	defer f.Close()

	events := make([]Event, 0, s.BatchSize)
	reader := bufio.NewReader(f)
	for {
		line, err := reader.ReadBytes(s.Separator)

		if err == nil && len(line) > 1 {
			events = append(events, Event{
				Headers: map[string]string{"foo": "bar"},
				Body:    line[:len(line)-1],
			})
		}

		if len(events) == s.BatchSize || err != nil {
			if err2 := s.sender.Send(events); err2 != nil {
				return err2
			}
			if err == nil {
				position.Update(f)
			} else {
				// ocuer err such as EOF IOERR, must decrease the offset len(line)
				position.UpdateWithFix(f, -int64(len(line)))
			}
			events = events[:0]
		}

		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}
Example #2
0
func (s *Syncer) syncFile(file string) error {
	f, err := position.Open(file)
	if err != nil {
		if err == position.ErrNoUpdate {
			err = nil
		}
		return err
	}
	defer f.Close()

	events := make([]Event, 0, s.BatchSize)
	reader := bufio.NewReader(f)
	for {
		line, err := reader.ReadBytes(s.Separator)
		if len(line) > 0 && line[len(line)-1] == s.Separator {
			line = line[:len(line)-1]
		}
		if len(line) != 0 {
			events = append(events, Event{
				Headers: map[string]interface{}{"foo": "bar"},
				Body:    line,
			})
		}

		if len(events) == s.BatchSize || err != nil {
			if err2 := s.sender.Send(events); err2 != nil {
				return err2
			}
			events = events[:0]
			position.Update(f)
		}

		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return err
		}
	}
}