Example #1
0
File: file.go Project: ARodri/hume
func (fr *FileReader) ChannelIterator(bufferSize int) <-chan *record.Record {
	output := make(chan *record.Record, bufferSize)
	go func() {
		var err error
		for err == nil {
			r := record.Record{}
			r.Number = fr.lineNumber
			r.Fields = &fr.Header

			r.Values, err = fr.reader.Read()
			if err != nil {
				break
			}
			r.InitMap()

			r.Id = uuid.NewRandom().String()
			if fr.IdField != "" {
				recId, ok := r.Map[fr.IdField]
				if ok {
					r.Id = recId
				} else {
					fr.log.WithFields(log.Fields{
						"file_line": fr.lineNumber,
					}).Errorf("Missing record id field %s", fr.IdField)
				}
			}

			output <- &r
			fr.lineNumber += 1
		}

		if err == io.EOF {
			fr.log.WithFields(log.Fields{
				"file_line": fr.lineNumber,
			}).Debug("End of file reached")
		} else {
			fr.log.Error(err)
		}
		close(output)
	}()
	return output
}
Example #2
0
func (ft *FieldCount) Process(rec *record.Record) {
	l := strconv.Itoa(rec.ValueCount())
	ft.Count(l)
}