Example #1
0
// NewCSVInputReader returns a CSVInputReader configured to read data from the
// given io.Reader, extracting only the specified fields using exactly "numDecoders"
// goroutines.
func NewCSVInputReader(fields []string, in io.Reader, numDecoders int) *CSVInputReader {
	szCount := &sizeTrackingReader{in, 0}
	csvReader := csv.NewReader(szCount)
	// allow variable number of fields in document
	csvReader.FieldsPerRecord = -1
	csvReader.TrimLeadingSpace = true
	return &CSVInputReader{
		fields:       fields,
		csvReader:    csvReader,
		numProcessed: uint64(0),
		numDecoders:  numDecoders,
		sizeTracker:  szCount,
	}
}
Example #2
0
File: csv.go Project: Machyne/mongo
// NewCSVInputReader returns a CSVInputReader configured to read data from the
// given io.Reader, extracting only the specified columns using exactly "numDecoders"
// goroutines.
func NewCSVInputReader(colSpecs []ColumnSpec, in io.Reader, rejects io.Writer, numDecoders int, ignoreBlanks bool) *CSVInputReader {
	szCount := newSizeTrackingReader(newBomDiscardingReader(in))
	csvReader := csv.NewReader(szCount)
	// allow variable number of colSpecs in document
	csvReader.FieldsPerRecord = -1
	csvReader.TrimLeadingSpace = true
	return &CSVInputReader{
		colSpecs:        colSpecs,
		csvReader:       csvReader,
		csvRejectWriter: gocsv.NewWriter(rejects),
		numProcessed:    uint64(0),
		numDecoders:     numDecoders,
		sizeTracker:     szCount,
		ignoreBlanks:    ignoreBlanks,
	}
}