func inferSchema(conn *csvConn) (schemaType, error) { tbl, err := csvutil.Open(conn.cfg.File) if err != nil { return nil, err } defer tbl.Close() tbl.Reader.Comma = conn.cfg.Comma tbl.Reader.Comment = conn.cfg.Comment return inferSchemaFromTable(tbl) }
func TestCSVReaderScanEOF(t *testing.T) { fname := "testdata/simple.csv" tbl, err := csvutil.Open(fname) if err != nil { t.Errorf("could not open %s: %v\n", fname, err) } defer tbl.Close() tbl.Reader.Comma = ';' tbl.Reader.Comment = '#' rows, err := tbl.ReadRows(0, 12) if err != nil { t.Errorf("could read rows [0, 12): %v\n", err) } defer rows.Close() irow := 0 for rows.Next() { data := struct { I int F float64 S string }{} err = rows.Scan(&data) if err != nil { if irow == 10 { break } t.Errorf("error reading row %d: %v\n", irow, err) } exp := fmt.Sprintf("%d;%d;str-%d", irow, irow, irow) got := fmt.Sprintf("%v;%v;%v", data.I, data.F, data.S) if exp != got { t.Errorf("error reading row %d\nexp=%q\ngot=%q\n", irow, exp, got, ) } irow++ } if irow != 10 { t.Errorf("error. expected irow==10. got=%v\n", irow) } err = rows.Err() if err != io.EOF { t.Errorf("error: expected io.EOF. got=%v\n", err) } }
func TestCSVReaderScanArgsSubSample(t *testing.T) { fname := "testdata/simple.csv" tbl, err := csvutil.Open(fname) if err != nil { t.Errorf("could not open %s: %v\n", fname, err) } defer tbl.Close() tbl.Reader.Comma = ';' tbl.Reader.Comment = '#' rows, err := tbl.ReadRows(2, 10) if err != nil { t.Errorf("could read rows [2, 10): %v\n", err) } defer rows.Close() irow := 2 for rows.Next() { var ( i int f float64 s string ) err = rows.Scan(&i, &f, &s) if err != nil { t.Errorf("error reading row %d: %v\n", irow, err) } exp := fmt.Sprintf("%d;%d;str-%d", irow, irow, irow) got := fmt.Sprintf("%v;%v;%v", i, f, s) if exp != got { t.Errorf("error reading row %d\nexp=%q\ngot=%q\n", irow, exp, got, ) } irow++ } err = rows.Err() if err != nil { t.Errorf("error iterating over rows: %v\n", err) } if irow-2 != 8 { t.Errorf("error: got %d rows. expected 8\n", irow-2) } }
func (conn *csvConn) importCSV() error { tbl, err := csvutil.Open(conn.cfg.File) if err != nil { return err } defer tbl.Close() tbl.Reader.Comma = conn.cfg.Comma tbl.Reader.Comment = conn.cfg.Comment schema, err := inferSchema(conn) if err != nil { return err } tx, err := conn.Begin() if err != nil { return err } defer tx.Commit() _, err = conn.Exec("create table csv ("+schema.Decl()+")", nil) if err != nil { return err } _, err = conn.Exec("create index csv_id on csv (id());", nil) if err != nil { return err } rows, err := tbl.ReadRows(0, -1) if err != nil { return err } defer rows.Close() vargs, pargs := schema.Args() def := schema.Def() insert := "insert into csv values(" + def + ");" for rows.Next() { err = rows.Scan(pargs...) if err != nil { return err } for i, arg := range pargs { vargs[i] = reflect.ValueOf(arg).Elem().Interface() } _, err = conn.Exec(insert, vargs) if err != nil { return err } } err = rows.Err() if err == io.EOF { err = nil } if err != nil { return err } err = tx.Commit() if err != nil { return err } return nil }