Beispiel #1
0
func inferSchema(conn *csvConn) (schemaType, error) {
	tbl, err := csvutil.Open(conn.cfg.File)
	if err != nil {
		return nil, err
	}
	defer tbl.Close()
	tbl.Reader.Comma = conn.cfg.Comma
	tbl.Reader.Comment = conn.cfg.Comment

	return inferSchemaFromTable(tbl)
}
Beispiel #2
0
func TestCSVReaderScanEOF(t *testing.T) {
	fname := "testdata/simple.csv"
	tbl, err := csvutil.Open(fname)
	if err != nil {
		t.Errorf("could not open %s: %v\n", fname, err)
	}
	defer tbl.Close()
	tbl.Reader.Comma = ';'
	tbl.Reader.Comment = '#'

	rows, err := tbl.ReadRows(0, 12)
	if err != nil {
		t.Errorf("could read rows [0, 12): %v\n", err)
	}
	defer rows.Close()

	irow := 0
	for rows.Next() {
		data := struct {
			I int
			F float64
			S string
		}{}
		err = rows.Scan(&data)
		if err != nil {
			if irow == 10 {
				break
			}
			t.Errorf("error reading row %d: %v\n", irow, err)
		}
		exp := fmt.Sprintf("%d;%d;str-%d", irow, irow, irow)
		got := fmt.Sprintf("%v;%v;%v", data.I, data.F, data.S)
		if exp != got {
			t.Errorf("error reading row %d\nexp=%q\ngot=%q\n",
				irow, exp, got,
			)
		}
		irow++
	}

	if irow != 10 {
		t.Errorf("error. expected irow==10. got=%v\n", irow)
	}

	err = rows.Err()
	if err != io.EOF {
		t.Errorf("error: expected io.EOF. got=%v\n", err)
	}
}
Beispiel #3
0
func TestCSVReaderScanArgsSubSample(t *testing.T) {
	fname := "testdata/simple.csv"
	tbl, err := csvutil.Open(fname)
	if err != nil {
		t.Errorf("could not open %s: %v\n", fname, err)
	}
	defer tbl.Close()
	tbl.Reader.Comma = ';'
	tbl.Reader.Comment = '#'

	rows, err := tbl.ReadRows(2, 10)
	if err != nil {
		t.Errorf("could read rows [2, 10): %v\n", err)
	}
	defer rows.Close()

	irow := 2
	for rows.Next() {
		var (
			i int
			f float64
			s string
		)
		err = rows.Scan(&i, &f, &s)
		if err != nil {
			t.Errorf("error reading row %d: %v\n", irow, err)
		}
		exp := fmt.Sprintf("%d;%d;str-%d", irow, irow, irow)
		got := fmt.Sprintf("%v;%v;%v", i, f, s)
		if exp != got {
			t.Errorf("error reading row %d\nexp=%q\ngot=%q\n",
				irow, exp, got,
			)
		}
		irow++
	}

	err = rows.Err()
	if err != nil {
		t.Errorf("error iterating over rows: %v\n", err)
	}

	if irow-2 != 8 {
		t.Errorf("error: got %d rows. expected 8\n", irow-2)
	}
}
Beispiel #4
0
func (conn *csvConn) importCSV() error {
	tbl, err := csvutil.Open(conn.cfg.File)
	if err != nil {
		return err
	}
	defer tbl.Close()
	tbl.Reader.Comma = conn.cfg.Comma
	tbl.Reader.Comment = conn.cfg.Comment

	schema, err := inferSchema(conn)
	if err != nil {
		return err
	}

	tx, err := conn.Begin()
	if err != nil {
		return err
	}
	defer tx.Commit()

	_, err = conn.Exec("create table csv ("+schema.Decl()+")", nil)
	if err != nil {
		return err
	}

	_, err = conn.Exec("create index csv_id on csv (id());", nil)
	if err != nil {
		return err
	}

	rows, err := tbl.ReadRows(0, -1)
	if err != nil {
		return err
	}
	defer rows.Close()

	vargs, pargs := schema.Args()
	def := schema.Def()
	insert := "insert into csv values(" + def + ");"
	for rows.Next() {
		err = rows.Scan(pargs...)
		if err != nil {
			return err
		}
		for i, arg := range pargs {
			vargs[i] = reflect.ValueOf(arg).Elem().Interface()
		}
		_, err = conn.Exec(insert, vargs)
		if err != nil {
			return err
		}
	}

	err = rows.Err()
	if err == io.EOF {
		err = nil
	}
	if err != nil {
		return err
	}

	err = tx.Commit()
	if err != nil {
		return err
	}

	return nil
}