func (s *testSuite) TestShove() {
s.LdbFlagName = "-source-ldb"
sn := "storeName"
source1 := dataset.NewDataset(datas.NewDataStore(chunks.NewLevelDBStore(s.LdbDir, sn, 1, false)), "foo")
source1, err := source1.Commit(types.Int32(42))
s.NoError(err)
source2, err := source1.Commit(types.Int32(43))
s.NoError(err)
source1HeadRef := source1.Head().Ref()
source2.Store().Close() // Close DataStore backing both Datasets
ldb2dir := path.Join(s.TempDir, "ldb2")
out := s.Run(main, []string{"-source-store", sn, "-source", source1HeadRef.String(), "-sink-ldb", ldb2dir, "-sink-store", sn, "-sink-ds", "bar"})
s.Equal("", out)
dest := dataset.NewDataset(datas.NewDataStore(chunks.NewLevelDBStore(ldb2dir, sn, 1, false)), "bar")
s.True(types.Int32(42).Equals(dest.Head().Value()))
dest.Store().Close()
out = s.Run(main, []string{"-source-store", sn, "-source", "foo", "-sink-ldb", ldb2dir, "-sink-ds", "bar"})
s.Equal("", out)
dest = dataset.NewDataset(datas.NewDataStore(chunks.NewLevelDBStore(ldb2dir, sn, 1, false)), "bar")
s.True(types.Int32(43).Equals(dest.Head().Value()))
dest.Store().Close()
}
func (s *testSuite) TestCSVImporterWithPipe() {
oldDelimiter := delimiter
newDelimiter := "|"
delimiter = &newDelimiter
defer func() { delimiter = oldDelimiter }()
input, err := ioutil.TempFile(s.TempDir, "")
d.Chk.NoError(err)
defer input.Close()
defer os.Remove(input.Name())
_, err = input.WriteString("a|b\n1|2\n")
d.Chk.NoError(err)
storeName := "store"
setName := "csv"
out := s.Run(main, []string{"-store", storeName, "-column-types", "String,Uint8", "-ds", setName, input.Name()})
s.Equal("", out)
cs := chunks.NewLevelDBStore(s.LdbDir, storeName, 1, false)
ds := dataset.NewDataset(datas.NewDataStore(cs), setName)
defer ds.Store().Close()
defer os.RemoveAll(s.LdbDir)
l := ds.Head().Value().(types.List)
s.Equal(uint64(1), l.Len())
v := l.Get(0)
st := v.(types.Struct)
s.Equal(types.NewString("1"), st.Get("a"))
s.Equal(types.Uint8(2), st.Get("b"))
}
func (s *testSuite) TestCSVImporter() {
input, err := ioutil.TempFile(s.TempDir, "")
d.Chk.NoError(err)
defer input.Close()
defer os.Remove(input.Name())
_, err = input.WriteString("a,b\n")
d.Chk.NoError(err)
for i := 0; i < 100; i++ {
_, err = input.WriteString(fmt.Sprintf("a%d,%d\n", i, i))
d.Chk.NoError(err)
}
_, err = input.Seek(0, 0)
d.Chk.NoError(err)
storeName := "store"
setName := "csv"
out := s.Run(main, []string{"-store", storeName, "-column-types", "String,Uint8", "-ds", setName, input.Name()})
s.Equal("", out)
cs := chunks.NewLevelDBStore(s.LdbDir, storeName, 1, false)
ds := dataset.NewDataset(datas.NewDataStore(cs), setName)
defer ds.Store().Close()
defer os.RemoveAll(s.LdbDir)
l := ds.Head().Value().(types.List)
s.Equal(uint64(100), l.Len())
i := uint64(0)
l.IterAll(func(v types.Value, j uint64) {
s.Equal(i, j)
st := v.(types.Struct)
s.Equal(types.NewString(fmt.Sprintf("a%d", i)), st.Get("a"))
s.Equal(types.Uint8(i), st.Get("b"))
i++
})
}
// FIXME: run with pipe
func (s *testSuite) TestCSVExporter() {
storeName := "store"
setName := "csv"
header := []string{"a", "b", "c"}
payload := [][]string{
[]string{"5", "7", "100"},
[]string{"4", "10", "255"},
[]string{"512", "12", "55"},
}
structName := "SomeStruct"
// Setup data store
cs := chunks.NewLevelDBStore(s.LdbDir, storeName, 1, false)
ds := dataset.NewDataset(datas.NewDataStore(cs), setName)
// Build Struct fields based on header
f := make([]types.Field, 0, len(header))
for _, key := range header {
f = append(f, types.Field{
Name: key,
T: types.MakePrimitiveType(types.StringKind),
})
}
typeDef := types.MakeStructType(structName, f, types.Choices{})
pkg := types.NewPackage([]types.Type{typeDef}, []ref.Ref{})
pkgRef := types.RegisterPackage(&pkg)
typeRef := types.MakeType(pkgRef, 0)
structFields := typeDef.Desc.(types.StructDesc).Fields
// Build data rows
structs := make([]types.Value, len(payload))
for i, row := range payload {
fields := make(map[string]types.Value)
for j, v := range row {
fields[structFields[j].Name] = types.NewString(v)
}
structs[i] = types.NewStruct(typeRef, typeDef, fields)
}
listType := types.MakeCompoundType(types.ListKind, typeRef)
ds.Commit(types.NewTypedList(listType, structs...))
ds.Store().Close()
// Run exporter
out := s.Run(main, []string{"-store", storeName, "-ds", setName})
// Verify output
csvReader := csv.NewReader(strings.NewReader(out))
row, err := csvReader.Read()
d.Chk.NoError(err)
s.Equal(header, row)
for i := 0; i < len(payload); i++ {
row, err := csvReader.Read()
d.Chk.NoError(err)
s.Equal(payload[i], row)
}
row, err = csvReader.Read()
s.Equal(io.EOF, err)
}