func (s *testSuite) TestSync() { source1 := dataset.NewDataset(datas.NewDatabase(chunks.NewLevelDBStore(s.LdbDir, "", 1, false)), "foo") source1, err := source1.Commit(types.Number(42)) s.NoError(err) source2, err := source1.Commit(types.Number(43)) s.NoError(err) source1HeadRef := source1.Head().Hash() source2.Database().Close() // Close Database backing both Datasets sourceSpec := test_util.CreateValueSpecString("ldb", s.LdbDir, source1HeadRef.String()) ldb2dir := path.Join(s.TempDir, "ldb2") sinkDatasetSpec := test_util.CreateValueSpecString("ldb", ldb2dir, "bar") out := s.Run(main, []string{sourceSpec, sinkDatasetSpec}) s.Equal("", out) dest := dataset.NewDataset(datas.NewDatabase(chunks.NewLevelDBStore(ldb2dir, "", 1, false)), "bar") s.True(types.Number(42).Equals(dest.HeadValue())) dest.Database().Close() sourceDataset := test_util.CreateValueSpecString("ldb", s.LdbDir, "foo") out = s.Run(main, []string{sourceDataset, sinkDatasetSpec}) s.Equal("", out) dest = dataset.NewDataset(datas.NewDatabase(chunks.NewLevelDBStore(ldb2dir, "", 1, false)), "bar") s.True(types.Number(43).Equals(dest.HeadValue())) dest.Database().Close() }
func (s *testSuite) TestCSVImporterToMap() { input, err := ioutil.TempFile(s.TempDir, "") d.Chk.NoError(err) defer input.Close() defer os.Remove(input.Name()) _, err = input.WriteString("a,b,c\n") d.Chk.NoError(err) for i := 0; i < 20; i++ { _, err = input.WriteString(fmt.Sprintf("a%d,%d,%d\n", i, i, i*2)) d.Chk.NoError(err) } _, err = input.Seek(0, 0) d.Chk.NoError(err) setName := "csv" dataspec := test_util.CreateValueSpecString("ldb", s.LdbDir, setName) out := s.Run(main, []string{"-no-progress", "-column-types", "String,Number,Number", "-dest-type", "map:1", dataspec, input.Name()}) s.Equal("", out) cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) ds := dataset.NewDataset(datas.NewDatabase(cs), setName) defer ds.Database().Close() defer os.RemoveAll(s.LdbDir) m := ds.HeadValue().(types.Map) s.Equal(uint64(20), m.Len()) for i := 0; i < 20; i++ { m.Get(types.Number(i)).(types.Struct).Equals(types.NewStruct("", map[string]types.Value{ "a": types.String(fmt.Sprintf("a%d", i)), "c": types.Number(i * 2), })) } }
func (s *testSuite) TestCSVImporterWithExternalHeader() { input, err := ioutil.TempFile(s.TempDir, "") d.Chk.NoError(err) defer input.Close() defer os.Remove(input.Name()) _, err = input.WriteString("7,8\n") d.Chk.NoError(err) setName := "csv" dataspec := test_util.CreateValueSpecString("ldb", s.LdbDir, setName) out := s.Run(main, []string{"-no-progress", "-column-types", "String,Number", "-header", "x,y", dataspec, input.Name()}) s.Equal("", out) cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) ds := dataset.NewDataset(datas.NewDatabase(cs), setName) defer ds.Database().Close() defer os.RemoveAll(s.LdbDir) l := ds.HeadValue().(types.List) s.Equal(uint64(1), l.Len()) v := l.Get(0) st := v.(types.Struct) s.Equal(types.String("7"), st.Get("x")) s.Equal(types.Number(8), st.Get("y")) }
func (s *testSuite) TestCSVImporterFromBlob() { test := func(pathFlag string) { defer os.RemoveAll(s.LdbDir) newDB := func() datas.Database { cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) return datas.NewDatabase(cs) } db := newDB() rawDS := dataset.NewDataset(db, "raw") csv := &bytes.Buffer{} writeCSV(csv) rawDS.CommitValue(types.NewBlob(csv)) db.Close() stdout, stderr := s.Run(main, []string{ "--no-progress", "--column-types", "String,Number", pathFlag, spec.CreateValueSpecString("ldb", s.LdbDir, "raw.value"), spec.CreateValueSpecString("ldb", s.LdbDir, "csv"), }) s.Equal("", stdout) s.Equal("", stderr) db = newDB() defer db.Close() csvDS := dataset.NewDataset(db, "csv") validateCSV(s, csvDS.HeadValue().(types.List)) } test("--path") test("-p") }
func (s *testSuite) TestCSVImportSkipRecordsCustomHeader() { input, err := ioutil.TempFile(s.TempDir, "") d.Chk.NoError(err) defer input.Close() defer os.Remove(input.Name()) _, err = input.WriteString("a,b\n") d.Chk.NoError(err) _, err = input.WriteString("7,8\n") d.Chk.NoError(err) setName := "csv" dataspec := spec.CreateValueSpecString("ldb", s.LdbDir, setName) stdout, stderr := s.Run(main, []string{"--no-progress", "--skip-records", "1", "--header", "x,y", input.Name(), dataspec}) s.Equal("", stdout) s.Equal("", stderr) cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) ds := dataset.NewDataset(datas.NewDatabase(cs), setName) defer ds.Database().Close() defer os.RemoveAll(s.LdbDir) l := ds.HeadValue().(types.List) s.Equal(uint64(1), l.Len()) v := l.Get(0) st := v.(types.Struct) s.Equal(types.String("7"), st.Get("x")) s.Equal(types.String("8"), st.Get("y")) }
func (s *testSuite) TestCSVImporterWithPipe() { input, err := ioutil.TempFile(s.TempDir, "") d.Chk.NoError(err) defer input.Close() defer os.Remove(input.Name()) _, err = input.WriteString("a|b\n1|2\n") d.Chk.NoError(err) setName := "csv" dataspec := spec.CreateValueSpecString("ldb", s.LdbDir, setName) stdout, stderr := s.Run(main, []string{"--no-progress", "--column-types", "String,Number", "--delimiter", "|", input.Name(), dataspec}) s.Equal("", stdout) s.Equal("", stderr) cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) ds := dataset.NewDataset(datas.NewDatabase(cs), setName) defer ds.Database().Close() defer os.RemoveAll(s.LdbDir) l := ds.HeadValue().(types.List) s.Equal(uint64(1), l.Len()) v := l.Get(0) st := v.(types.Struct) s.Equal(types.String("1"), st.Get("a")) s.Equal(types.Number(2), st.Get("b")) }
// FIXME: run with pipe func (s *testSuite) TestCSVExporter() { setName := "csv" header := []string{"a", "b", "c"} payload := [][]string{ []string{"5", "7", "100"}, []string{"4", "10", "255"}, []string{"512", "12", "55"}, } structName := "SomeStruct" // Setup data store cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) ds := dataset.NewDataset(datas.NewDatabase(cs), setName) // Build Struct fields based on header f := make(types.TypeMap, len(header)) for _, key := range header { f[key] = types.StringType } typ := types.MakeStructType(structName, f) // Build data rows structs := make([]types.Value, len(payload)) for i, row := range payload { fields := make(map[string]types.Value) for j, v := range row { name := header[j] fields[name] = types.String(v) } structs[i] = types.NewStructWithType(typ, fields) } ds.Commit(types.NewList(structs...)) ds.Database().Close() // Run exporter dataspec := test_util.CreateValueSpecString("ldb", s.LdbDir, setName) out := s.Run(main, []string{dataspec}) // Verify output csvReader := csv.NewReader(strings.NewReader(out)) row, err := csvReader.Read() d.Chk.NoError(err) s.Equal(header, row) for i := 0; i < len(payload); i++ { row, err := csvReader.Read() d.Chk.NoError(err) s.Equal(payload[i], row) } _, err = csvReader.Read() s.Equal(io.EOF, err) }
func (s *testSuite) TestEmptyNomsDs() { dir := s.LdbDir cs := chunks.NewLevelDBStore(dir+"/name", "", 24, false) ds := datas.NewDatabase(cs) ds.Close() dbSpec := test_util.CreateDatabaseSpecString("ldb", dir+"/name") rtnVal := s.Run(main, []string{dbSpec}) s.Equal("", rtnVal) }
func (s *testSuite) TestNomsDs() { dir := s.LdbDir cs := chunks.NewLevelDBStore(dir+"/name", "", 24, false) ds := datas.NewDatabase(cs) id := "testdataset" set := dataset.NewDataset(ds, id) set, err := set.Commit(types.String("Commit Value")) s.NoError(err) id2 := "testdataset2" set2 := dataset.NewDataset(ds, id2) set2, err = set2.Commit(types.String("Commit Value2")) s.NoError(err) err = ds.Close() s.NoError(err) dbSpec := test_util.CreateDatabaseSpecString("ldb", dir+"/name") datasetName := test_util.CreateValueSpecString("ldb", dir+"/name", id) dataset2Name := test_util.CreateValueSpecString("ldb", dir+"/name", id2) // both datasets show up rtnVal := s.Run(main, []string{dbSpec}) s.Equal(id+"\n"+id2+"\n", rtnVal) // both datasets again, to make sure printing doesn't change them rtnVal = s.Run(main, []string{dbSpec}) s.Equal(id+"\n"+id2+"\n", rtnVal) // delete one dataset, print message at delete rtnVal = s.Run(main, []string{"-d", datasetName}) s.Equal("Deleted dataset "+id+" (was sha1-d54b79552cda9ebe8e446eeb19aab0e69b6ceee3)\n\n", rtnVal) // print datasets, just one left rtnVal = s.Run(main, []string{dbSpec}) s.Equal(id2+"\n", rtnVal) // delete the second dataset rtnVal = s.Run(main, []string{"-d", dataset2Name}) s.Equal("Deleted dataset "+id2+" (was sha1-7b75b0ebfc2a0815ba6fb2b31d03c8f9976ae530)\n\n", rtnVal) // print datasets, none left rtnVal = s.Run(main, []string{dbSpec}) s.Equal("", rtnVal) }
func (s *nomsDsTestSuite) TestNomsDs() { dir := s.LdbDir cs := chunks.NewLevelDBStore(dir+"/name", "", 24, false) ds := datas.NewDatabase(cs) id := "testdataset" set := dataset.NewDataset(ds, id) set, err := set.CommitValue(types.String("Commit Value")) s.NoError(err) id2 := "testdataset2" set2 := dataset.NewDataset(ds, id2) set2, err = set2.CommitValue(types.String("Commit Value2")) s.NoError(err) err = ds.Close() s.NoError(err) dbSpec := spec.CreateDatabaseSpecString("ldb", dir+"/name") datasetName := spec.CreateValueSpecString("ldb", dir+"/name", id) dataset2Name := spec.CreateValueSpecString("ldb", dir+"/name", id2) // both datasets show up rtnVal, _ := s.Run(main, []string{"ds", dbSpec}) s.Equal(id+"\n"+id2+"\n", rtnVal) // both datasets again, to make sure printing doesn't change them rtnVal, _ = s.Run(main, []string{"ds", dbSpec}) s.Equal(id+"\n"+id2+"\n", rtnVal) // delete one dataset, print message at delete rtnVal, _ = s.Run(main, []string{"ds", "-d", datasetName}) s.Equal("Deleted "+datasetName+" (was #6ebc05f71q4sk2psi534fom9se228161)\n", rtnVal) // print datasets, just one left rtnVal, _ = s.Run(main, []string{"ds", dbSpec}) s.Equal(id2+"\n", rtnVal) // delete the second dataset rtnVal, _ = s.Run(main, []string{"ds", "-d", dataset2Name}) s.Equal("Deleted "+dataset2Name+" (was #f5qtovr9mv7mjj2uoq7flcfpksgf0s2j)\n", rtnVal) // print datasets, none left rtnVal, _ = s.Run(main, []string{"ds", dbSpec}) s.Equal("", rtnVal) }
func (s *nomsSyncTestSuite) TestSyncValidation() { source1 := dataset.NewDataset(datas.NewDatabase(chunks.NewLevelDBStore(s.LdbDir, "", 1, false)), "src") source1, err := source1.CommitValue(types.Number(42)) s.NoError(err) source1HeadRef := source1.Head().Hash() source1.Database().Close() sourceSpecMissingHashSymbol := spec.CreateValueSpecString("ldb", s.LdbDir, source1HeadRef.String()) ldb2dir := path.Join(s.TempDir, "ldb2") sinkDatasetSpec := spec.CreateValueSpecString("ldb", ldb2dir, "dest") defer func() { err := recover() s.Equal(exitError{-1}, err) }() s.Run(main, []string{"sync", sourceSpecMissingHashSymbol, sinkDatasetSpec}) }
func (s *testSuite) TestCSVImporter() { input, err := ioutil.TempFile(s.TempDir, "") d.Chk.NoError(err) writeCSV(input) defer input.Close() defer os.Remove(input.Name()) setName := "csv" dataspec := spec.CreateValueSpecString("ldb", s.LdbDir, setName) stdout, stderr := s.Run(main, []string{"--no-progress", "--column-types", "String,Number", input.Name(), dataspec}) s.Equal("", stdout) s.Equal("", stderr) cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) ds := dataset.NewDataset(datas.NewDatabase(cs), setName) defer ds.Database().Close() defer os.RemoveAll(s.LdbDir) validateCSV(s, ds.HeadValue().(types.List)) }
func (s *testSuite) TestCSVImporter() { input, err := ioutil.TempFile(s.TempDir, "") d.Chk.NoError(err) defer input.Close() defer os.Remove(input.Name()) _, err = input.WriteString("a,b\n") d.Chk.NoError(err) for i := 0; i < 100; i++ { _, err = input.WriteString(fmt.Sprintf("a%d,%d\n", i, i)) d.Chk.NoError(err) } _, err = input.Seek(0, 0) d.Chk.NoError(err) setName := "csv" dataspec := test_util.CreateValueSpecString("ldb", s.LdbDir, setName) out := s.Run(main, []string{"-no-progress", "-column-types", "String,Number", dataspec, input.Name()}) s.Equal("", out) cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) ds := dataset.NewDataset(datas.NewDatabase(cs), setName) defer ds.Database().Close() defer os.RemoveAll(s.LdbDir) l := ds.HeadValue().(types.List) s.Equal(uint64(100), l.Len()) i := uint64(0) l.IterAll(func(v types.Value, j uint64) { s.Equal(i, j) st := v.(types.Struct) s.Equal(types.String(fmt.Sprintf("a%d", i)), st.Get("a")) s.Equal(types.Number(i), st.Get("b")) i++ }) }