func TestReadToMap(t *testing.T) { assert := assert.New(t) ds := datas.NewDatabase(chunks.NewMemoryStore()) dataString := `a,1,true b,2,false ` r := NewCSVReader(bytes.NewBufferString(dataString), ',') headers := []string{"A", "B", "C"} kinds := KindSlice{types.StringKind, types.NumberKind, types.BoolKind} m := ReadToMap(r, headers, 0, kinds, ds) assert.Equal(uint64(2), m.Len()) assert.True(m.Type().Equals( types.MakeMapType(types.StringType, types.MakeStructType("", map[string]*types.Type{ "B": types.NumberType, "C": types.BoolType, })))) assert.True(m.Get(types.String("a")).Equals(types.NewStruct("", map[string]types.Value{ "B": types.Number(1), "C": types.Bool(true), }))) assert.True(m.Get(types.String("b")).Equals(types.NewStruct("", map[string]types.Value{ "B": types.Number(2), "C": types.Bool(false), }))) }
func (s *testSuite) TestCSVImporterToMap() { input, err := ioutil.TempFile(s.TempDir, "") d.Chk.NoError(err) defer input.Close() defer os.Remove(input.Name()) _, err = input.WriteString("a,b,c\n") d.Chk.NoError(err) for i := 0; i < 20; i++ { _, err = input.WriteString(fmt.Sprintf("a%d,%d,%d\n", i, i, i*2)) d.Chk.NoError(err) } _, err = input.Seek(0, 0) d.Chk.NoError(err) setName := "csv" dataspec := test_util.CreateValueSpecString("ldb", s.LdbDir, setName) out := s.Run(main, []string{"-no-progress", "-column-types", "String,Number,Number", "-dest-type", "map:1", dataspec, input.Name()}) s.Equal("", out) cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) ds := dataset.NewDataset(datas.NewDatabase(cs), setName) defer ds.Database().Close() defer os.RemoveAll(s.LdbDir) m := ds.HeadValue().(types.Map) s.Equal(uint64(20), m.Len()) for i := 0; i < 20; i++ { m.Get(types.Number(i)).(types.Struct).Equals(types.NewStruct("", map[string]types.Value{ "a": types.String(fmt.Sprintf("a%d", i)), "c": types.Number(i * 2), })) } }
// Note: This test is asserting that findCommon correctly separates refs which are exclusive to |taller| from those which are |common|. func TestFindCommon(t *testing.T) { taller := &types.RefByHeight{} shorter := &types.RefByHeight{} for i := 0; i < 50; i++ { shorter.PushBack(types.NewRef(types.Number(i))) } for i := 50; i < 250; i++ { shorter.PushBack(types.NewRef(types.Number(i))) taller.PushBack(types.NewRef(types.Number(i))) } for i := 250; i < 275; i++ { taller.PushBack(types.NewRef(types.Number(i))) } sort.Sort(shorter) sort.Sort(taller) tallRefs, comRefs := findCommon(taller, shorter, 1) assert.Equal(t, 25, len(tallRefs)) assert.Equal(t, 200, len(comRefs)) assert.Equal(t, 0, len(*taller)) assert.Equal(t, 50, len(*shorter)) }
func (s *testSuite) TestSync() { source1 := dataset.NewDataset(datas.NewDatabase(chunks.NewLevelDBStore(s.LdbDir, "", 1, false)), "foo") source1, err := source1.Commit(types.Number(42)) s.NoError(err) source2, err := source1.Commit(types.Number(43)) s.NoError(err) source1HeadRef := source1.Head().Hash() source2.Database().Close() // Close Database backing both Datasets sourceSpec := test_util.CreateValueSpecString("ldb", s.LdbDir, source1HeadRef.String()) ldb2dir := path.Join(s.TempDir, "ldb2") sinkDatasetSpec := test_util.CreateValueSpecString("ldb", ldb2dir, "bar") out := s.Run(main, []string{sourceSpec, sinkDatasetSpec}) s.Equal("", out) dest := dataset.NewDataset(datas.NewDatabase(chunks.NewLevelDBStore(ldb2dir, "", 1, false)), "bar") s.True(types.Number(42).Equals(dest.HeadValue())) dest.Database().Close() sourceDataset := test_util.CreateValueSpecString("ldb", s.LdbDir, "foo") out = s.Run(main, []string{sourceDataset, sinkDatasetSpec}) s.Equal("", out) dest = dataset.NewDataset(datas.NewDatabase(chunks.NewLevelDBStore(ldb2dir, "", 1, false)), "bar") s.True(types.Number(43).Equals(dest.HeadValue())) dest.Database().Close() }
// StringToValue takes a piece of data as a string and attempts to convert it to a types.Value of the appropriate types.NomsKind. func StringToValue(s string, k types.NomsKind) (types.Value, error) { switch k { case types.NumberKind: if s == "" { return types.Number(float64(0)), nil } fval, err := strconv.ParseFloat(s, 64) if err != nil { return nil, fmt.Errorf("Could not parse '%s' into number (%s)", s, err) } return types.Number(fval), nil case types.BoolKind: // TODO: This should probably be configurable. switch s { case "true", "1", "y", "Y": return types.Bool(true), nil case "false", "0", "n", "N", "": return types.Bool(false), nil default: return nil, fmt.Errorf("Could not parse '%s' into bool", s) } case types.StringKind: return types.String(s), nil default: d.PanicIfTrue(true, "Invalid column type kind:", k) } panic("not reached") }
func TestReadToList(t *testing.T) { assert := assert.New(t) ds := datas.NewDatabase(chunks.NewMemoryStore()) dataString := `a,1,true b,2,false ` r := NewCSVReader(bytes.NewBufferString(dataString), ',') headers := []string{"A", "B", "C"} kinds := KindSlice{types.StringKind, types.NumberKind, types.BoolKind} l, typ := ReadToList(r, "test", headers, kinds, ds) assert.Equal(uint64(2), l.Len()) assert.Equal(types.StructKind, typ.Kind()) desc, ok := typ.Desc.(types.StructDesc) assert.True(ok) assert.Equal(desc.Len(), 3) assert.Equal(types.StringKind, desc.Field("A").Kind()) assert.Equal(types.NumberKind, desc.Field("B").Kind()) assert.Equal(types.BoolKind, desc.Field("C").Kind()) assert.True(l.Get(0).(types.Struct).Get("A").Equals(types.String("a"))) assert.True(l.Get(1).(types.Struct).Get("A").Equals(types.String("b"))) assert.True(l.Get(0).(types.Struct).Get("B").Equals(types.Number(1))) assert.True(l.Get(1).(types.Struct).Get("B").Equals(types.Number(2))) assert.True(l.Get(0).(types.Struct).Get("C").Equals(types.Bool(true))) assert.True(l.Get(1).(types.Struct).Get("C").Equals(types.Bool(false))) }
func (suite *WalkAllTestSuite) TestWalkComposites() { suite.walkWorker(suite.storeAndRef(types.NewList()), 2) suite.walkWorker(suite.storeAndRef(types.NewList(types.Bool(false), types.Number(8))), 4) suite.walkWorker(suite.storeAndRef(types.NewSet()), 2) suite.walkWorker(suite.storeAndRef(types.NewSet(types.Bool(false), types.Number(8))), 4) suite.walkWorker(suite.storeAndRef(types.NewMap()), 2) suite.walkWorker(suite.storeAndRef(types.NewMap(types.Number(8), types.Bool(true), types.Number(0), types.Bool(false))), 6) }
func TestNewCommit(t *testing.T) { assert := assert.New(t) commitFieldNames := []string{MetaField, ParentsField, ValueField} assertTypeEquals := func(e, a *types.Type) { assert.True(a.Equals(e), "Actual: %s\nExpected %s", a.Describe(), e.Describe()) } commit := NewCommit(types.Number(1), types.NewSet(), types.EmptyStruct) at := commit.Type() et := types.MakeStructType("Commit", commitFieldNames, []*types.Type{ types.EmptyStructType, types.MakeSetType(types.MakeRefType(types.MakeCycleType(0))), types.NumberType, }) assertTypeEquals(et, at) // Commiting another Number commit2 := NewCommit(types.Number(2), types.NewSet(types.NewRef(commit)), types.EmptyStruct) at2 := commit2.Type() et2 := et assertTypeEquals(et2, at2) // Now commit a String commit3 := NewCommit(types.String("Hi"), types.NewSet(types.NewRef(commit2)), types.EmptyStruct) at3 := commit3.Type() et3 := types.MakeStructType("Commit", commitFieldNames, []*types.Type{ types.EmptyStructType, types.MakeSetType(types.MakeRefType(types.MakeStructType("Commit", commitFieldNames, []*types.Type{ types.EmptyStructType, types.MakeSetType(types.MakeRefType(types.MakeCycleType(0))), types.MakeUnionType(types.NumberType, types.StringType), }))), types.StringType, }) assertTypeEquals(et3, at3) // Now commit a String with MetaInfo meta := types.NewStruct("Meta", types.StructData{"date": types.String("some date"), "number": types.Number(9)}) metaType := types.MakeStructType("Meta", []string{"date", "number"}, []*types.Type{types.StringType, types.NumberType}) assertTypeEquals(metaType, meta.Type()) commit4 := NewCommit(types.String("Hi"), types.NewSet(types.NewRef(commit2)), meta) at4 := commit4.Type() et4 := types.MakeStructType("Commit", commitFieldNames, []*types.Type{ metaType, types.MakeSetType(types.MakeRefType(types.MakeStructType("Commit", commitFieldNames, []*types.Type{ types.MakeUnionType(types.EmptyStructType, metaType), types.MakeSetType(types.MakeRefType(types.MakeCycleType(0))), types.MakeUnionType(types.NumberType, types.StringType), }))), types.StringType, }) assertTypeEquals(et4, at4) }
func makeAttr(mode uint32) types.Struct { now := time.Now() ctime := types.Number(float64(now.Unix()) + float64(now.Nanosecond())/1000000000) mtime := ctime user := fuse.CurrentOwner() gid := types.Number(float64(user.Gid)) uid := types.Number(float64(user.Uid)) return types.NewStructWithType(attrType, types.ValueSlice{ctime, gid, types.Number(mode), mtime, uid, types.NewMap()}) }
func buildListOfHeight(height int, vw types.ValueWriter) types.List { unique := 0 l := types.NewList(types.Number(unique), types.Number(unique+1)) unique += 2 for i := 0; i < height; i++ { r1, r2 := vw.WriteValue(types.Number(unique)), vw.WriteValue(l) unique++ l = types.NewList(r1, r2) } return l }
func TestCommitWithoutMetaField(t *testing.T) { assert := assert.New(t) metaCommit := types.NewStruct("Commit", types.StructData{ "value": types.Number(9), "parents": types.NewSet(), "meta": types.EmptyStruct, }) assert.True(IsCommitType(metaCommit.Type())) noMetaCommit := types.NewStruct("Commit", types.StructData{ "value": types.Number(9), "parents": types.NewSet(), }) assert.False(IsCommitType(noMetaCommit.Type())) }
func createStruct(i uint64) types.Value { return types.NewStructWithType(structType, map[string]types.Value{ "str": types.String(fmt.Sprintf("i am a 55 bytes............................%12d", i)), "num": types.Number(i), "bool": types.Bool(i%2 == 0), }) }
func (s *nomsShowTestSuite) TestNomsShow() { datasetName := "dsTest" str := test_util.CreateValueSpecString("ldb", s.LdbDir, datasetName) sp, err := spec.ParseDatasetSpec(str) d.Chk.NoError(err) ds, err := sp.Dataset() d.Chk.NoError(err) s1 := types.String("test string") r := writeTestData(ds, s1) s.Equal(res1, s.Run(main, []string{str})) spec1 := test_util.CreateValueSpecString("ldb", s.LdbDir, r.TargetHash().String()) s.Equal(res2, s.Run(main, []string{spec1})) ds, err = sp.Dataset() list := types.NewList(types.String("elem1"), types.Number(2), types.String("elem3")) r = writeTestData(ds, list) s.Equal(res3, s.Run(main, []string{str})) spec1 = test_util.CreateValueSpecString("ldb", s.LdbDir, r.TargetHash().String()) s.Equal(res4, s.Run(main, []string{spec1})) ds, err = sp.Dataset() _ = writeTestData(ds, s1) s.Equal(res5, s.Run(main, []string{str})) }
func main() { flag.Usage = func() { fmt.Fprintf(os.Stderr, "usage: %s [options] <dataset>\n", os.Args[0]) flag.PrintDefaults() } spec.RegisterDatabaseFlags() flag.Parse() if flag.NArg() != 1 { util.CheckError(errors.New("expected dataset arg")) } ds, err := spec.GetDataset(flag.Arg(0)) util.CheckError(err) defer ds.Database().Close() newVal := uint64(1) if lastVal, ok := ds.MaybeHeadValue(); ok { newVal = uint64(lastVal.(types.Number)) + 1 } _, err = ds.Commit(types.Number(newVal)) d.Exp.NoError(err) fmt.Println(newVal) }
func (s *testSuite) Setup() { db := s.Database() defer db.Close() ds := dataset.NewDataset(db, dsName) _, err := ds.CommitValue(types.Number(42)) s.NoError(err) }
func main() { flag.Usage = func() { fmt.Fprintf(os.Stderr, "usage: %s [options] <dataset>\n", os.Args[0]) flag.PrintDefaults() } spec.RegisterDatabaseFlags(flag.CommandLine) flag.Parse(true) if flag.NArg() != 1 { fmt.Fprintln(os.Stderr, "Missing required dataset argument") return } ds, err := spec.GetDataset(flag.Arg(0)) if err != nil { fmt.Fprintf(os.Stderr, "Could not create dataset: %s\n", err) return } defer ds.Database().Close() newVal := uint64(1) if lastVal, ok := ds.MaybeHeadValue(); ok { newVal = uint64(lastVal.(types.Number)) + 1 } _, err = ds.CommitValue(types.Number(newVal)) if err != nil { fmt.Fprintf(os.Stderr, "Error committing: %s\n", err) return } fmt.Println(newVal) }
func (s *testSuite) TestCSVImporterWithExternalHeader() { input, err := ioutil.TempFile(s.TempDir, "") d.Chk.NoError(err) defer input.Close() defer os.Remove(input.Name()) _, err = input.WriteString("7,8\n") d.Chk.NoError(err) setName := "csv" dataspec := test_util.CreateValueSpecString("ldb", s.LdbDir, setName) out := s.Run(main, []string{"-no-progress", "-column-types", "String,Number", "-header", "x,y", dataspec, input.Name()}) s.Equal("", out) cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) ds := dataset.NewDataset(datas.NewDatabase(cs), setName) defer ds.Database().Close() defer os.RemoveAll(s.LdbDir) l := ds.HeadValue().(types.List) s.Equal(uint64(1), l.Len()) v := l.Get(0) st := v.(types.Struct) s.Equal(types.String("7"), st.Get("x")) s.Equal(types.Number(8), st.Get("y")) }
func TestDefaults(t *testing.T) { assert := assert.New(t) ds := datas.NewDatabase(chunks.NewMemoryStore()) dataString := "42,,,\n" r := NewCSVReader(bytes.NewBufferString(dataString), ',') headers := []string{"A", "B", "C", "D"} kinds := KindSlice{types.NumberKind, types.NumberKind, types.BoolKind, types.StringKind} l, _ := ReadToList(r, "test", headers, kinds, ds) assert.Equal(uint64(1), l.Len()) row := l.Get(0).(types.Struct) assert.Equal(types.Number(42), row.Get("A")) assert.Equal(types.Number(0), row.Get("B")) assert.Equal(types.Bool(false), row.Get("C")) assert.Equal(types.String(""), row.Get("D")) }
func diffLists(dq *diffQueue, w io.Writer, p types.Path, v1, v2 types.List) { wroteHeader := false splices := v2.Diff(v1) for _, splice := range splices { if splice.SpRemoved == splice.SpAdded { for i := uint64(0); i < splice.SpRemoved; i++ { lastEl := v1.Get(splice.SpAt + i) newEl := v2.Get(splice.SpFrom + i) if canCompare(lastEl, newEl) { idx := types.Number(splice.SpAt + i) p1 := p.AddIndex(idx) dq.PushBack(diffInfo{p1, idx, lastEl, newEl}) } else { wroteHeader = writeHeader(w, wroteHeader, p) line(w, subPrefix, nil, v1.Get(splice.SpAt+i)) line(w, addPrefix, nil, v2.Get(splice.SpFrom+i)) } } } else { for i := uint64(0); i < splice.SpRemoved; i++ { wroteHeader = writeHeader(w, wroteHeader, p) line(w, subPrefix, nil, v1.Get(splice.SpAt+i)) } for i := uint64(0); i < splice.SpAdded; i++ { wroteHeader = writeHeader(w, wroteHeader, p) line(w, addPrefix, nil, v2.Get(splice.SpFrom+i)) } } } writeFooter(w, wroteHeader) }
func (s *nomsShowTestSuite) TestNomsShow() { datasetName := "dsTest" str := spec.CreateValueSpecString("ldb", s.LdbDir, datasetName) s1 := types.String("test string") r := writeTestData(str, s1) res, _ := s.Run(main, []string{"show", str}) s.Equal(res1, res) str1 := spec.CreateValueSpecString("ldb", s.LdbDir, "#"+r.TargetHash().String()) res, _ = s.Run(main, []string{"show", str1}) s.Equal(res2, res) list := types.NewList(types.String("elem1"), types.Number(2), types.String("elem3")) r = writeTestData(str, list) res, _ = s.Run(main, []string{"show", str}) test.EqualsIgnoreHashes(s.T(), res3, res) str1 = spec.CreateValueSpecString("ldb", s.LdbDir, "#"+r.TargetHash().String()) res, _ = s.Run(main, []string{"show", str1}) s.Equal(res4, res) _ = writeTestData(str, s1) res, _ = s.Run(main, []string{"show", str}) test.EqualsIgnoreHashes(s.T(), res5, res) }
func (suite *WalkTestSuite) SetupTest() { suite.vs = types.NewTestValueStore() suite.shouldSeeItem = types.String("zzz") suite.shouldSee = types.NewList(suite.shouldSeeItem) suite.deadValue = types.Number(0xDEADBEEF) suite.mustSkip = types.NewList(suite.deadValue) }
func createStruct(i uint64) types.Value { return types.NewStructWithType(structType, types.ValueSlice{ types.Bool(i%2 == 0), types.Number(i), types.String(fmt.Sprintf("i am a 55 bytes............................%12d", i)), }) }
func (s *testSuite) TestCSVImporterWithPipe() { input, err := ioutil.TempFile(s.TempDir, "") d.Chk.NoError(err) defer input.Close() defer os.Remove(input.Name()) _, err = input.WriteString("a|b\n1|2\n") d.Chk.NoError(err) setName := "csv" dataspec := spec.CreateValueSpecString("ldb", s.LdbDir, setName) stdout, stderr := s.Run(main, []string{"--no-progress", "--column-types", "String,Number", "--delimiter", "|", input.Name(), dataspec}) s.Equal("", stdout) s.Equal("", stderr) cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false) ds := dataset.NewDataset(datas.NewDatabase(cs), setName) defer ds.Database().Close() defer os.RemoveAll(s.LdbDir) l := ds.HeadValue().(types.List) s.Equal(uint64(1), l.Len()) v := l.Get(0) st := v.(types.Struct) s.Equal(types.String("1"), st.Get("a")) s.Equal(types.Number(2), st.Get("b")) }
func (s *testSuite) Teardown() { s.Equal("43\n", s.NodeOutput()) db := s.Database() defer db.Close() ds := dataset.NewDataset(db, dsName) s.True(ds.HeadValue().Equals(types.Number(43))) }
func TestEscapeFieldNames(t *testing.T) { assert := assert.New(t) ds := datas.NewDatabase(chunks.NewMemoryStore()) dataString := "1,2\n" r := NewCSVReader(bytes.NewBufferString(dataString), ',') headers := []string{"A A", "B"} kinds := KindSlice{types.NumberKind, types.NumberKind} l, _ := ReadToList(r, "test", headers, kinds, ds) assert.Equal(uint64(1), l.Len()) assert.Equal(types.Number(1), l.Get(0).(types.Struct).Get(types.EscapeStructField("A A"))) r = NewCSVReader(bytes.NewBufferString(dataString), ',') m := ReadToMap(r, headers, 1, kinds, ds) assert.Equal(uint64(1), l.Len()) assert.Equal(types.Number(1), m.Get(types.Number(2)).(types.Struct).Get(types.EscapeStructField("A A"))) }
func TestPullWithMeta(t *testing.T) { assert := assert.New(t) sink := createTestDataset("sink") source := createTestDataset("source") v1 := types.Number(1) m1 := types.NewStruct("Meta", types.StructData{ "name": types.String("one"), }) source, err := source.Commit(v1, CommitOptions{Meta: m1}) assert.NoError(err) v2 := types.Number(2) m2 := types.NewStruct("Meta", types.StructData{ "name": types.String("two"), }) source, err = source.Commit(v2, CommitOptions{Meta: m2}) assert.NoError(err) h2 := source.Head() v3 := types.Number(3) m3 := types.NewStruct("Meta", types.StructData{ "name": types.String("three"), }) source, err = source.Commit(v3, CommitOptions{Meta: m3}) assert.NoError(err) v4 := types.Number(4) m4 := types.NewStruct("Meta", types.StructData{ "name": types.String("three"), }) source, err = source.Commit(v4, CommitOptions{Meta: m4}) assert.NoError(err) h4 := source.Head() sink, err = sink.Pull(source.Database(), types.NewRef(h2), 1, nil) assert.NoError(err) sink, err = sink.Pull(source.Database(), types.NewRef(h4), 1, nil) assert.NoError(err) assert.True(source.Head().Equals(sink.Head())) }
func testCommitInResults(s *nomsLogTestSuite, str string, i int) { ds, err := spec.GetDataset(str) s.NoError(err) ds, err = ds.CommitValue(types.Number(i)) s.NoError(err) commit := ds.Head() ds.Database().Close() res, _ := s.Run(main, []string{"log", str}) s.Contains(res, commit.Hash().String()) }
func diffLists(w io.Writer, p types.Path, v1, v2 types.List) (err error) { spliceChan := make(chan types.Splice) stopChan := make(chan struct{}, 1) // buffer size of 1, so this won't block if diff already finished go func() { v2.Diff(v1, spliceChan, stopChan) close(spliceChan) }() wroteHdr := false for splice := range spliceChan { if err != nil { break } if splice.SpRemoved == splice.SpAdded { // Heuristic: list only has modifications. for i := uint64(0); i < splice.SpRemoved; i++ { lastEl := v1.Get(splice.SpAt + i) newEl := v2.Get(splice.SpFrom + i) if shouldDescend(lastEl, newEl) { idx := types.Number(splice.SpAt + i) writeFooter(w, &wroteHdr) err = diff(w, p.AddIndex(idx), idx, lastEl, newEl) } else { writeHeader(w, p, &wroteHdr) line(w, DEL, nil, v1.Get(splice.SpAt+i)) err = line(w, ADD, nil, v2.Get(splice.SpFrom+i)) } } continue } // Heuristic: list only has additions/removals. for i := uint64(0); i < splice.SpRemoved && err == nil; i++ { writeHeader(w, p, &wroteHdr) err = line(w, DEL, nil, v1.Get(splice.SpAt+i)) } for i := uint64(0); i < splice.SpAdded && err == nil; i++ { writeHeader(w, p, &wroteHdr) err = line(w, ADD, nil, v2.Get(splice.SpFrom+i)) } } err = writeFooter(w, &wroteHdr) if err != nil { stopChan <- struct{}{} // Wait for diff to stop. for range spliceChan { } } return }
func addPerson(ds dataset.Dataset) { if flag.NArg() != 4 { fmt.Fprintln(os.Stderr, "Not enough arguments for command add-person") return } id, err := strconv.ParseUint(flag.Arg(1), 10, 64) if err != nil { fmt.Fprintf(os.Stderr, "Invalid person-id: %s", flag.Arg(1)) return } np := types.NewStruct("Person", map[string]types.Value{ "id": types.Number(id), "name": types.String(flag.Arg(2)), "title": types.String(flag.Arg(3)), }) ds.Commit(getPersons(ds).Set(types.Number(id), np)) }
func testCommitInResults(s *nomsShowTestSuite, str string, i int) { sp, err := spec.ParseDatasetSpec(str) s.NoError(err) ds, err := sp.Dataset() s.NoError(err) ds, err = ds.Commit(types.Number(i)) s.NoError(err) commit := ds.Head() ds.Database().Close() s.Contains(s.Run(main, []string{str}), commit.Hash().String()) }