func testFileConsistency(t *testing.T, bs chunk.SplitterGen, nbytes int) { should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, bs(read)) if err != nil { t.Fatal(err) } r, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(r) if err != nil { t.Fatal(err) } err = arrComp(out, should) if err != nil { t.Fatal(err) } }
func TestBuilderConsistency(t *testing.T) { nbytes := 100000 buf := new(bytes.Buffer) io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes)) should := dup(buf.Bytes()) dagserv := mdtest.Mock() nd, err := buildTestDag(dagserv, chunk.DefaultSplitter(buf)) if err != nil { t.Fatal(err) } r, err := uio.NewDagReader(context.Background(), nd, dagserv) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(r) if err != nil { t.Fatal(err) } err = arrComp(out, should) if err != nil { t.Fatal(err) } }
func BenchmarkDagmodWrite(b *testing.B) { b.StopTimer() dserv := testu.GetDAGServ() n := testu.GetEmptyNode(b, dserv) ctx, cancel := context.WithCancel(context.Background()) defer cancel() wrsize := 4096 dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) if err != nil { b.Fatal(err) } buf := make([]byte, b.N*wrsize) u.NewTimeSeededRand().Read(buf) b.StartTimer() b.SetBytes(int64(wrsize)) for i := 0; i < b.N; i++ { n, err := dagmod.Write(buf[i*wrsize : (i+1)*wrsize]) if err != nil { b.Fatal(err) } if n != wrsize { b.Fatal("Wrote bad size") } } }
func TestFetchGraph(t *testing.T) { var dservs []DAGService bsis := bstest.Mocks(2) for _, bsi := range bsis { dservs = append(dservs, NewDAGService(bsi)) } read := io.LimitReader(u.NewTimeSeededRand(), 1024*32) root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512)) if err != nil { t.Fatal(err) } err = FetchGraph(context.TODO(), root.Cid(), dservs[1]) if err != nil { t.Fatal(err) } // create an offline dagstore and ensure all blocks were fetched bs := bserv.New(bsis[1].Blockstore(), offline.Exchange(bsis[1].Blockstore())) offline_ds := NewDAGService(bs) err = EnumerateChildren(context.Background(), offline_ds, root.Cid(), func(_ *cid.Cid) bool { return true }, false) if err != nil { t.Fatal(err) } }
func randNode() (*merkledag.ProtoNode, key.Key) { node := new(merkledag.ProtoNode) node.SetData(make([]byte, 32)) util.NewTimeSeededRand().Read(node.Data()) k := node.Key() return node, k }
func TestEnumerateChildren(t *testing.T) { bsi := bstest.Mocks(1) ds := NewDAGService(bsi[0]) read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024) root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512)) if err != nil { t.Fatal(err) } set := cid.NewSet() err = EnumerateChildren(context.Background(), ds, root.Cid(), set.Visit, false) if err != nil { t.Fatal(err) } var traverse func(n node.Node) traverse = func(n node.Node) { // traverse dag and check for _, lnk := range n.Links() { c := lnk.Cid if !set.Has(c) { t.Fatal("missing key in set! ", lnk.Cid.String()) } child, err := ds.Get(context.Background(), c) if err != nil { t.Fatal(err) } traverse(child) } } traverse(root) }
func TestSeekEndSingleBlockFile(t *testing.T) { nbytes := int64(100) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 5000)) if err != nil { t.Fatal(err) } rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } seeked, err := rs.Seek(0, os.SEEK_END) if err != nil { t.Fatal(err) } if seeked != nbytes { t.Fatal("Failed to seek to end") } }
func randBuf(t *testing.T, size int) []byte { buf := make([]byte, size) if _, err := u.NewTimeSeededRand().Read(buf); err != nil { t.Fatal("failed to read enough randomness") } return buf }
func TestIndirectBlocks(t *testing.T) { splitter := chunk.SizeSplitterGen(512) nbytes := 1024 * 1024 buf := make([]byte, nbytes) u.NewTimeSeededRand().Read(buf) read := bytes.NewReader(buf) ds := mdtest.Mock() dag, err := buildTestDag(ds, splitter(read)) if err != nil { t.Fatal(err) } reader, err := uio.NewDagReader(context.Background(), dag, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(reader) if err != nil { t.Fatal(err) } if !bytes.Equal(out, buf) { t.Fatal("Not equal!") } }
func TestBalancedDag(t *testing.T) { ds := mdtest.Mock() buf := make([]byte, 10000) u.NewTimeSeededRand().Read(buf) r := bytes.NewReader(buf) nd, err := BuildDagFromReader(ds, chunk.DefaultSplitter(r)) if err != nil { t.Fatal(err) } dr, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(dr) if err != nil { t.Fatal(err) } if !bytes.Equal(out, buf) { t.Fatal("bad read") } }
func RandCidV0() (*cid.Cid, error) { buf := make([]byte, 16) if _, err := io.ReadFull(u.NewTimeSeededRand(), buf); err != nil { return &cid.Cid{}, err } return cid.NewCidV0(buf), nil }
// RandPeerID generates random "valid" peer IDs. it does not NEED to generate // keys because it is as if we lost the key right away. fine to read randomness // and hash it. to generate proper keys and corresponding PeerID, use: // sk, pk, _ := testutil.RandKeyPair() // id, _ := peer.IDFromPublicKey(pk) func RandPeerID() (peer.ID, error) { buf := make([]byte, 16) if _, err := io.ReadFull(u.NewTimeSeededRand(), buf); err != nil { return "", err } h := u.Hash(buf) return peer.ID(h), nil }
func getTrickleDag(t testing.TB, size int64, blksize int64) (node.Node, dag.DAGService) { ds := mdtest.Mock() r := io.LimitReader(u.NewTimeSeededRand(), size) nd, err := BuildTrickleDagFromReader(ds, chunk.NewSizeSplitter(r, blksize)) if err != nil { t.Fatal(err) } return nd, ds }
func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64) ([]byte, node.Node) { in := io.LimitReader(u.NewTimeSeededRand(), size) buf, err := ioutil.ReadAll(in) if err != nil { t.Fatal(err) } node := GetNode(t, dserv, buf) return buf, node }
func randObj(t *testing.T, nd *core.IpfsNode, size int64) (node.Node, []byte) { buf := make([]byte, size) u.NewTimeSeededRand().Read(buf) read := bytes.NewReader(buf) obj, err := importer.BuildTrickleDagFromReader(nd.DAG, chunk.DefaultSplitter(read)) if err != nil { t.Fatal(err) } return obj, buf }
func getTestDag(t *testing.T, ds dag.DAGService, size int64, blksize int64) (*dag.ProtoNode, []byte) { data := make([]byte, size) u.NewTimeSeededRand().Read(data) r := bytes.NewReader(data) nd, err := buildTestDag(ds, chunk.NewSizeSplitter(r, blksize)) if err != nil { t.Fatal(err) } return nd, data }
func TestMultiWrite(t *testing.T) { dserv := testu.GetDAGServ() n := testu.GetEmptyNode(t, dserv) ctx, cancel := context.WithCancel(context.Background()) defer cancel() dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) if err != nil { t.Fatal(err) } data := make([]byte, 4000) u.NewTimeSeededRand().Read(data) for i := 0; i < len(data); i++ { n, err := dagmod.WriteAt(data[i:i+1], int64(i)) if err != nil { t.Fatal(err) } if n != 1 { t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)") } size, err := dagmod.Size() if err != nil { t.Fatal(err) } if size != int64(i+1) { t.Fatal("Size was reported incorrectly") } } nd, err := dagmod.GetNode() if err != nil { t.Fatal(err) } read, err := uio.NewDagReader(context.Background(), nd, dserv) if err != nil { t.Fatal(err) } rbuf, err := ioutil.ReadAll(read) if err != nil { t.Fatal(err) } err = testu.ArrComp(rbuf, data) if err != nil { t.Fatal(err) } }
// This test appends one byte at a time to an empty file func TestMultipleAppends(t *testing.T) { ds := mdtest.Mock() // TODO: fix small size appends and make this number bigger nbytes := int64(1000) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(nil) nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500)) if err != nil { t.Fatal(err) } dbp := &h.DagBuilderParams{ Dagserv: ds, Maxlinks: 4, } spl := chunk.SizeSplitterGen(500) ctx := context.Background() for i := 0; i < len(should); i++ { nnode, err := TrickleAppend(ctx, nd, dbp.New(spl(bytes.NewReader(should[i:i+1])))) if err != nil { t.Fatal(err) } err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat) if err != nil { t.Fatal(err) } fread, err := uio.NewDagReader(ctx, nnode, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(fread) if err != nil { t.Fatal(err) } err = arrComp(out, should[:i+1]) if err != nil { t.Fatal(err) } } }
func TestAppend(t *testing.T) { nbytes := int64(128 * 1024) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) // Reader for half the bytes read := bytes.NewReader(should[:nbytes/2]) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500)) if err != nil { t.Fatal(err) } dbp := &h.DagBuilderParams{ Dagserv: ds, Maxlinks: h.DefaultLinksPerBlock, } r := bytes.NewReader(should[nbytes/2:]) ctx := context.Background() nnode, err := TrickleAppend(ctx, nd, dbp.New(chunk.NewSizeSplitter(r, 500))) if err != nil { t.Fatal(err) } err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat) if err != nil { t.Fatal(err) } fread, err := uio.NewDagReader(ctx, nnode, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(fread) if err != nil { t.Fatal(err) } err = arrComp(out, should) if err != nil { t.Fatal(err) } }
func TestSeekPastEndWrite(t *testing.T) { dserv := testu.GetDAGServ() n := testu.GetEmptyNode(t, dserv) ctx, cancel := context.WithCancel(context.Background()) defer cancel() dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) if err != nil { t.Fatal(err) } buf := make([]byte, 5000) u.NewTimeSeededRand().Read(buf[2500:]) nseek, err := dagmod.Seek(2500, os.SEEK_SET) if err != nil { t.Fatal(err) } if nseek != 2500 { t.Fatal("failed to seek") } wrote, err := dagmod.Write(buf[2500:]) if err != nil { t.Fatal(err) } if wrote != 2500 { t.Fatal("incorrect write amount") } _, err = dagmod.Seek(0, os.SEEK_SET) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(dagmod) if err != nil { t.Fatal(err) } if err = testu.ArrComp(out, buf); err != nil { t.Fatal(err) } }
func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier) []byte { newdata := make([]byte, size) r := u.NewTimeSeededRand() r.Read(newdata) if size+beg > uint64(len(orig)) { orig = append(orig, make([]byte, (size+beg)-uint64(len(orig)))...) } copy(orig[beg:], newdata) nmod, err := dm.WriteAt(newdata, int64(beg)) if err != nil { t.Fatal(err) } if nmod != int(size) { t.Fatalf("Mod length not correct! %d != %d", nmod, size) } nd, err := dm.GetNode() if err != nil { t.Fatal(err) } err = trickle.VerifyTrickleDagStructure(nd, dm.dagserv, h.DefaultLinksPerBlock, 4) if err != nil { t.Fatal(err) } rd, err := uio.NewDagReader(context.Background(), nd, dm.dagserv) if err != nil { t.Fatal(err) } after, err := ioutil.ReadAll(rd) if err != nil { t.Fatal(err) } err = testu.ArrComp(after, orig) if err != nil { t.Fatal(err) } return orig }
func TestSeekingStress(t *testing.T) { nbytes := int64(1024 * 1024) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 1000)) if err != nil { t.Fatal(err) } rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } testbuf := make([]byte, nbytes) for i := 0; i < 50; i++ { offset := mrand.Intn(int(nbytes)) l := int(nbytes) - offset n, err := rs.Seek(int64(offset), os.SEEK_SET) if err != nil { t.Fatal(err) } if n != int64(offset) { t.Fatal("Seek failed to move to correct position") } nread, err := rs.Read(testbuf[:l]) if err != nil { t.Fatal(err) } if nread != l { t.Fatal("Failed to read enough bytes") } err = arrComp(testbuf[:l], should[offset:offset+l]) if err != nil { t.Fatal(err) } } }
func TestSeekToAlmostBegin(t *testing.T) { nbytes := int64(10 * 1024) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500)) if err != nil { t.Fatal(err) } rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } n, err := io.CopyN(ioutil.Discard, rs, 1024*4) if err != nil { t.Fatal(err) } if n != 4096 { t.Fatal("Copy didnt copy enough bytes") } seeked, err := rs.Seek(1, os.SEEK_SET) if err != nil { t.Fatal(err) } if seeked != 1 { t.Fatal("Failed to seek to almost beginning") } out, err := ioutil.ReadAll(rs) if err != nil { t.Fatal(err) } err = arrComp(out, should[1:]) if err != nil { t.Fatal(err) } }
func TestSeekingConsistency(t *testing.T) { nbytes := int64(128 * 1024) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500)) if err != nil { t.Fatal(err) } rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } out := make([]byte, nbytes) for coff := nbytes - 4096; coff >= 0; coff -= 4096 { t.Log(coff) n, err := rs.Seek(coff, os.SEEK_SET) if err != nil { t.Fatal(err) } if n != coff { t.Fatal("wasnt able to seek to the right position") } nread, err := rs.Read(out[coff : coff+4096]) if err != nil { t.Fatal(err) } if nread != 4096 { t.Fatal("didnt read the correct number of bytes") } } err = arrComp(out, should) if err != nil { t.Fatal(err) } }
func TestWriteNewFile(t *testing.T) { dserv := testu.GetDAGServ() n := testu.GetEmptyNode(t, dserv) ctx, cancel := context.WithCancel(context.Background()) defer cancel() dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) if err != nil { t.Fatal(err) } towrite := make([]byte, 2000) u.NewTimeSeededRand().Read(towrite) nw, err := dagmod.Write(towrite) if err != nil { t.Fatal(err) } if nw != len(towrite) { t.Fatal("Wrote wrong amount") } nd, err := dagmod.GetNode() if err != nil { t.Fatal(err) } read, err := uio.NewDagReader(ctx, nd, dserv) if err != nil { t.Fatal(err) } data, err := ioutil.ReadAll(read) if err != nil { t.Fatal(err) } if err := testu.ArrComp(data, towrite); err != nil { t.Fatal(err) } }
func TestLargeWriteChunks(t *testing.T) { dserv := testu.GetDAGServ() n := testu.GetEmptyNode(t, dserv) ctx, cancel := context.WithCancel(context.Background()) defer cancel() dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) if err != nil { t.Fatal(err) } wrsize := 1000 datasize := 10000000 data := make([]byte, datasize) u.NewTimeSeededRand().Read(data) for i := 0; i < datasize/wrsize; i++ { n, err := dagmod.WriteAt(data[i*wrsize:(i+1)*wrsize], int64(i*wrsize)) if err != nil { t.Fatal(err) } if n != wrsize { t.Fatal("failed to write buffer") } } out, err := ioutil.ReadAll(dagmod) if err != nil { t.Fatal(err) } if err = testu.ArrComp(out, data); err != nil { t.Fatal(err) } }
func TestSeekingBasic(t *testing.T) { nbytes := int64(10 * 1024) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 512)) if err != nil { t.Fatal(err) } rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } start := int64(4000) n, err := rs.Seek(start, os.SEEK_SET) if err != nil { t.Fatal(err) } if n != start { t.Fatal("Failed to seek to correct offset") } out, err := ioutil.ReadAll(rs) if err != nil { t.Fatal(err) } err = arrComp(out, should[start:]) if err != nil { t.Fatal(err) } }
func TestMetadata(t *testing.T) { ctx := context.Background() // Make some random node ds := getDagserv(t) data := make([]byte, 1000) u.NewTimeSeededRand().Read(data) r := bytes.NewReader(data) nd, err := importer.BuildDagFromReader(ds, chunk.DefaultSplitter(r)) if err != nil { t.Fatal(err) } c := nd.Cid() m := new(ft.Metadata) m.MimeType = "THIS IS A TEST" // Such effort, many compromise ipfsnode := &core.IpfsNode{DAG: ds} mdk, err := AddMetadataTo(ipfsnode, c.String(), m) if err != nil { t.Fatal(err) } rec, err := Metadata(ipfsnode, mdk) if err != nil { t.Fatal(err) } if rec.MimeType != m.MimeType { t.Fatalf("something went wrong in conversion: '%s' != '%s'", rec.MimeType, m.MimeType) } cdk, err := cid.Decode(mdk) if err != nil { t.Fatal(err) } retnode, err := ds.Get(ctx, cdk) if err != nil { t.Fatal(err) } rtnpb, ok := retnode.(*merkledag.ProtoNode) if !ok { t.Fatal("expected protobuf node") } ndr, err := uio.NewDagReader(ctx, rtnpb, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(ndr) if err != nil { t.Fatal(err) } if !bytes.Equal(out, data) { t.Fatal("read incorrect data") } }
func randNode() *merkledag.ProtoNode { node := new(merkledag.ProtoNode) node.SetData(make([]byte, 32)) util.NewTimeSeededRand().Read(node.Data()) return node }
func getRandFile(t *testing.T, ds dag.DAGService, size int64) node.Node { r := io.LimitReader(u.NewTimeSeededRand(), size) return fileNodeFromReader(t, ds, r) }