func TestFetchGraph(t *testing.T) {
var dservs []DAGService
bsis := bstest.Mocks(2)
for _, bsi := range bsis {
dservs = append(dservs, NewDAGService(bsi))
}
read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512))
if err != nil {
t.Fatal(err)
}
err = FetchGraph(context.TODO(), root, dservs[1])
if err != nil {
t.Fatal(err)
}
// create an offline dagstore and ensure all blocks were fetched
bs := bserv.New(bsis[1].Blockstore, offline.Exchange(bsis[1].Blockstore))
offline_ds := NewDAGService(bs)
ks := key.NewKeySet()
err = EnumerateChildren(context.Background(), offline_ds, root, ks, false)
if err != nil {
t.Fatal(err)
}
}
func TestEnumerateChildren(t *testing.T) {
bsi := bstest.Mocks(1)
ds := NewDAGService(bsi[0])
read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512))
if err != nil {
t.Fatal(err)
}
ks := key.NewKeySet()
err = EnumerateChildren(context.Background(), ds, root, ks, false)
if err != nil {
t.Fatal(err)
}
var traverse func(n *Node)
traverse = func(n *Node) {
// traverse dag and check
for _, lnk := range n.Links {
k := key.Key(lnk.Hash)
if !ks.Has(k) {
t.Fatal("missing key in set!")
}
child, err := ds.Get(context.Background(), k)
if err != nil {
t.Fatal(err)
}
traverse(child)
}
}
traverse(root)
}
func TestEnumerateChildren(t *testing.T) {
bsi := bstest.Mocks(1)
ds := NewDAGService(bsi[0])
read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512))
if err != nil {
t.Fatal(err)
}
set := cid.NewSet()
err = EnumerateChildren(context.Background(), ds, root.Cid(), set.Visit, false)
if err != nil {
t.Fatal(err)
}
var traverse func(n node.Node)
traverse = func(n node.Node) {
// traverse dag and check
for _, lnk := range n.Links() {
c := lnk.Cid
if !set.Has(c) {
t.Fatal("missing key in set! ", lnk.Cid.String())
}
child, err := ds.Get(context.Background(), c)
if err != nil {
t.Fatal(err)
}
traverse(child)
}
}
traverse(root)
}
func benchAddSize(n *core.IpfsNode, cfg *BenchCfg, size int64) error {
f := func(b *testing.B) {
b.SetBytes(size)
for i := 0; i < b.N; i++ {
r := io.LimitReader(randbo.New(), size)
spl := chunk.NewSizeSplitter(r, cfg.Blocksize)
_, err := importer.BuildDagFromReader(n.DAG, spl, nil)
if err != nil {
fmt.Printf("ERRROR: ", err)
b.Fatal(err)
}
}
}
br := testing.Benchmark(f)
bs := humanize.IBytes(uint64(size))
fmt.Printf("Add File (%s):\t%s\n", bs, br)
err := cr.GarbageCollect(n, context.Background())
if err != nil {
return err
}
return nil
}
func TestSeekEndSingleBlockFile(t *testing.T) {
nbytes := int64(100)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 5000))
if err != nil {
t.Fatal(err)
}
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
seeked, err := rs.Seek(0, os.SEEK_END)
if err != nil {
t.Fatal(err)
}
if seeked != nbytes {
t.Fatal("Failed to seek to end")
}
}
func TestAppend(t *testing.T) {
nbytes := int64(128 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
// Reader for half the bytes
read := bytes.NewReader(should[:nbytes/2])
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
if err != nil {
t.Fatal(err)
}
dbp := &h.DagBuilderParams{
Dagserv: ds,
Maxlinks: h.DefaultLinksPerBlock,
}
r := bytes.NewReader(should[nbytes/2:])
blks, errs := chunk.Chan(chunk.NewSizeSplitter(r, 500))
ctx := context.Background()
nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
if err != nil {
t.Fatal(err)
}
err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat)
if err != nil {
t.Fatal(err)
}
fread, err := uio.NewDagReader(ctx, nnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(fread)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func getTrickleDag(t testing.TB, size int64, blksize int64) (*dag.Node, dag.DAGService) {
ds := mdtest.Mock()
r := io.LimitReader(u.NewTimeSeededRand(), size)
nd, err := BuildTrickleDagFromReader(ds, chunk.NewSizeSplitter(r, blksize), nil)
if err != nil {
t.Fatal(err)
}
return nd, ds
}
func getTestDag(t *testing.T, ds dag.DAGService, size int64, blksize int64) (*dag.Node, []byte) {
data := make([]byte, size)
u.NewTimeSeededRand().Read(data)
r := bytes.NewReader(data)
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(r, blksize))
if err != nil {
t.Fatal(err)
}
return nd, data
}
// This test appends one byte at a time to an empty file
func TestMultipleAppends(t *testing.T) {
ds := mdtest.Mock()
// TODO: fix small size appends and make this number bigger
nbytes := int64(1000)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(nil)
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
if err != nil {
t.Fatal(err)
}
dbp := &h.DagBuilderParams{
Dagserv: ds,
Maxlinks: 4,
}
spl := chunk.SizeSplitterGen(500)
ctx := context.Background()
for i := 0; i < len(should); i++ {
blks, errs := chunk.Chan(spl(bytes.NewReader(should[i : i+1])))
nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
if err != nil {
t.Fatal(err)
}
err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat)
if err != nil {
t.Fatal(err)
}
fread, err := uio.NewDagReader(ctx, nnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(fread)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should[:i+1])
if err != nil {
t.Fatal(err)
}
}
}
// expandSparse grows the file with zero blocks of 4096
// A small blocksize is chosen to aid in deduplication
func (dm *DagModifier) expandSparse(size int64) error {
r := io.LimitReader(zeroReader{}, size)
spl := chunk.NewSizeSplitter(r, 4096)
nnode, err := dm.appendData(dm.curNode, spl)
if err != nil {
return err
}
_, err = dm.dagserv.Add(nnode)
if err != nil {
return err
}
dm.curNode = nnode
return nil
}
// Builds a DAG from the given file, writing created blocks to disk as they are
// created
func BuildDagFromFile(fpath string, ds dag.DAGService, mp pin.ManualPinner) (*dag.Node, error) {
stat, err := os.Stat(fpath)
if err != nil {
return nil, err
}
if stat.IsDir() {
return nil, fmt.Errorf("`%s` is a directory", fpath)
}
f, err := os.Open(fpath)
if err != nil {
return nil, err
}
defer f.Close()
return BuildDagFromReader(ds, chunk.NewSizeSplitter(f, chunk.DefaultBlockSize), BasicPinnerCB(mp))
}
// Add builds a merkledag from the a reader, pinning all objects to the local
// datastore. Returns a key representing the root node.
func Add(n *core.IpfsNode, r io.Reader) (string, error) {
// TODO more attractive function signature importer.BuildDagFromReader
dagNode, err := importer.BuildDagFromReader(
n.DAG,
chunk.NewSizeSplitter(r, chunk.DefaultBlockSize),
importer.BasicPinnerCB(n.Pinning.GetManual()),
)
if err != nil {
return "", err
}
k, err := dagNode.Key()
if err != nil {
return "", err
}
return k.String(), nil
}
// Builds a DAG from the given file, writing created blocks to disk as they are
// created
func BuildDagFromFile(fpath string, ds dag.DAGService) (*dag.Node, error) {
stat, err := os.Lstat(fpath)
if err != nil {
return nil, err
}
if stat.IsDir() {
return nil, fmt.Errorf("`%s` is a directory", fpath)
}
f, err := files.NewSerialFile(fpath, fpath, false, stat)
if err != nil {
return nil, err
}
defer f.Close()
return BuildDagFromReader(ds, chunk.NewSizeSplitter(f, chunk.DefaultBlockSize))
}
func TestSeekingStress(t *testing.T) {
nbytes := int64(1024 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 1000))
if err != nil {
t.Fatal(err)
}
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
testbuf := make([]byte, nbytes)
for i := 0; i < 50; i++ {
offset := mrand.Intn(int(nbytes))
l := int(nbytes) - offset
n, err := rs.Seek(int64(offset), os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if n != int64(offset) {
t.Fatal("Seek failed to move to correct position")
}
nread, err := rs.Read(testbuf[:l])
if err != nil {
t.Fatal(err)
}
if nread != l {
t.Fatal("Failed to read enough bytes")
}
err = arrComp(testbuf[:l], should[offset:offset+l])
if err != nil {
t.Fatal(err)
}
}
}
func TestSeekToAlmostBegin(t *testing.T) {
nbytes := int64(10 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
if err != nil {
t.Fatal(err)
}
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
if err != nil {
t.Fatal(err)
}
if n != 4096 {
t.Fatal("Copy didnt copy enough bytes")
}
seeked, err := rs.Seek(1, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if seeked != 1 {
t.Fatal("Failed to seek to almost beginning")
}
out, err := ioutil.ReadAll(rs)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should[1:])
if err != nil {
t.Fatal(err)
}
}
func TestSeekingConsistency(t *testing.T) {
nbytes := int64(128 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
if err != nil {
t.Fatal(err)
}
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
out := make([]byte, nbytes)
for coff := nbytes - 4096; coff >= 0; coff -= 4096 {
t.Log(coff)
n, err := rs.Seek(coff, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if n != coff {
t.Fatal("wasnt able to seek to the right position")
}
nread, err := rs.Read(out[coff : coff+4096])
if err != nil {
t.Fatal(err)
}
if nread != 4096 {
t.Fatal("didnt read the correct number of bytes")
}
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
// Chunks file into blocks and adds each block to exchange
func putFileCmd(nodes []int, file string) error {
file = normalizePath(file)
reader, err := os.Open(file)
if err != nil {
return fmt.Errorf("Line %d: Failed to open file '%s'.", currLine, file)
}
bsize, err := strconv.Atoi(config["block_size"])
if err != nil {
return fmt.Errorf("Invalid block size in config.")
}
chunks := splitter.NewSizeSplitter(reader, int64(bsize))
files[file] = make([]key.Key, 0)
// waitgroup for chunks
var wg sync.WaitGroup
for {
chunk, err := chunks.NextBytes()
if err != nil {
if err == io.EOF {
break
}
return err
}
wg.Add(1)
block := blocks.NewBlock(chunk)
files[file] = append(files[file], block.Key())
go func(block *blocks.Block) {
err := putCmd(nodes, block)
check(err)
wg.Done()
}(block)
}
wg.Wait()
return nil
}
func TestSeekingBasic(t *testing.T) {
nbytes := int64(10 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 512))
if err != nil {
t.Fatal(err)
}
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
start := int64(4000)
n, err := rs.Seek(start, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if n != start {
t.Fatal("Failed to seek to correct offset")
}
out, err := ioutil.ReadAll(rs)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should[start:])
if err != nil {
t.Fatal(err)
}
}
func sizeSplitterGen(size int64) chunk.SplitterGen {
return func(r io.Reader) chunk.Splitter {
return chunk.NewSizeSplitter(r, size)
}
}
func runBatchFetchTest(t *testing.T, read io.Reader) {
ctx := context.Background()
var dagservs []DAGService
for _, bsi := range bstest.Mocks(5) {
dagservs = append(dagservs, NewDAGService(bsi))
}
spl := chunk.NewSizeSplitter(read, 512)
root, err := imp.BuildDagFromReader(dagservs[0], spl)
if err != nil {
t.Fatal(err)
}
t.Log("finished setup.")
dagr, err := uio.NewDagReader(ctx, root, dagservs[0])
if err != nil {
t.Fatal(err)
}
expected, err := ioutil.ReadAll(dagr)
if err != nil {
t.Fatal(err)
}
_, err = dagservs[0].Add(root)
if err != nil {
t.Fatal(err)
}
t.Log("Added file to first node.")
k, err := root.Key()
if err != nil {
t.Fatal(err)
}
wg := sync.WaitGroup{}
errs := make(chan error)
for i := 1; i < len(dagservs); i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
first, err := dagservs[i].Get(ctx, k)
if err != nil {
errs <- err
}
fmt.Println("Got first node back.")
read, err := uio.NewDagReader(ctx, first, dagservs[i])
if err != nil {
errs <- err
}
datagot, err := ioutil.ReadAll(read)
if err != nil {
errs <- err
}
if !bytes.Equal(datagot, expected) {
errs <- errors.New("Got bad data back!")
}
}(i)
}
go func() {
wg.Wait()
close(errs)
}()
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
}
func runBatchFetchTest(t *testing.T, read io.Reader) {
var dagservs []DAGService
for _, bsi := range bstest.Mocks(t, 5) {
dagservs = append(dagservs, NewDAGService(bsi))
}
spl := chunk.NewSizeSplitter(read, 512)
root, err := imp.BuildDagFromReader(dagservs[0], spl, nil)
if err != nil {
t.Fatal(err)
}
t.Log("finished setup.")
dagr, err := uio.NewDagReader(context.TODO(), root, dagservs[0])
if err != nil {
t.Fatal(err)
}
expected, err := ioutil.ReadAll(dagr)
if err != nil {
t.Fatal(err)
}
err = dagservs[0].AddRecursive(root)
if err != nil {
t.Fatal(err)
}
t.Log("Added file to first node.")
k, err := root.Key()
if err != nil {
t.Fatal(err)
}
wg := sync.WaitGroup{}
for i := 1; i < len(dagservs); i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
first, err := dagservs[i].Get(context.Background(), k)
if err != nil {
t.Fatal(err)
}
fmt.Println("Got first node back.")
read, err := uio.NewDagReader(context.TODO(), first, dagservs[i])
if err != nil {
t.Fatal(err)
}
datagot, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(datagot, expected) {
t.Fatal("Got bad data back!")
}
}(i)
}
wg.Wait()
}