func TestFetchGraph(t *testing.T) {
var dservs []DAGService
bsis := bstest.Mocks(2)
for _, bsi := range bsis {
dservs = append(dservs, NewDAGService(bsi))
}
read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512))
if err != nil {
t.Fatal(err)
}
err = FetchGraph(context.TODO(), root, dservs[1])
if err != nil {
t.Fatal(err)
}
// create an offline dagstore and ensure all blocks were fetched
bs := bserv.New(bsis[1].Blockstore, offline.Exchange(bsis[1].Blockstore))
offline_ds := NewDAGService(bs)
ks := key.NewKeySet()
err = EnumerateChildren(context.Background(), offline_ds, root, ks, false)
if err != nil {
t.Fatal(err)
}
}
func TestEnumerateChildren(t *testing.T) {
bsi := bstest.Mocks(1)
ds := NewDAGService(bsi[0])
read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512))
if err != nil {
t.Fatal(err)
}
ks := key.NewKeySet()
err = EnumerateChildren(context.Background(), ds, root, ks, false)
if err != nil {
t.Fatal(err)
}
var traverse func(n *Node)
traverse = func(n *Node) {
// traverse dag and check
for _, lnk := range n.Links {
k := key.Key(lnk.Hash)
if !ks.Has(k) {
t.Fatal("missing key in set!")
}
child, err := ds.Get(context.Background(), k)
if err != nil {
t.Fatal(err)
}
traverse(child)
}
}
traverse(root)
}
func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
rall, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
r = bytes.NewReader(rall)
tr := tar.NewReader(r)
root := new(dag.Node)
root.Data = []byte("ipfs/tar")
e := dagutil.NewDagEditor(root, ds)
for {
h, err := tr.Next()
if err != nil {
if err == io.EOF {
break
}
return nil, err
}
header := new(dag.Node)
headerBytes, err := marshalHeader(h)
if err != nil {
return nil, err
}
header.Data = headerBytes
if h.Size > 0 {
spl := chunk.NewRabin(tr, uint64(chunk.DefaultBlockSize))
nd, err := importer.BuildDagFromReader(ds, spl)
if err != nil {
return nil, err
}
err = header.AddNodeLinkClean("data", nd)
if err != nil {
return nil, err
}
}
_, err = ds.Add(header)
if err != nil {
return nil, err
}
path := escapePath(h.Name)
err = e.InsertNodeAtPath(context.Background(), path, header, func() *dag.Node { return new(dag.Node) })
if err != nil {
return nil, err
}
}
return e.Finalize(ds)
}
func fileNodeFromReader(t *testing.T, ds dag.DAGService, r io.Reader) *dag.Node {
nd, err := importer.BuildDagFromReader(ds, chunk.DefaultSplitter(r))
if err != nil {
t.Fatal(err)
}
return nd
}
// Perform the actual add & pin locally, outputting results to reader
func add(n *core.IpfsNode, reader io.Reader, useTrickle bool) (*dag.Node, error) {
var node *dag.Node
var err error
if useTrickle {
node, err = importer.BuildTrickleDagFromReader(
reader,
n.DAG,
chunk.DefaultSplitter,
importer.PinIndirectCB(n.Pinning.GetManual()),
)
} else {
node, err = importer.BuildDagFromReader(
reader,
n.DAG,
chunk.DefaultSplitter,
importer.PinIndirectCB(n.Pinning.GetManual()),
)
}
if err != nil {
return nil, err
}
return node, nil
}
func TestEnumerateChildren(t *testing.T) {
bsi := bstest.Mocks(1)
ds := NewDAGService(bsi[0])
read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512))
if err != nil {
t.Fatal(err)
}
set := cid.NewSet()
err = EnumerateChildren(context.Background(), ds, root.Cid(), set.Visit, false)
if err != nil {
t.Fatal(err)
}
var traverse func(n node.Node)
traverse = func(n node.Node) {
// traverse dag and check
for _, lnk := range n.Links() {
c := lnk.Cid
if !set.Has(c) {
t.Fatal("missing key in set! ", lnk.Cid.String())
}
child, err := ds.Get(context.Background(), c)
if err != nil {
t.Fatal(err)
}
traverse(child)
}
}
traverse(root)
}
// TODO(cryptix): find these helpers somewhere else
func (i *gatewayHandler) newDagFromReader(r io.Reader) (*dag.Node, error) {
// TODO(cryptix): change and remove this helper once PR1136 is merged
// return ufs.AddFromReader(i.node, r.Body)
return importer.BuildDagFromReader(
i.node.DAG,
chunk.DefaultSplitter(r))
}
func benchAddSize(n *core.IpfsNode, cfg *BenchCfg, size int64) error {
f := func(b *testing.B) {
b.SetBytes(size)
for i := 0; i < b.N; i++ {
r := io.LimitReader(randbo.New(), size)
spl := chunk.NewSizeSplitter(r, cfg.Blocksize)
_, err := importer.BuildDagFromReader(n.DAG, spl, nil)
if err != nil {
fmt.Printf("ERRROR: ", err)
b.Fatal(err)
}
}
}
br := testing.Benchmark(f)
bs := humanize.IBytes(uint64(size))
fmt.Printf("Add File (%s):\t%s\n", bs, br)
err := cr.GarbageCollect(n, context.Background())
if err != nil {
return err
}
return nil
}
// Perform the actual add & pin locally, outputting results to reader
func add(n *core.IpfsNode, reader io.Reader, useTrickle bool, chunker string) (*dag.Node, error) {
chnk, err := chunk.FromString(reader, chunker)
if err != nil {
return nil, err
}
var node *dag.Node
if useTrickle {
node, err = importer.BuildTrickleDagFromReader(
n.DAG,
chnk,
importer.PinIndirectCB(n.Pinning.GetManual()),
)
} else {
node, err = importer.BuildDagFromReader(
n.DAG,
chnk,
importer.PinIndirectCB(n.Pinning.GetManual()),
)
}
if err != nil {
return nil, err
}
return node, nil
}
func getRandFile(t *testing.T, ds dag.DAGService, size int64) *dag.Node {
r := io.LimitReader(u.NewTimeSeededRand(), size)
nd, err := importer.BuildDagFromReader(ds, chunk.DefaultSplitter(r))
if err != nil {
t.Fatal(err)
}
return nd
}
func add(n *core.IpfsNode, reader io.Reader) (*merkledag.Node, error) {
mp := n.Pinning.GetManual()
return importer.BuildDagFromReader(
n.DAG,
chunk.DefaultSplitter(reader),
importer.PinIndirectCB(mp),
)
}
func (s *Shell) Add(r io.Reader) (string, error) {
dag, err := importer.BuildDagFromReader(
s.node.DAG,
chunk.DefaultSplitter(r),
)
if err != nil {
return "", errgo.Notef(err, "add: importing DAG failed.")
}
return dag.Key().B58String(), nil
}
func TestMetadata(t *testing.T) {
ctx := context.Background()
// Make some random node
ds := getDagserv(t)
data := make([]byte, 1000)
u.NewTimeSeededRand().Read(data)
r := bytes.NewReader(data)
nd, err := importer.BuildDagFromReader(ds, chunk.DefaultSplitter(r))
if err != nil {
t.Fatal(err)
}
k, err := nd.Key()
if err != nil {
t.Fatal(err)
}
m := new(ft.Metadata)
m.MimeType = "THIS IS A TEST"
// Such effort, many compromise
ipfsnode := &core.IpfsNode{DAG: ds}
mdk, err := AddMetadataTo(ipfsnode, k.B58String(), m)
if err != nil {
t.Fatal(err)
}
rec, err := Metadata(ipfsnode, mdk)
if err != nil {
t.Fatal(err)
}
if rec.MimeType != m.MimeType {
t.Fatalf("something went wrong in conversion: '%s' != '%s'", rec.MimeType, m.MimeType)
}
retnode, err := ds.Get(ctx, key.B58KeyDecode(mdk))
if err != nil {
t.Fatal(err)
}
ndr, err := uio.NewDagReader(ctx, retnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(ndr)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(out, data) {
t.Fatal("read incorrect data")
}
}
func add(n *core.IpfsNode, reader io.Reader) (*merkledag.Node, error) {
mp := n.Pinning.GetManual()
node, err := importer.BuildDagFromReader(
reader,
n.DAG,
chunk.DefaultSplitter,
importer.PinIndirectCB(mp),
)
if err != nil {
return nil, err
}
return node, nil
}
// Perform the actual add & pin locally, outputting results to reader
func (adder Adder) add(reader io.Reader) (*dag.Node, error) {
chnk, err := chunk.FromString(reader, adder.Chunker)
if err != nil {
return nil, err
}
if adder.Trickle {
return importer.BuildTrickleDagFromReader(
adder.node.DAG,
chnk,
)
}
return importer.BuildDagFromReader(
adder.node.DAG,
chnk,
)
}
// Add builds a merkledag from the a reader, pinning all objects to the local
// datastore. Returns a key representing the root node.
func Add(n *core.IpfsNode, r io.Reader) (string, error) {
// TODO more attractive function signature importer.BuildDagFromReader
dagNode, err := importer.BuildDagFromReader(
n.DAG,
chunk.NewSizeSplitter(r, chunk.DefaultBlockSize),
importer.BasicPinnerCB(n.Pinning.GetManual()),
)
if err != nil {
return "", err
}
k, err := dagNode.Key()
if err != nil {
return "", err
}
return k.String(), nil
}
func runBatchFetchTest(t *testing.T, read io.Reader) {
ctx := context.Background()
var dagservs []DAGService
for _, bsi := range bstest.Mocks(5) {
dagservs = append(dagservs, NewDAGService(bsi))
}
spl := chunk.NewSizeSplitter(read, 512)
root, err := imp.BuildDagFromReader(dagservs[0], spl)
if err != nil {
t.Fatal(err)
}
t.Log("finished setup.")
dagr, err := uio.NewDagReader(ctx, root, dagservs[0])
if err != nil {
t.Fatal(err)
}
expected, err := ioutil.ReadAll(dagr)
if err != nil {
t.Fatal(err)
}
_, err = dagservs[0].Add(root)
if err != nil {
t.Fatal(err)
}
t.Log("Added file to first node.")
k, err := root.Key()
if err != nil {
t.Fatal(err)
}
wg := sync.WaitGroup{}
errs := make(chan error)
for i := 1; i < len(dagservs); i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
first, err := dagservs[i].Get(ctx, k)
if err != nil {
errs <- err
}
fmt.Println("Got first node back.")
read, err := uio.NewDagReader(ctx, first, dagservs[i])
if err != nil {
errs <- err
}
datagot, err := ioutil.ReadAll(read)
if err != nil {
errs <- err
}
if !bytes.Equal(datagot, expected) {
errs <- errors.New("Got bad data back!")
}
}(i)
}
go func() {
wg.Wait()
close(errs)
}()
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
}
func runBatchFetchTest(t *testing.T, read io.Reader) {
var dagservs []DAGService
for _, bsi := range bstest.Mocks(t, 5) {
dagservs = append(dagservs, NewDAGService(bsi))
}
spl := &chunk.SizeSplitter{512}
root, err := imp.BuildDagFromReader(read, dagservs[0], spl, nil)
if err != nil {
t.Fatal(err)
}
t.Log("finished setup.")
dagr, err := uio.NewDagReader(context.TODO(), root, dagservs[0])
if err != nil {
t.Fatal(err)
}
expected, err := ioutil.ReadAll(dagr)
if err != nil {
t.Fatal(err)
}
err = dagservs[0].AddRecursive(root)
if err != nil {
t.Fatal(err)
}
t.Log("Added file to first node.")
k, err := root.Key()
if err != nil {
t.Fatal(err)
}
wg := sync.WaitGroup{}
for i := 1; i < len(dagservs); i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
first, err := dagservs[i].Get(context.Background(), k)
if err != nil {
t.Fatal(err)
}
fmt.Println("Got first node back.")
read, err := uio.NewDagReader(context.TODO(), first, dagservs[i])
if err != nil {
t.Fatal(err)
}
datagot, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(datagot, expected) {
t.Fatal("Got bad data back!")
}
}(i)
}
wg.Wait()
}