func TestIndirectBlocks(t *testing.T) {
splitter := &chunk.SizeSplitter{512}
nbytes := 1024 * 1024
buf := make([]byte, nbytes)
u.NewTimeSeededRand().Read(buf)
read := bytes.NewReader(buf)
ds := mdtest.Mock(t)
dag, err := buildTestDag(read, ds, splitter)
if err != nil {
t.Fatal(err)
}
reader, err := uio.NewDagReader(context.Background(), dag, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(out, buf) {
t.Fatal("Not equal!")
}
}
func TestBuilderConsistency(t *testing.T) {
nbytes := 100000
buf := new(bytes.Buffer)
io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes))
should := dup(buf.Bytes())
dagserv := mdtest.Mock(t)
nd, err := buildTestDag(buf, dagserv, chunk.DefaultSplitter)
if err != nil {
t.Fatal(err)
}
r, err := uio.NewDagReader(context.Background(), nd, dagserv)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(r)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func testFileConsistency(t *testing.T, bs chunk.BlockSplitter, nbytes int) {
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock(t)
nd, err := buildTestDag(read, ds, bs)
if err != nil {
t.Fatal(err)
}
r, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(r)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func TestSeekToAlmostBegin(t *testing.T) {
ds := mdtest.Mock()
nd, should := getTestDag(t, ds, 10*1024, 500)
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
if err != nil {
t.Fatal(err)
}
if n != 4096 {
t.Fatal("Copy didnt copy enough bytes")
}
seeked, err := rs.Seek(1, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if seeked != 1 {
t.Fatal("Failed to seek to almost beginning")
}
dagrArrComp(t, rs, should[1:])
}
func TestBalancedDag(t *testing.T) {
ds := mdtest.Mock()
buf := make([]byte, 10000)
u.NewTimeSeededRand().Read(buf)
r := bytes.NewReader(buf)
nd, err := BuildDagFromReader(ds, chunk.DefaultSplitter(r), nil)
if err != nil {
t.Fatal(err)
}
dr, err := uio.NewDagReader(context.TODO(), nd, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(dr)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(out, buf) {
t.Fatal("bad read")
}
}
func TestAddLink(t *testing.T) {
ds := mdtest.Mock()
fishnode := &dag.Node{
Data: []byte("fishcakes!"),
}
fk, err := ds.Add(fishnode)
if err != nil {
t.Fatal(err)
}
nd := new(dag.Node)
nnode, err := addLink(context.Background(), ds, nd, "fish", fishnode)
if err != nil {
t.Fatal(err)
}
fnprime, err := nnode.GetLinkedNode(context.Background(), ds, "fish")
if err != nil {
t.Fatal(err)
}
fnpkey, err := fnprime.Key()
if err != nil {
t.Fatal(err)
}
if fnpkey != fk {
t.Fatal("wrong child node found!")
}
}
func TestSeekingConsistency(t *testing.T) {
nbytes := int64(128 * 1024)
ds := mdtest.Mock()
nd, should := getTestDag(t, ds, nbytes, 500)
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
out := make([]byte, nbytes)
for coff := nbytes - 4096; coff >= 0; coff -= 4096 {
t.Log(coff)
n, err := rs.Seek(coff, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if n != coff {
t.Fatal("wasnt able to seek to the right position")
}
nread, err := rs.Read(out[coff : coff+4096])
if err != nil {
t.Fatal(err)
}
if nread != 4096 {
t.Fatal("didnt read the correct number of bytes")
}
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func TestAddLink(t *testing.T) {
ds := mdtest.Mock()
fishnode := dag.NodeWithData([]byte("fishcakes!"))
fk, err := ds.Add(fishnode)
if err != nil {
t.Fatal(err)
}
nd := new(dag.ProtoNode)
nnode, err := addLink(context.Background(), ds, nd, "fish", fishnode)
if err != nil {
t.Fatal(err)
}
fnprime, err := nnode.GetLinkedNode(context.Background(), ds, "fish")
if err != nil {
t.Fatal(err)
}
fnpkey := fnprime.Cid()
if !fnpkey.Equals(fk) {
t.Fatal("wrong child node found!")
}
}
func TestSeekToBegin(t *testing.T) {
ds := mdtest.Mock()
nd, should := getTestDag(t, ds, 10*1024, 500)
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
if err != nil {
t.Fatal(err)
}
if n != 4096 {
t.Fatal("Copy didnt copy enough bytes")
}
seeked, err := rs.Seek(0, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if seeked != 0 {
t.Fatal("Failed to seek to beginning")
}
out, err := ioutil.ReadAll(rs)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func TestSeekingBasic(t *testing.T) {
nbytes := int64(10 * 1024)
ds := mdtest.Mock()
nd, should := getTestDag(t, ds, nbytes, 500)
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
start := int64(4000)
n, err := rs.Seek(start, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if n != start {
t.Fatal("Failed to seek to correct offset")
}
out, err := ioutil.ReadAll(rs)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should[start:])
if err != nil {
t.Fatal(err)
}
}
func TestEmptyKey(t *testing.T) {
ds := dstest.Mock()
_, err := ds.Get(context.Background(), key.Key(""))
if err != ErrNotFound {
t.Error("dag service should error when key is nil", err)
}
}
func TestSeekEndSingleBlockFile(t *testing.T) {
nbytes := int64(100)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock(t)
nd, err := buildTestDag(read, ds, &chunk.SizeSplitter{5000})
if err != nil {
t.Fatal(err)
}
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
seeked, err := rs.Seek(0, os.SEEK_END)
if err != nil {
t.Fatal(err)
}
if seeked != nbytes {
t.Fatal("Failed to seek to end")
}
}
func getBalancedDag(t testing.TB, size int64, blksize int) (*dag.Node, dag.DAGService) {
ds := mdtest.Mock(t)
r := io.LimitReader(u.NewTimeSeededRand(), size)
nd, err := BuildDagFromReader(r, ds, &chunk.SizeSplitter{blksize}, nil)
if err != nil {
t.Fatal(err)
}
return nd, ds
}
func getTrickleDag(t testing.TB, size int64, blksize int64) (*dag.Node, dag.DAGService) {
ds := mdtest.Mock()
r := io.LimitReader(u.NewTimeSeededRand(), size)
nd, err := BuildTrickleDagFromReader(ds, chunk.NewSizeSplitter(r, blksize), nil)
if err != nil {
t.Fatal(err)
}
return nd, ds
}
func TestCantGet(t *testing.T) {
ds := dstest.Mock()
a := NodeWithData([]byte("A"))
c := a.Cid()
_, err := ds.Get(context.Background(), c)
if !strings.Contains(err.Error(), "not found") {
t.Fatal("expected err not found, got: ", err)
}
}
func TestFetchFailure(t *testing.T) {
ds := dstest.Mock()
ds_bad := dstest.Mock()
top := new(Node)
for i := 0; i < 10; i++ {
nd := &Node{Data: []byte{byte('a' + i)}}
_, err := ds.Add(nd)
if err != nil {
t.Fatal(err)
}
err = top.AddNodeLinkClean(fmt.Sprintf("AA%d", i), nd)
if err != nil {
t.Fatal(err)
}
}
for i := 0; i < 10; i++ {
nd := &Node{Data: []byte{'f', 'a' + byte(i)}}
_, err := ds_bad.Add(nd)
if err != nil {
t.Fatal(err)
}
err = top.AddNodeLinkClean(fmt.Sprintf("BB%d", i), nd)
if err != nil {
t.Fatal(err)
}
}
getters := GetDAG(context.Background(), ds, top)
for i, getter := range getters {
_, err := getter.Get(context.Background())
if err != nil && i < 10 {
t.Fatal(err)
}
if err == nil && i >= 10 {
t.Fatal("should have failed request")
}
}
}
func testFileConsistency(t *testing.T, nbytes int64, blksize int64) {
ds := mdtest.Mock()
nd, should := getTestDag(t, ds, nbytes, blksize)
r, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
dagrArrComp(t, r, should)
}
func TestTwoChunks(t *testing.T) {
ds := mdtest.Mock()
nd, should := getTestDag(t, ds, 2000, 1000)
r, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
dagrArrComp(t, r, should)
}
func TestGetRawNodes(t *testing.T) {
rn := NewRawNode([]byte("test"))
ds := dstest.Mock()
c, err := ds.Add(rn)
if err != nil {
t.Fatal(err)
}
if !c.Equals(rn.Cid()) {
t.Fatal("output cids didnt match")
}
out, err := ds.Get(context.TODO(), c)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(out.RawData(), []byte("test")) {
t.Fatal("raw block should match input data")
}
if out.Links() != nil {
t.Fatal("raw blocks shouldnt have links")
}
if out.Tree("", -1) != nil {
t.Fatal("tree should return no paths in a raw block")
}
size, err := out.Size()
if err != nil {
t.Fatal(err)
}
if size != 4 {
t.Fatal("expected size to be 4")
}
ns, err := out.Stat()
if err != nil {
t.Fatal(err)
}
if ns.DataSize != 4 {
t.Fatal("expected size to be 4, got: ", ns.DataSize)
}
_, _, err = out.Resolve([]string{"foo"})
if err != ErrLinkNotFound {
t.Fatal("shouldnt find links under raw blocks")
}
}
func TestFindLink(t *testing.T) {
ds := mdtest.Mock()
k, err := ds.Add(new(ProtoNode))
if err != nil {
t.Fatal(err)
}
nd := &ProtoNode{}
nd.SetLinks([]*node.Link{
{Name: "a", Cid: k},
{Name: "c", Cid: k},
{Name: "b", Cid: k},
})
_, err = ds.Add(nd)
if err != nil {
t.Fatal(err)
}
lnk, err := nd.GetNodeLink("b")
if err != nil {
t.Fatal(err)
}
if lnk.Name != "b" {
t.Fatal("got wrong link back")
}
_, err = nd.GetNodeLink("f")
if err != ErrLinkNotFound {
t.Fatal("shouldnt have found link")
}
_, err = nd.GetLinkedNode(context.Background(), ds, "b")
if err != nil {
t.Fatal(err)
}
outnd, err := nd.UpdateNodeLink("b", nd)
if err != nil {
t.Fatal(err)
}
olnk, err := outnd.GetNodeLink("b")
if err != nil {
t.Fatal(err)
}
if olnk.Cid.String() == k.String() {
t.Fatal("new link should have different hash")
}
}
func TestRecurivePathResolution(t *testing.T) {
ctx := context.Background()
dagService := dagmock.Mock()
a, _ := randNode()
b, _ := randNode()
c, cKey := randNode()
err := b.AddNodeLink("grandchild", c)
if err != nil {
t.Fatal(err)
}
err = a.AddNodeLink("child", b)
if err != nil {
t.Fatal(err)
}
for _, n := range []*merkledag.Node{a, b, c} {
_, err = dagService.Add(n)
if err != nil {
t.Fatal(err)
}
}
aKey, err := a.Key()
if err != nil {
t.Fatal(err)
}
segments := []string{aKey.String(), "child", "grandchild"}
p, err := path.FromSegments("/ipfs/", segments...)
if err != nil {
t.Fatal(err)
}
resolver := &path.Resolver{DAG: dagService}
node, err := resolver.ResolvePath(ctx, p)
if err != nil {
t.Fatal(err)
}
key, err := node.Key()
if err != nil {
t.Fatal(err)
}
if key.String() != cKey.String() {
t.Fatal(fmt.Errorf(
"recursive path resolution failed for %s: %s != %s",
p.String(), key.String(), cKey.String()))
}
}
// This test appends one byte at a time to an empty file
func TestMultipleAppends(t *testing.T) {
ds := mdtest.Mock()
// TODO: fix small size appends and make this number bigger
nbytes := int64(1000)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(nil)
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
if err != nil {
t.Fatal(err)
}
dbp := &h.DagBuilderParams{
Dagserv: ds,
Maxlinks: 4,
}
spl := chunk.SizeSplitterGen(500)
ctx := context.Background()
for i := 0; i < len(should); i++ {
blks, errs := chunk.Chan(spl(bytes.NewReader(should[i : i+1])))
nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
if err != nil {
t.Fatal(err)
}
err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat)
if err != nil {
t.Fatal(err)
}
fread, err := uio.NewDagReader(ctx, nnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(fread)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should[:i+1])
if err != nil {
t.Fatal(err)
}
}
}
func TestSet(t *testing.T) {
ds := mdtest.Mock()
limit := 10000 // 10000 reproduces the pinloss issue fairly reliably
if os.Getenv("STRESS_IT_OUT_YO") != "" {
limit = 10000000
}
var inputs []*cid.Cid
for i := 0; i < limit; i++ {
c, err := ds.Add(dag.NodeWithData([]byte(fmt.Sprint(i))))
if err != nil {
t.Fatal(err)
}
inputs = append(inputs, c)
}
out, err := storeSet(context.Background(), ds, inputs, ignoreCids)
if err != nil {
t.Fatal(err)
}
// weird wrapper node because loadSet expects us to pass an
// object pointing to multiple named sets
setroot := &dag.ProtoNode{}
err = setroot.AddNodeLinkClean("foo", out)
if err != nil {
t.Fatal(err)
}
outset, err := loadSet(context.Background(), ds, setroot, "foo", ignoreCids)
if err != nil {
t.Fatal(err)
}
if len(outset) != limit {
t.Fatal("got wrong number", len(outset), limit)
}
seen := cid.NewSet()
for _, c := range outset {
seen.Add(c)
}
for _, c := range inputs {
if !seen.Has(c) {
t.Fatalf("expected to have %s, didnt find it")
}
}
}
func TestAppend(t *testing.T) {
nbytes := int64(128 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
// Reader for half the bytes
read := bytes.NewReader(should[:nbytes/2])
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
if err != nil {
t.Fatal(err)
}
dbp := &h.DagBuilderParams{
Dagserv: ds,
Maxlinks: h.DefaultLinksPerBlock,
}
r := bytes.NewReader(should[nbytes/2:])
blks, errs := chunk.Chan(chunk.NewSizeSplitter(r, 500))
ctx := context.Background()
nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
if err != nil {
t.Fatal(err)
}
err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat)
if err != nil {
t.Fatal(err)
}
fread, err := uio.NewDagReader(ctx, nnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(fread)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func TestAppendSingleBytesToEmpty(t *testing.T) {
ds := mdtest.Mock()
data := []byte("AB")
nd := new(merkledag.Node)
nd.Data = ft.FilePBData(nil, 0)
dbp := &h.DagBuilderParams{
Dagserv: ds,
Maxlinks: 4,
}
spl := chunk.SizeSplitterGen(500)
blks, errs := chunk.Chan(spl(bytes.NewReader(data[:1])))
ctx := context.Background()
nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
if err != nil {
t.Fatal(err)
}
blks, errs = chunk.Chan(spl(bytes.NewReader(data[1:])))
nnode, err = TrickleAppend(ctx, nnode, dbp.New(blks, errs))
if err != nil {
t.Fatal(err)
}
fread, err := uio.NewDagReader(ctx, nnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(fread)
if err != nil {
t.Fatal(err)
}
fmt.Println(out, data)
err = arrComp(out, data)
if err != nil {
t.Fatal(err)
}
}
func TestAppend(t *testing.T) {
nbytes := int64(128 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
// Reader for half the bytes
read := bytes.NewReader(should[:nbytes/2])
ds := mdtest.Mock(t)
nd, err := buildTestDag(read, ds, &chunk.SizeSplitter{500})
if err != nil {
t.Fatal(err)
}
dbp := &h.DagBuilderParams{
Dagserv: ds,
Maxlinks: h.DefaultLinksPerBlock,
}
spl := &chunk.SizeSplitter{500}
blks := spl.Split(bytes.NewReader(should[nbytes/2:]))
nnode, err := TrickleAppend(nd, dbp.New(blks))
if err != nil {
t.Fatal(err)
}
err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat)
if err != nil {
t.Fatal(err)
}
fread, err := uio.NewDagReader(context.TODO(), nnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(fread)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func TestAppendSingleBytesToEmpty(t *testing.T) {
ds := mdtest.Mock(t)
data := []byte("AB")
nd := new(merkledag.Node)
nd.Data = ft.FilePBData(nil, 0)
dbp := &h.DagBuilderParams{
Dagserv: ds,
Maxlinks: 4,
}
spl := &chunk.SizeSplitter{500}
blks := spl.Split(bytes.NewReader(data[:1]))
nnode, err := TrickleAppend(nd, dbp.New(blks))
if err != nil {
t.Fatal(err)
}
blks = spl.Split(bytes.NewReader(data[1:]))
nnode, err = TrickleAppend(nnode, dbp.New(blks))
if err != nil {
t.Fatal(err)
}
fread, err := uio.NewDagReader(context.TODO(), nnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(fread)
if err != nil {
t.Fatal(err)
}
fmt.Println(out, data)
err = arrComp(out, data)
if err != nil {
t.Fatal(err)
}
}
func TestSeekEndSingleBlockFile(t *testing.T) {
nbytes := int64(100)
ds := mdtest.Mock()
nd, _ := getTestDag(t, ds, nbytes, 5000)
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
seeked, err := rs.Seek(0, os.SEEK_END)
if err != nil {
t.Fatal(err)
}
if seeked != nbytes {
t.Fatal("Failed to seek to end")
}
}
func TestIndirectBlocks(t *testing.T) {
ds := mdtest.Mock()
dag, buf := getTestDag(t, ds, 1024*1024, 512)
reader, err := uio.NewDagReader(context.Background(), dag, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(out, buf) {
t.Fatal("Not equal!")
}
}
func TestBasicAddGet(t *testing.T) {
ds := dstest.Mock()
nd := new(ProtoNode)
c, err := ds.Add(nd)
if err != nil {
t.Fatal(err)
}
out, err := ds.Get(context.Background(), c)
if err != nil {
t.Fatal(err)
}
if !nd.Cid().Equals(out.Cid()) {
t.Fatal("output didnt match input")
}
}