func TestIndirectBlocks(t *testing.T) {
splitter := chunk.SizeSplitterGen(512)
nbytes := 1024 * 1024
buf := make([]byte, nbytes)
u.NewTimeSeededRand().Read(buf)
read := bytes.NewReader(buf)
ds := mdtest.Mock()
dag, err := buildTestDag(ds, splitter(read))
if err != nil {
t.Fatal(err)
}
reader, err := uio.NewDagReader(context.Background(), dag, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(out, buf) {
t.Fatal("Not equal!")
}
}
func (s *Node) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
k, err := s.Nd.Key()
if err != nil {
return err
}
// setup our logging event
lm := make(lgbl.DeferredMap)
lm["fs"] = "ipfs"
lm["key"] = func() interface{} { return k.Pretty() }
lm["req_offset"] = req.Offset
lm["req_size"] = req.Size
defer log.EventBegin(ctx, "fuseRead", lm).Done()
r, err := uio.NewDagReader(ctx, s.Nd, s.Ipfs.DAG)
if err != nil {
return err
}
o, err := r.Seek(req.Offset, os.SEEK_SET)
lm["res_offset"] = o
if err != nil {
return err
}
buf := resp.Data[:min(req.Size, int(int64(r.Size())-req.Offset))]
n, err := io.ReadFull(r, buf)
if err != nil && err != io.EOF {
return err
}
resp.Data = resp.Data[:n]
lm["res_size"] = n
return nil // may be non-nil / not succeeded
}
func testFileConsistency(t *testing.T, bs chunk.SplitterGen, nbytes int) {
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock()
nd, err := buildTestDag(ds, bs(read))
if err != nil {
t.Fatal(err)
}
r, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(r)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func TestSeekingConsistency(t *testing.T) {
nbytes := int64(128 * 1024)
ds := mdtest.Mock()
nd, should := getTestDag(t, ds, nbytes, 500)
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
out := make([]byte, nbytes)
for coff := nbytes - 4096; coff >= 0; coff -= 4096 {
t.Log(coff)
n, err := rs.Seek(coff, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if n != coff {
t.Fatal("wasnt able to seek to the right position")
}
nread, err := rs.Read(out[coff : coff+4096])
if err != nil {
t.Fatal(err)
}
if nread != 4096 {
t.Fatal("didnt read the correct number of bytes")
}
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func TestBuilderConsistency(t *testing.T) {
nbytes := 100000
buf := new(bytes.Buffer)
io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes))
should := dup(buf.Bytes())
dagserv := mdtest.Mock()
nd, err := buildTestDag(dagserv, chunk.DefaultSplitter(buf))
if err != nil {
t.Fatal(err)
}
r, err := uio.NewDagReader(context.Background(), nd, dagserv)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(r)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func (dm *DagModifier) readPrep() error {
err := dm.Sync()
if err != nil {
return err
}
if dm.read == nil {
ctx, cancel := context.WithCancel(dm.ctx)
dr, err := uio.NewDagReader(ctx, dm.curNode, dm.dagserv)
if err != nil {
return err
}
i, err := dr.Seek(int64(dm.curWrOff), os.SEEK_SET)
if err != nil {
return err
}
if i != int64(dm.curWrOff) {
return ErrSeekFail
}
dm.readCancel = cancel
dm.read = dr
}
return nil
}
func TestSeekToAlmostBegin(t *testing.T) {
ds := mdtest.Mock()
nd, should := getTestDag(t, ds, 10*1024, 500)
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
if err != nil {
t.Fatal(err)
}
if n != 4096 {
t.Fatal("Copy didnt copy enough bytes")
}
seeked, err := rs.Seek(1, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if seeked != 1 {
t.Fatal("Failed to seek to almost beginning")
}
out, err := ioutil.ReadAll(rs)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should[1:])
if err != nil {
t.Fatal(err)
}
}
func TestSeekingBasic(t *testing.T) {
nbytes := int64(10 * 1024)
ds := mdtest.Mock()
nd, should := getTestDag(t, ds, nbytes, 500)
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
start := int64(4000)
n, err := rs.Seek(start, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if n != start {
t.Fatal("Failed to seek to correct offset")
}
out, err := ioutil.ReadAll(rs)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should[start:])
if err != nil {
t.Fatal(err)
}
}
func TestBalancedDag(t *testing.T) {
ds := mdtest.Mock()
buf := make([]byte, 10000)
u.NewTimeSeededRand().Read(buf)
r := bytes.NewReader(buf)
nd, err := BuildDagFromReader(ds, chunk.DefaultSplitter(r), nil)
if err != nil {
t.Fatal(err)
}
dr, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(dr)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(out, buf) {
t.Fatal("bad read")
}
}
func TestSeekEndSingleBlockFile(t *testing.T) {
nbytes := int64(100)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 5000))
if err != nil {
t.Fatal(err)
}
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
seeked, err := rs.Seek(0, os.SEEK_END)
if err != nil {
t.Fatal(err)
}
if seeked != nbytes {
t.Fatal("Failed to seek to end")
}
}
func Cat(n *core.IpfsNode, pstr string) (*uio.DagReader, error) {
p := path.FromString(pstr)
dagNode, err := n.Resolver.ResolvePath(n.Context(), p)
if err != nil {
return nil, err
}
return uio.NewDagReader(n.Context(), dagNode, n.DAG)
}
func TestMetadata(t *testing.T) {
ctx := context.Background()
// Make some random node
ds := getDagserv(t)
data := make([]byte, 1000)
u.NewTimeSeededRand().Read(data)
r := bytes.NewReader(data)
nd, err := importer.BuildDagFromReader(ds, chunk.DefaultSplitter(r), nil)
if err != nil {
t.Fatal(err)
}
k, err := nd.Key()
if err != nil {
t.Fatal(err)
}
m := new(ft.Metadata)
m.MimeType = "THIS IS A TEST"
// Such effort, many compromise
ipfsnode := &core.IpfsNode{DAG: ds}
mdk, err := AddMetadataTo(ipfsnode, k.B58String(), m)
if err != nil {
t.Fatal(err)
}
rec, err := Metadata(ipfsnode, mdk)
if err != nil {
t.Fatal(err)
}
if rec.MimeType != m.MimeType {
t.Fatalf("something went wrong in conversion: '%s' != '%s'", rec.MimeType, m.MimeType)
}
retnode, err := ds.Get(ctx, key.B58KeyDecode(mdk))
if err != nil {
t.Fatal(err)
}
ndr, err := uio.NewDagReader(ctx, retnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(ndr)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(out, data) {
t.Fatal("read incorrect data")
}
}
func TestMultiWrite(t *testing.T) {
dserv, pins := getMockDagServ(t)
_, n := getNode(t, dserv, 0, pins)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512))
if err != nil {
t.Fatal(err)
}
data := make([]byte, 4000)
u.NewTimeSeededRand().Read(data)
for i := 0; i < len(data); i++ {
n, err := dagmod.WriteAt(data[i:i+1], int64(i))
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
}
size, err := dagmod.Size()
if err != nil {
t.Fatal(err)
}
if size != int64(i+1) {
t.Fatal("Size was reported incorrectly")
}
}
nd, err := dagmod.GetNode()
if err != nil {
t.Fatal(err)
}
read, err := uio.NewDagReader(context.Background(), nd, dserv)
if err != nil {
t.Fatal(err)
}
rbuf, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
err = arrComp(rbuf, data)
if err != nil {
t.Fatal(err)
}
}
// This test appends one byte at a time to an empty file
func TestMultipleAppends(t *testing.T) {
ds := mdtest.Mock()
// TODO: fix small size appends and make this number bigger
nbytes := int64(1000)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(nil)
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
if err != nil {
t.Fatal(err)
}
dbp := &h.DagBuilderParams{
Dagserv: ds,
Maxlinks: 4,
}
spl := chunk.SizeSplitterGen(500)
ctx := context.Background()
for i := 0; i < len(should); i++ {
blks, errs := chunk.Chan(spl(bytes.NewReader(should[i : i+1])))
nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
if err != nil {
t.Fatal(err)
}
err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat)
if err != nil {
t.Fatal(err)
}
fread, err := uio.NewDagReader(ctx, nnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(fread)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should[:i+1])
if err != nil {
t.Fatal(err)
}
}
}
func runReadBench(b *testing.B, nd *dag.Node, ds dag.DAGService) {
for i := 0; i < b.N; i++ {
ctx, cancel := context.WithCancel(context.Background())
read, err := uio.NewDagReader(ctx, nd, ds)
if err != nil {
b.Fatal(err)
}
_, err = read.WriteTo(ioutil.Discard)
if err != nil && err != io.EOF {
b.Fatal(err)
}
cancel()
}
}
func TestAppend(t *testing.T) {
nbytes := int64(128 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
// Reader for half the bytes
read := bytes.NewReader(should[:nbytes/2])
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
if err != nil {
t.Fatal(err)
}
dbp := &h.DagBuilderParams{
Dagserv: ds,
Maxlinks: h.DefaultLinksPerBlock,
}
r := bytes.NewReader(should[nbytes/2:])
blks, errs := chunk.Chan(chunk.NewSizeSplitter(r, 500))
ctx := context.Background()
nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
if err != nil {
t.Fatal(err)
}
err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat)
if err != nil {
t.Fatal(err)
}
fread, err := uio.NewDagReader(ctx, nnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(fread)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func TestAppendSingleBytesToEmpty(t *testing.T) {
ds := mdtest.Mock()
data := []byte("AB")
nd := new(merkledag.Node)
nd.Data = ft.FilePBData(nil, 0)
dbp := &h.DagBuilderParams{
Dagserv: ds,
Maxlinks: 4,
}
spl := chunk.SizeSplitterGen(500)
blks, errs := chunk.Chan(spl(bytes.NewReader(data[:1])))
ctx := context.Background()
nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
if err != nil {
t.Fatal(err)
}
blks, errs = chunk.Chan(spl(bytes.NewReader(data[1:])))
nnode, err = TrickleAppend(ctx, nnode, dbp.New(blks, errs))
if err != nil {
t.Fatal(err)
}
fread, err := uio.NewDagReader(ctx, nnode, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(fread)
if err != nil {
t.Fatal(err)
}
fmt.Println(out, data)
err = arrComp(out, data)
if err != nil {
t.Fatal(err)
}
}
func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier) []byte {
newdata := make([]byte, size)
r := u.NewTimeSeededRand()
r.Read(newdata)
if size+beg > uint64(len(orig)) {
orig = append(orig, make([]byte, (size+beg)-uint64(len(orig)))...)
}
copy(orig[beg:], newdata)
nmod, err := dm.WriteAt(newdata, int64(beg))
if err != nil {
t.Fatal(err)
}
if nmod != int(size) {
t.Fatalf("Mod length not correct! %d != %d", nmod, size)
}
nd, err := dm.GetNode()
if err != nil {
t.Fatal(err)
}
err = trickle.VerifyTrickleDagStructure(nd, dm.dagserv, h.DefaultLinksPerBlock, 4)
if err != nil {
t.Fatal(err)
}
rd, err := uio.NewDagReader(context.Background(), nd, dm.dagserv)
if err != nil {
t.Fatal(err)
}
after, err := ioutil.ReadAll(rd)
if err != nil {
t.Fatal(err)
}
err = arrComp(after, orig)
if err != nil {
t.Fatal(err)
}
return orig
}
func TestSeekEndSingleBlockFile(t *testing.T) {
nbytes := int64(100)
ds := mdtest.Mock()
nd, _ := getTestDag(t, ds, nbytes, 5000)
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
seeked, err := rs.Seek(0, os.SEEK_END)
if err != nil {
t.Fatal(err)
}
if seeked != nbytes {
t.Fatal("Failed to seek to end")
}
}
func TestIndirectBlocks(t *testing.T) {
ds := mdtest.Mock()
dag, buf := getTestDag(t, ds, 1024*1024, 512)
reader, err := uio.NewDagReader(context.Background(), dag, ds)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(out, buf) {
t.Fatal("Not equal!")
}
}
func TestSeekingStress(t *testing.T) {
nbytes := int64(1024 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 1000))
if err != nil {
t.Fatal(err)
}
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
testbuf := make([]byte, nbytes)
for i := 0; i < 50; i++ {
offset := mrand.Intn(int(nbytes))
l := int(nbytes) - offset
n, err := rs.Seek(int64(offset), os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if n != int64(offset) {
t.Fatal("Seek failed to move to correct position")
}
nread, err := rs.Read(testbuf[:l])
if err != nil {
t.Fatal(err)
}
if nread != l {
t.Fatal("Failed to read enough bytes")
}
err = arrComp(testbuf[:l], should[offset:offset+l])
if err != nil {
t.Fatal(err)
}
}
}
func getNode(t testing.TB, dserv mdag.DAGService, size int64, pinner pin.ManualPinner) ([]byte, *mdag.Node) {
in := io.LimitReader(u.NewTimeSeededRand(), size)
node, err := imp.BuildTrickleDagFromReader(dserv, sizeSplitterGen(500)(in), imp.BasicPinnerCB(pinner))
if err != nil {
t.Fatal(err)
}
dr, err := uio.NewDagReader(context.Background(), node, dserv)
if err != nil {
t.Fatal(err)
}
b, err := ioutil.ReadAll(dr)
if err != nil {
t.Fatal(err)
}
return b, node
}
func TestSeekToAlmostBegin(t *testing.T) {
nbytes := int64(10 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500))
if err != nil {
t.Fatal(err)
}
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
if err != nil {
t.Fatal(err)
}
if n != 4096 {
t.Fatal("Copy didnt copy enough bytes")
}
seeked, err := rs.Seek(1, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if seeked != 1 {
t.Fatal("Failed to seek to almost beginning")
}
out, err := ioutil.ReadAll(rs)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should[1:])
if err != nil {
t.Fatal(err)
}
}
func TestBuilderConsistency(t *testing.T) {
dagserv := mdtest.Mock()
nd, should := getTestDag(t, dagserv, 100000, chunk.DefaultBlockSize)
r, err := uio.NewDagReader(context.Background(), nd, dagserv)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(r)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should)
if err != nil {
t.Fatal(err)
}
}
func TestWriteNewFile(t *testing.T) {
dserv, pins := getMockDagServ(t)
_, n := getNode(t, dserv, 0, pins)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512))
if err != nil {
t.Fatal(err)
}
towrite := make([]byte, 2000)
u.NewTimeSeededRand().Read(towrite)
nw, err := dagmod.Write(towrite)
if err != nil {
t.Fatal(err)
}
if nw != len(towrite) {
t.Fatal("Wrote wrong amount")
}
nd, err := dagmod.GetNode()
if err != nil {
t.Fatal(err)
}
read, err := uio.NewDagReader(ctx, nd, dserv)
if err != nil {
t.Fatal(err)
}
data, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
if err := arrComp(data, towrite); err != nil {
t.Fatal(err)
}
}
func TestSeekingBasic(t *testing.T) {
nbytes := int64(10 * 1024)
should := make([]byte, nbytes)
u.NewTimeSeededRand().Read(should)
read := bytes.NewReader(should)
ds := mdtest.Mock()
nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 512))
if err != nil {
t.Fatal(err)
}
rs, err := uio.NewDagReader(context.Background(), nd, ds)
if err != nil {
t.Fatal(err)
}
start := int64(4000)
n, err := rs.Seek(start, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if n != start {
t.Fatal("Failed to seek to correct offset")
}
out, err := ioutil.ReadAll(rs)
if err != nil {
t.Fatal(err)
}
err = arrComp(out, should[start:])
if err != nil {
t.Fatal(err)
}
}
func (tr *tarReader) Read(b []byte) (int, error) {
// if we have a header to be read, it takes priority
if tr.hdrBuf != nil {
n, err := tr.hdrBuf.Read(b)
if err == io.EOF {
tr.hdrBuf = nil
return n, nil
}
return n, err
}
// no header remaining, check for recursive
if tr.childRead != nil {
n, err := tr.childRead.Read(b)
if err == io.EOF {
tr.childRead = nil
return n, nil
}
return n, err
}
// check for filedata to be read
if tr.fileRead != nil {
n, err := tr.fileRead.Read(b)
if err == io.EOF {
nr := tr.fileRead.n
tr.pad = (blockSize - (nr % blockSize)) % blockSize
tr.fileRead.Close()
tr.fileRead = nil
return n, nil
}
return n, err
}
// filedata reads must be padded out to 512 byte offsets
if tr.pad > 0 {
n := copy(b, zeroBlock[:tr.pad])
tr.pad -= n
return n, nil
}
if len(tr.links) == 0 {
return 0, io.EOF
}
next := tr.links[0]
tr.links = tr.links[1:]
headerNd, err := next.GetNode(tr.ctx, tr.ds)
if err != nil {
return 0, err
}
tr.hdrBuf = bytes.NewReader(headerNd.Data)
dataNd, err := headerNd.GetLinkedNode(tr.ctx, tr.ds, "data")
if err != nil && err != dag.ErrNotFound {
return 0, err
}
if err == nil {
dr, err := uio.NewDagReader(tr.ctx, dataNd, tr.ds)
if err != nil {
log.Error("dagreader error: ", err)
return 0, err
}
tr.fileRead = &countReader{r: dr}
} else if len(headerNd.Links) > 0 {
tr.childRead = &tarReader{
links: headerNd.Links,
ds: tr.ds,
ctx: tr.ctx,
}
}
return tr.Read(b)
}
func TestCorrectPinning(t *testing.T) {
dserv, bstore, pins := getMockDagServAndBstore(t)
b, n := getNode(t, dserv, 50000, pins)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512))
if err != nil {
t.Fatal(err)
}
buf := make([]byte, 1024)
for i := 0; i < 100; i++ {
size, err := dagmod.Size()
if err != nil {
t.Fatal(err)
}
offset := rand.Intn(int(size))
u.NewTimeSeededRand().Read(buf)
if offset+len(buf) > int(size) {
b = append(b[:offset], buf...)
} else {
copy(b[offset:], buf)
}
n, err := dagmod.WriteAt(buf, int64(offset))
if err != nil {
t.Fatal(err)
}
if n != len(buf) {
t.Fatal("wrote incorrect number of bytes")
}
}
fisize, err := dagmod.Size()
if err != nil {
t.Fatal(err)
}
if int(fisize) != len(b) {
t.Fatal("reported filesize incorrect", fisize, len(b))
}
// Run a GC, then ensure we can still read the file correctly
basicGC(t, bstore, pins)
nd, err := dagmod.GetNode()
if err != nil {
t.Fatal(err)
}
read, err := uio.NewDagReader(context.Background(), nd, dserv)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
if err = arrComp(out, b); err != nil {
t.Fatal(err)
}
rootk, err := nd.Key()
if err != nil {
t.Fatal(err)
}
// Verify only one recursive pin
recpins := pins.RecursiveKeys()
if len(recpins) != 1 {
t.Fatal("Incorrect number of pinned entries")
}
// verify the correct node is pinned
if recpins[0] != rootk {
t.Fatal("Incorrect node recursively pinned")
}
indirpins := pins.IndirectKeys()
children := enumerateChildren(t, nd, dserv)
if len(indirpins) != len(children) {
t.Log(len(indirpins), len(children))
t.Fatal("Incorrect number of indirectly pinned blocks")
}
}
func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithCancel(i.node.Context())
defer cancel()
urlPath := r.URL.Path
// IPNSHostnameOption might have constructed an IPNS path using the Host header.
// In this case, we need the original path for constructing redirects
// and links that match the requested URL.
// For example, http://example.net would become /ipns/example.net, and
// the redirects and links would end up as http://example.net/ipns/example.net
originalUrlPath := urlPath
ipnsHostname := false
hdr := r.Header["X-IPNS-Original-Path"]
if len(hdr) > 0 {
originalUrlPath = hdr[0]
ipnsHostname = true
}
if i.config.BlockList != nil && i.config.BlockList.ShouldBlock(urlPath) {
w.WriteHeader(http.StatusForbidden)
w.Write([]byte("403 - Forbidden"))
return
}
nd, err := core.Resolve(ctx, i.node, path.Path(urlPath))
if err != nil {
webError(w, "Path Resolve error", err, http.StatusBadRequest)
return
}
etag := gopath.Base(urlPath)
if r.Header.Get("If-None-Match") == etag {
w.WriteHeader(http.StatusNotModified)
return
}
i.addUserHeaders(w) // ok, _now_ write user's headers.
w.Header().Set("X-IPFS-Path", urlPath)
// Suborigin header, sandboxes apps from each other in the browser (even
// though they are served from the same gateway domain).
//
// Omited if the path was treated by IPNSHostnameOption(), for example
// a request for http://example.net/ would be changed to /ipns/example.net/,
// which would turn into an incorrect Suborigin: example.net header.
//
// NOTE: This is not yet widely supported by browsers.
if !ipnsHostname {
pathRoot := strings.SplitN(urlPath, "/", 4)[2]
w.Header().Set("Suborigin", pathRoot)
}
dr, err := uio.NewDagReader(ctx, nd, i.node.DAG)
if err != nil && err != uio.ErrIsDir {
// not a directory and still an error
internalWebError(w, err)
return
}
// set these headers _after_ the error, for we may just not have it
// and dont want the client to cache a 500 response...
// and only if it's /ipfs!
// TODO: break this out when we split /ipfs /ipns routes.
modtime := time.Now()
if strings.HasPrefix(urlPath, ipfsPathPrefix) {
w.Header().Set("Etag", etag)
w.Header().Set("Cache-Control", "public, max-age=29030400")
// set modtime to a really long time ago, since files are immutable and should stay cached
modtime = time.Unix(1, 0)
}
if err == nil {
defer dr.Close()
_, name := gopath.Split(urlPath)
http.ServeContent(w, r, name, modtime, dr)
return
}
// storage for directory listing
var dirListing []directoryItem
// loop through files
foundIndex := false
for _, link := range nd.Links {
if link.Name == "index.html" {
log.Debugf("found index.html link for %s", urlPath)
foundIndex = true
if urlPath[len(urlPath)-1] != '/' {
// See comment above where originalUrlPath is declared.
http.Redirect(w, r, originalUrlPath+"/", 302)
log.Debugf("redirect to %s", originalUrlPath+"/")
return
}
// return index page instead.
nd, err := core.Resolve(ctx, i.node, path.Path(urlPath+"/index.html"))
if err != nil {
internalWebError(w, err)
return
}
dr, err := uio.NewDagReader(ctx, nd, i.node.DAG)
if err != nil {
internalWebError(w, err)
return
}
defer dr.Close()
// write to request
if r.Method != "HEAD" {
io.Copy(w, dr)
}
break
}
// See comment above where originalUrlPath is declared.
di := directoryItem{humanize.Bytes(link.Size), link.Name, gopath.Join(originalUrlPath, link.Name)}
dirListing = append(dirListing, di)
}
if !foundIndex {
if r.Method != "HEAD" {
// construct the correct back link
// https://github.com/djbarber/ipfs-hack/issues/1365
var backLink string = urlPath
// don't go further up than /ipfs/$hash/
pathSplit := strings.Split(backLink, "/")
switch {
// keep backlink
case len(pathSplit) == 3: // url: /ipfs/$hash
// keep backlink
case len(pathSplit) == 4 && pathSplit[3] == "": // url: /ipfs/$hash/
// add the correct link depending on wether the path ends with a slash
default:
if strings.HasSuffix(backLink, "/") {
backLink += "./.."
} else {
backLink += "/.."
}
}
// strip /ipfs/$hash from backlink if IPNSHostnameOption touched the path.
if ipnsHostname {
backLink = "/"
if len(pathSplit) > 5 {
// also strip the trailing segment, because it's a backlink
backLinkParts := pathSplit[3 : len(pathSplit)-2]
backLink += strings.Join(backLinkParts, "/") + "/"
}
}
// See comment above where originalUrlPath is declared.
tplData := listingTemplateData{
Listing: dirListing,
Path: originalUrlPath,
BackLink: backLink,
}
err := listingTemplate.Execute(w, tplData)
if err != nil {
internalWebError(w, err)
return
}
}
}
}
func runBatchFetchTest(t *testing.T, read io.Reader) {
ctx := context.Background()
var dagservs []DAGService
for _, bsi := range bstest.Mocks(5) {
dagservs = append(dagservs, NewDAGService(bsi))
}
spl := chunk.NewSizeSplitter(read, 512)
root, err := imp.BuildDagFromReader(dagservs[0], spl, nil)
if err != nil {
t.Fatal(err)
}
t.Log("finished setup.")
dagr, err := uio.NewDagReader(ctx, root, dagservs[0])
if err != nil {
t.Fatal(err)
}
expected, err := ioutil.ReadAll(dagr)
if err != nil {
t.Fatal(err)
}
err = dagservs[0].AddRecursive(root)
if err != nil {
t.Fatal(err)
}
t.Log("Added file to first node.")
k, err := root.Key()
if err != nil {
t.Fatal(err)
}
wg := sync.WaitGroup{}
for i := 1; i < len(dagservs); i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
first, err := dagservs[i].Get(ctx, k)
if err != nil {
t.Fatal(err)
}
fmt.Println("Got first node back.")
read, err := uio.NewDagReader(ctx, first, dagservs[i])
if err != nil {
t.Fatal(err)
}
datagot, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(datagot, expected) {
t.Fatal("Got bad data back!")
}
}(i)
}
wg.Wait()
}