func TestIndirectBlocks(t *testing.T) { splitter := chunk.SizeSplitterGen(512) nbytes := 1024 * 1024 buf := make([]byte, nbytes) u.NewTimeSeededRand().Read(buf) read := bytes.NewReader(buf) ds := mdtest.Mock() dag, err := buildTestDag(ds, splitter(read)) if err != nil { t.Fatal(err) } reader, err := uio.NewDagReader(context.Background(), dag, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(reader) if err != nil { t.Fatal(err) } if !bytes.Equal(out, buf) { t.Fatal("Not equal!") } }
func (s *Node) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { k, err := s.Nd.Key() if err != nil { return err } // setup our logging event lm := make(lgbl.DeferredMap) lm["fs"] = "ipfs" lm["key"] = func() interface{} { return k.Pretty() } lm["req_offset"] = req.Offset lm["req_size"] = req.Size defer log.EventBegin(ctx, "fuseRead", lm).Done() r, err := uio.NewDagReader(ctx, s.Nd, s.Ipfs.DAG) if err != nil { return err } o, err := r.Seek(req.Offset, os.SEEK_SET) lm["res_offset"] = o if err != nil { return err } buf := resp.Data[:min(req.Size, int(int64(r.Size())-req.Offset))] n, err := io.ReadFull(r, buf) if err != nil && err != io.EOF { return err } resp.Data = resp.Data[:n] lm["res_size"] = n return nil // may be non-nil / not succeeded }
func testFileConsistency(t *testing.T, bs chunk.SplitterGen, nbytes int) { should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, bs(read)) if err != nil { t.Fatal(err) } r, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(r) if err != nil { t.Fatal(err) } err = arrComp(out, should) if err != nil { t.Fatal(err) } }
func TestSeekingConsistency(t *testing.T) { nbytes := int64(128 * 1024) ds := mdtest.Mock() nd, should := getTestDag(t, ds, nbytes, 500) rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } out := make([]byte, nbytes) for coff := nbytes - 4096; coff >= 0; coff -= 4096 { t.Log(coff) n, err := rs.Seek(coff, os.SEEK_SET) if err != nil { t.Fatal(err) } if n != coff { t.Fatal("wasnt able to seek to the right position") } nread, err := rs.Read(out[coff : coff+4096]) if err != nil { t.Fatal(err) } if nread != 4096 { t.Fatal("didnt read the correct number of bytes") } } err = arrComp(out, should) if err != nil { t.Fatal(err) } }
func TestBuilderConsistency(t *testing.T) { nbytes := 100000 buf := new(bytes.Buffer) io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes)) should := dup(buf.Bytes()) dagserv := mdtest.Mock() nd, err := buildTestDag(dagserv, chunk.DefaultSplitter(buf)) if err != nil { t.Fatal(err) } r, err := uio.NewDagReader(context.Background(), nd, dagserv) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(r) if err != nil { t.Fatal(err) } err = arrComp(out, should) if err != nil { t.Fatal(err) } }
func (dm *DagModifier) readPrep() error { err := dm.Sync() if err != nil { return err } if dm.read == nil { ctx, cancel := context.WithCancel(dm.ctx) dr, err := uio.NewDagReader(ctx, dm.curNode, dm.dagserv) if err != nil { return err } i, err := dr.Seek(int64(dm.curWrOff), os.SEEK_SET) if err != nil { return err } if i != int64(dm.curWrOff) { return ErrSeekFail } dm.readCancel = cancel dm.read = dr } return nil }
func TestSeekToAlmostBegin(t *testing.T) { ds := mdtest.Mock() nd, should := getTestDag(t, ds, 10*1024, 500) rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } n, err := io.CopyN(ioutil.Discard, rs, 1024*4) if err != nil { t.Fatal(err) } if n != 4096 { t.Fatal("Copy didnt copy enough bytes") } seeked, err := rs.Seek(1, os.SEEK_SET) if err != nil { t.Fatal(err) } if seeked != 1 { t.Fatal("Failed to seek to almost beginning") } out, err := ioutil.ReadAll(rs) if err != nil { t.Fatal(err) } err = arrComp(out, should[1:]) if err != nil { t.Fatal(err) } }
func TestSeekingBasic(t *testing.T) { nbytes := int64(10 * 1024) ds := mdtest.Mock() nd, should := getTestDag(t, ds, nbytes, 500) rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } start := int64(4000) n, err := rs.Seek(start, os.SEEK_SET) if err != nil { t.Fatal(err) } if n != start { t.Fatal("Failed to seek to correct offset") } out, err := ioutil.ReadAll(rs) if err != nil { t.Fatal(err) } err = arrComp(out, should[start:]) if err != nil { t.Fatal(err) } }
func TestBalancedDag(t *testing.T) { ds := mdtest.Mock() buf := make([]byte, 10000) u.NewTimeSeededRand().Read(buf) r := bytes.NewReader(buf) nd, err := BuildDagFromReader(ds, chunk.DefaultSplitter(r), nil) if err != nil { t.Fatal(err) } dr, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(dr) if err != nil { t.Fatal(err) } if !bytes.Equal(out, buf) { t.Fatal("bad read") } }
func TestSeekEndSingleBlockFile(t *testing.T) { nbytes := int64(100) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 5000)) if err != nil { t.Fatal(err) } rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } seeked, err := rs.Seek(0, os.SEEK_END) if err != nil { t.Fatal(err) } if seeked != nbytes { t.Fatal("Failed to seek to end") } }
func Cat(n *core.IpfsNode, pstr string) (*uio.DagReader, error) { p := path.FromString(pstr) dagNode, err := n.Resolver.ResolvePath(n.Context(), p) if err != nil { return nil, err } return uio.NewDagReader(n.Context(), dagNode, n.DAG) }
func TestMetadata(t *testing.T) { ctx := context.Background() // Make some random node ds := getDagserv(t) data := make([]byte, 1000) u.NewTimeSeededRand().Read(data) r := bytes.NewReader(data) nd, err := importer.BuildDagFromReader(ds, chunk.DefaultSplitter(r), nil) if err != nil { t.Fatal(err) } k, err := nd.Key() if err != nil { t.Fatal(err) } m := new(ft.Metadata) m.MimeType = "THIS IS A TEST" // Such effort, many compromise ipfsnode := &core.IpfsNode{DAG: ds} mdk, err := AddMetadataTo(ipfsnode, k.B58String(), m) if err != nil { t.Fatal(err) } rec, err := Metadata(ipfsnode, mdk) if err != nil { t.Fatal(err) } if rec.MimeType != m.MimeType { t.Fatalf("something went wrong in conversion: '%s' != '%s'", rec.MimeType, m.MimeType) } retnode, err := ds.Get(ctx, key.B58KeyDecode(mdk)) if err != nil { t.Fatal(err) } ndr, err := uio.NewDagReader(ctx, retnode, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(ndr) if err != nil { t.Fatal(err) } if !bytes.Equal(out, data) { t.Fatal("read incorrect data") } }
func TestMultiWrite(t *testing.T) { dserv, pins := getMockDagServ(t) _, n := getNode(t, dserv, 0, pins) ctx, cancel := context.WithCancel(context.Background()) defer cancel() dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512)) if err != nil { t.Fatal(err) } data := make([]byte, 4000) u.NewTimeSeededRand().Read(data) for i := 0; i < len(data); i++ { n, err := dagmod.WriteAt(data[i:i+1], int64(i)) if err != nil { t.Fatal(err) } if n != 1 { t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)") } size, err := dagmod.Size() if err != nil { t.Fatal(err) } if size != int64(i+1) { t.Fatal("Size was reported incorrectly") } } nd, err := dagmod.GetNode() if err != nil { t.Fatal(err) } read, err := uio.NewDagReader(context.Background(), nd, dserv) if err != nil { t.Fatal(err) } rbuf, err := ioutil.ReadAll(read) if err != nil { t.Fatal(err) } err = arrComp(rbuf, data) if err != nil { t.Fatal(err) } }
// This test appends one byte at a time to an empty file func TestMultipleAppends(t *testing.T) { ds := mdtest.Mock() // TODO: fix small size appends and make this number bigger nbytes := int64(1000) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(nil) nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500)) if err != nil { t.Fatal(err) } dbp := &h.DagBuilderParams{ Dagserv: ds, Maxlinks: 4, } spl := chunk.SizeSplitterGen(500) ctx := context.Background() for i := 0; i < len(should); i++ { blks, errs := chunk.Chan(spl(bytes.NewReader(should[i : i+1]))) nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs)) if err != nil { t.Fatal(err) } err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat) if err != nil { t.Fatal(err) } fread, err := uio.NewDagReader(ctx, nnode, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(fread) if err != nil { t.Fatal(err) } err = arrComp(out, should[:i+1]) if err != nil { t.Fatal(err) } } }
func runReadBench(b *testing.B, nd *dag.Node, ds dag.DAGService) { for i := 0; i < b.N; i++ { ctx, cancel := context.WithCancel(context.Background()) read, err := uio.NewDagReader(ctx, nd, ds) if err != nil { b.Fatal(err) } _, err = read.WriteTo(ioutil.Discard) if err != nil && err != io.EOF { b.Fatal(err) } cancel() } }
func TestAppend(t *testing.T) { nbytes := int64(128 * 1024) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) // Reader for half the bytes read := bytes.NewReader(should[:nbytes/2]) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500)) if err != nil { t.Fatal(err) } dbp := &h.DagBuilderParams{ Dagserv: ds, Maxlinks: h.DefaultLinksPerBlock, } r := bytes.NewReader(should[nbytes/2:]) blks, errs := chunk.Chan(chunk.NewSizeSplitter(r, 500)) ctx := context.Background() nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs)) if err != nil { t.Fatal(err) } err = VerifyTrickleDagStructure(nnode, ds, dbp.Maxlinks, layerRepeat) if err != nil { t.Fatal(err) } fread, err := uio.NewDagReader(ctx, nnode, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(fread) if err != nil { t.Fatal(err) } err = arrComp(out, should) if err != nil { t.Fatal(err) } }
func TestAppendSingleBytesToEmpty(t *testing.T) { ds := mdtest.Mock() data := []byte("AB") nd := new(merkledag.Node) nd.Data = ft.FilePBData(nil, 0) dbp := &h.DagBuilderParams{ Dagserv: ds, Maxlinks: 4, } spl := chunk.SizeSplitterGen(500) blks, errs := chunk.Chan(spl(bytes.NewReader(data[:1]))) ctx := context.Background() nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs)) if err != nil { t.Fatal(err) } blks, errs = chunk.Chan(spl(bytes.NewReader(data[1:]))) nnode, err = TrickleAppend(ctx, nnode, dbp.New(blks, errs)) if err != nil { t.Fatal(err) } fread, err := uio.NewDagReader(ctx, nnode, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(fread) if err != nil { t.Fatal(err) } fmt.Println(out, data) err = arrComp(out, data) if err != nil { t.Fatal(err) } }
func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier) []byte { newdata := make([]byte, size) r := u.NewTimeSeededRand() r.Read(newdata) if size+beg > uint64(len(orig)) { orig = append(orig, make([]byte, (size+beg)-uint64(len(orig)))...) } copy(orig[beg:], newdata) nmod, err := dm.WriteAt(newdata, int64(beg)) if err != nil { t.Fatal(err) } if nmod != int(size) { t.Fatalf("Mod length not correct! %d != %d", nmod, size) } nd, err := dm.GetNode() if err != nil { t.Fatal(err) } err = trickle.VerifyTrickleDagStructure(nd, dm.dagserv, h.DefaultLinksPerBlock, 4) if err != nil { t.Fatal(err) } rd, err := uio.NewDagReader(context.Background(), nd, dm.dagserv) if err != nil { t.Fatal(err) } after, err := ioutil.ReadAll(rd) if err != nil { t.Fatal(err) } err = arrComp(after, orig) if err != nil { t.Fatal(err) } return orig }
func TestSeekEndSingleBlockFile(t *testing.T) { nbytes := int64(100) ds := mdtest.Mock() nd, _ := getTestDag(t, ds, nbytes, 5000) rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } seeked, err := rs.Seek(0, os.SEEK_END) if err != nil { t.Fatal(err) } if seeked != nbytes { t.Fatal("Failed to seek to end") } }
func TestIndirectBlocks(t *testing.T) { ds := mdtest.Mock() dag, buf := getTestDag(t, ds, 1024*1024, 512) reader, err := uio.NewDagReader(context.Background(), dag, ds) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(reader) if err != nil { t.Fatal(err) } if !bytes.Equal(out, buf) { t.Fatal("Not equal!") } }
func TestSeekingStress(t *testing.T) { nbytes := int64(1024 * 1024) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 1000)) if err != nil { t.Fatal(err) } rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } testbuf := make([]byte, nbytes) for i := 0; i < 50; i++ { offset := mrand.Intn(int(nbytes)) l := int(nbytes) - offset n, err := rs.Seek(int64(offset), os.SEEK_SET) if err != nil { t.Fatal(err) } if n != int64(offset) { t.Fatal("Seek failed to move to correct position") } nread, err := rs.Read(testbuf[:l]) if err != nil { t.Fatal(err) } if nread != l { t.Fatal("Failed to read enough bytes") } err = arrComp(testbuf[:l], should[offset:offset+l]) if err != nil { t.Fatal(err) } } }
func getNode(t testing.TB, dserv mdag.DAGService, size int64, pinner pin.ManualPinner) ([]byte, *mdag.Node) { in := io.LimitReader(u.NewTimeSeededRand(), size) node, err := imp.BuildTrickleDagFromReader(dserv, sizeSplitterGen(500)(in), imp.BasicPinnerCB(pinner)) if err != nil { t.Fatal(err) } dr, err := uio.NewDagReader(context.Background(), node, dserv) if err != nil { t.Fatal(err) } b, err := ioutil.ReadAll(dr) if err != nil { t.Fatal(err) } return b, node }
func TestSeekToAlmostBegin(t *testing.T) { nbytes := int64(10 * 1024) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 500)) if err != nil { t.Fatal(err) } rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } n, err := io.CopyN(ioutil.Discard, rs, 1024*4) if err != nil { t.Fatal(err) } if n != 4096 { t.Fatal("Copy didnt copy enough bytes") } seeked, err := rs.Seek(1, os.SEEK_SET) if err != nil { t.Fatal(err) } if seeked != 1 { t.Fatal("Failed to seek to almost beginning") } out, err := ioutil.ReadAll(rs) if err != nil { t.Fatal(err) } err = arrComp(out, should[1:]) if err != nil { t.Fatal(err) } }
func TestBuilderConsistency(t *testing.T) { dagserv := mdtest.Mock() nd, should := getTestDag(t, dagserv, 100000, chunk.DefaultBlockSize) r, err := uio.NewDagReader(context.Background(), nd, dagserv) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(r) if err != nil { t.Fatal(err) } err = arrComp(out, should) if err != nil { t.Fatal(err) } }
func TestWriteNewFile(t *testing.T) { dserv, pins := getMockDagServ(t) _, n := getNode(t, dserv, 0, pins) ctx, cancel := context.WithCancel(context.Background()) defer cancel() dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512)) if err != nil { t.Fatal(err) } towrite := make([]byte, 2000) u.NewTimeSeededRand().Read(towrite) nw, err := dagmod.Write(towrite) if err != nil { t.Fatal(err) } if nw != len(towrite) { t.Fatal("Wrote wrong amount") } nd, err := dagmod.GetNode() if err != nil { t.Fatal(err) } read, err := uio.NewDagReader(ctx, nd, dserv) if err != nil { t.Fatal(err) } data, err := ioutil.ReadAll(read) if err != nil { t.Fatal(err) } if err := arrComp(data, towrite); err != nil { t.Fatal(err) } }
func TestSeekingBasic(t *testing.T) { nbytes := int64(10 * 1024) should := make([]byte, nbytes) u.NewTimeSeededRand().Read(should) read := bytes.NewReader(should) ds := mdtest.Mock() nd, err := buildTestDag(ds, chunk.NewSizeSplitter(read, 512)) if err != nil { t.Fatal(err) } rs, err := uio.NewDagReader(context.Background(), nd, ds) if err != nil { t.Fatal(err) } start := int64(4000) n, err := rs.Seek(start, os.SEEK_SET) if err != nil { t.Fatal(err) } if n != start { t.Fatal("Failed to seek to correct offset") } out, err := ioutil.ReadAll(rs) if err != nil { t.Fatal(err) } err = arrComp(out, should[start:]) if err != nil { t.Fatal(err) } }
func (tr *tarReader) Read(b []byte) (int, error) { // if we have a header to be read, it takes priority if tr.hdrBuf != nil { n, err := tr.hdrBuf.Read(b) if err == io.EOF { tr.hdrBuf = nil return n, nil } return n, err } // no header remaining, check for recursive if tr.childRead != nil { n, err := tr.childRead.Read(b) if err == io.EOF { tr.childRead = nil return n, nil } return n, err } // check for filedata to be read if tr.fileRead != nil { n, err := tr.fileRead.Read(b) if err == io.EOF { nr := tr.fileRead.n tr.pad = (blockSize - (nr % blockSize)) % blockSize tr.fileRead.Close() tr.fileRead = nil return n, nil } return n, err } // filedata reads must be padded out to 512 byte offsets if tr.pad > 0 { n := copy(b, zeroBlock[:tr.pad]) tr.pad -= n return n, nil } if len(tr.links) == 0 { return 0, io.EOF } next := tr.links[0] tr.links = tr.links[1:] headerNd, err := next.GetNode(tr.ctx, tr.ds) if err != nil { return 0, err } tr.hdrBuf = bytes.NewReader(headerNd.Data) dataNd, err := headerNd.GetLinkedNode(tr.ctx, tr.ds, "data") if err != nil && err != dag.ErrNotFound { return 0, err } if err == nil { dr, err := uio.NewDagReader(tr.ctx, dataNd, tr.ds) if err != nil { log.Error("dagreader error: ", err) return 0, err } tr.fileRead = &countReader{r: dr} } else if len(headerNd.Links) > 0 { tr.childRead = &tarReader{ links: headerNd.Links, ds: tr.ds, ctx: tr.ctx, } } return tr.Read(b) }
func TestCorrectPinning(t *testing.T) { dserv, bstore, pins := getMockDagServAndBstore(t) b, n := getNode(t, dserv, 50000, pins) ctx, cancel := context.WithCancel(context.Background()) defer cancel() dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512)) if err != nil { t.Fatal(err) } buf := make([]byte, 1024) for i := 0; i < 100; i++ { size, err := dagmod.Size() if err != nil { t.Fatal(err) } offset := rand.Intn(int(size)) u.NewTimeSeededRand().Read(buf) if offset+len(buf) > int(size) { b = append(b[:offset], buf...) } else { copy(b[offset:], buf) } n, err := dagmod.WriteAt(buf, int64(offset)) if err != nil { t.Fatal(err) } if n != len(buf) { t.Fatal("wrote incorrect number of bytes") } } fisize, err := dagmod.Size() if err != nil { t.Fatal(err) } if int(fisize) != len(b) { t.Fatal("reported filesize incorrect", fisize, len(b)) } // Run a GC, then ensure we can still read the file correctly basicGC(t, bstore, pins) nd, err := dagmod.GetNode() if err != nil { t.Fatal(err) } read, err := uio.NewDagReader(context.Background(), nd, dserv) if err != nil { t.Fatal(err) } out, err := ioutil.ReadAll(read) if err != nil { t.Fatal(err) } if err = arrComp(out, b); err != nil { t.Fatal(err) } rootk, err := nd.Key() if err != nil { t.Fatal(err) } // Verify only one recursive pin recpins := pins.RecursiveKeys() if len(recpins) != 1 { t.Fatal("Incorrect number of pinned entries") } // verify the correct node is pinned if recpins[0] != rootk { t.Fatal("Incorrect node recursively pinned") } indirpins := pins.IndirectKeys() children := enumerateChildren(t, nd, dserv) if len(indirpins) != len(children) { t.Log(len(indirpins), len(children)) t.Fatal("Incorrect number of indirectly pinned blocks") } }
func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithCancel(i.node.Context()) defer cancel() urlPath := r.URL.Path // IPNSHostnameOption might have constructed an IPNS path using the Host header. // In this case, we need the original path for constructing redirects // and links that match the requested URL. // For example, http://example.net would become /ipns/example.net, and // the redirects and links would end up as http://example.net/ipns/example.net originalUrlPath := urlPath ipnsHostname := false hdr := r.Header["X-IPNS-Original-Path"] if len(hdr) > 0 { originalUrlPath = hdr[0] ipnsHostname = true } if i.config.BlockList != nil && i.config.BlockList.ShouldBlock(urlPath) { w.WriteHeader(http.StatusForbidden) w.Write([]byte("403 - Forbidden")) return } nd, err := core.Resolve(ctx, i.node, path.Path(urlPath)) if err != nil { webError(w, "Path Resolve error", err, http.StatusBadRequest) return } etag := gopath.Base(urlPath) if r.Header.Get("If-None-Match") == etag { w.WriteHeader(http.StatusNotModified) return } i.addUserHeaders(w) // ok, _now_ write user's headers. w.Header().Set("X-IPFS-Path", urlPath) // Suborigin header, sandboxes apps from each other in the browser (even // though they are served from the same gateway domain). // // Omited if the path was treated by IPNSHostnameOption(), for example // a request for http://example.net/ would be changed to /ipns/example.net/, // which would turn into an incorrect Suborigin: example.net header. // // NOTE: This is not yet widely supported by browsers. if !ipnsHostname { pathRoot := strings.SplitN(urlPath, "/", 4)[2] w.Header().Set("Suborigin", pathRoot) } dr, err := uio.NewDagReader(ctx, nd, i.node.DAG) if err != nil && err != uio.ErrIsDir { // not a directory and still an error internalWebError(w, err) return } // set these headers _after_ the error, for we may just not have it // and dont want the client to cache a 500 response... // and only if it's /ipfs! // TODO: break this out when we split /ipfs /ipns routes. modtime := time.Now() if strings.HasPrefix(urlPath, ipfsPathPrefix) { w.Header().Set("Etag", etag) w.Header().Set("Cache-Control", "public, max-age=29030400") // set modtime to a really long time ago, since files are immutable and should stay cached modtime = time.Unix(1, 0) } if err == nil { defer dr.Close() _, name := gopath.Split(urlPath) http.ServeContent(w, r, name, modtime, dr) return } // storage for directory listing var dirListing []directoryItem // loop through files foundIndex := false for _, link := range nd.Links { if link.Name == "index.html" { log.Debugf("found index.html link for %s", urlPath) foundIndex = true if urlPath[len(urlPath)-1] != '/' { // See comment above where originalUrlPath is declared. http.Redirect(w, r, originalUrlPath+"/", 302) log.Debugf("redirect to %s", originalUrlPath+"/") return } // return index page instead. nd, err := core.Resolve(ctx, i.node, path.Path(urlPath+"/index.html")) if err != nil { internalWebError(w, err) return } dr, err := uio.NewDagReader(ctx, nd, i.node.DAG) if err != nil { internalWebError(w, err) return } defer dr.Close() // write to request if r.Method != "HEAD" { io.Copy(w, dr) } break } // See comment above where originalUrlPath is declared. di := directoryItem{humanize.Bytes(link.Size), link.Name, gopath.Join(originalUrlPath, link.Name)} dirListing = append(dirListing, di) } if !foundIndex { if r.Method != "HEAD" { // construct the correct back link // https://github.com/djbarber/ipfs-hack/issues/1365 var backLink string = urlPath // don't go further up than /ipfs/$hash/ pathSplit := strings.Split(backLink, "/") switch { // keep backlink case len(pathSplit) == 3: // url: /ipfs/$hash // keep backlink case len(pathSplit) == 4 && pathSplit[3] == "": // url: /ipfs/$hash/ // add the correct link depending on wether the path ends with a slash default: if strings.HasSuffix(backLink, "/") { backLink += "./.." } else { backLink += "/.." } } // strip /ipfs/$hash from backlink if IPNSHostnameOption touched the path. if ipnsHostname { backLink = "/" if len(pathSplit) > 5 { // also strip the trailing segment, because it's a backlink backLinkParts := pathSplit[3 : len(pathSplit)-2] backLink += strings.Join(backLinkParts, "/") + "/" } } // See comment above where originalUrlPath is declared. tplData := listingTemplateData{ Listing: dirListing, Path: originalUrlPath, BackLink: backLink, } err := listingTemplate.Execute(w, tplData) if err != nil { internalWebError(w, err) return } } } }
func runBatchFetchTest(t *testing.T, read io.Reader) { ctx := context.Background() var dagservs []DAGService for _, bsi := range bstest.Mocks(5) { dagservs = append(dagservs, NewDAGService(bsi)) } spl := chunk.NewSizeSplitter(read, 512) root, err := imp.BuildDagFromReader(dagservs[0], spl, nil) if err != nil { t.Fatal(err) } t.Log("finished setup.") dagr, err := uio.NewDagReader(ctx, root, dagservs[0]) if err != nil { t.Fatal(err) } expected, err := ioutil.ReadAll(dagr) if err != nil { t.Fatal(err) } err = dagservs[0].AddRecursive(root) if err != nil { t.Fatal(err) } t.Log("Added file to first node.") k, err := root.Key() if err != nil { t.Fatal(err) } wg := sync.WaitGroup{} for i := 1; i < len(dagservs); i++ { wg.Add(1) go func(i int) { defer wg.Done() first, err := dagservs[i].Get(ctx, k) if err != nil { t.Fatal(err) } fmt.Println("Got first node back.") read, err := uio.NewDagReader(ctx, first, dagservs[i]) if err != nil { t.Fatal(err) } datagot, err := ioutil.ReadAll(read) if err != nil { t.Fatal(err) } if !bytes.Equal(datagot, expected) { t.Fatal("Got bad data back!") } }(i) } wg.Wait() }