func walkItems(ctx context.Context, dag merkledag.DAGService, n *merkledag.ProtoNode, fn walkerFunc, children keyObserver) error { hdr, err := readHdr(n) if err != nil { return err } // readHdr guarantees fanout is a safe value fanout := hdr.GetFanout() for i, l := range n.Links()[fanout:] { if err := fn(i, l); err != nil { return err } } for _, l := range n.Links()[:fanout] { c := l.Cid children(c) if c.Equals(emptyKey) { continue } subtree, err := l.GetNode(ctx, dag) if err != nil { return err } stpb, ok := subtree.(*merkledag.ProtoNode) if !ok { return merkledag.ErrNotProtobuf } if err := walkItems(ctx, dag, stpb, fn, children); err != nil { return err } } return nil }
// converts the Node object into a real dag.ProtoNode func deserializeNode(nd *Node, dataFieldEncoding string) (*dag.ProtoNode, error) { dagnode := new(dag.ProtoNode) switch dataFieldEncoding { case "text": dagnode.SetData([]byte(nd.Data)) case "base64": data, _ := base64.StdEncoding.DecodeString(nd.Data) dagnode.SetData(data) default: return nil, fmt.Errorf("Unkown data field encoding") } dagnode.SetLinks(make([]*node.Link, len(nd.Links))) for i, link := range nd.Links { c, err := cid.Decode(link.Hash) if err != nil { return nil, err } dagnode.Links()[i] = &node.Link{ Name: link.Name, Size: link.Size, Cid: c, } } return dagnode, nil }
func readHdr(n *merkledag.ProtoNode) (*pb.Set, error) { hdrLenRaw, consumed := binary.Uvarint(n.Data()) if consumed <= 0 { return nil, errors.New("invalid Set header length") } pbdata := n.Data()[consumed:] if hdrLenRaw > uint64(len(pbdata)) { return nil, errors.New("impossibly large Set header length") } // as hdrLenRaw was <= an int, we now know it fits in an int hdrLen := int(hdrLenRaw) var hdr pb.Set if err := proto.Unmarshal(pbdata[:hdrLen], &hdr); err != nil { return nil, err } if v := hdr.GetVersion(); v != 1 { return nil, fmt.Errorf("unsupported Set version: %d", v) } if uint64(hdr.GetFanout()) > uint64(len(n.Links())) { return nil, errors.New("impossibly large Fanout") } return &hdr, nil }
func copyDag(nd *dag.ProtoNode, from, to dag.DAGService) error { _, err := to.Add(nd) if err != nil { return err } for _, lnk := range nd.Links() { child, err := lnk.GetNode(context.Background(), from) if err != nil { if err == dag.ErrNotFound { // not found means we didnt modify it, and it should // already be in the target datastore continue } return err } childpb, ok := child.(*dag.ProtoNode) if !ok { return dag.ErrNotProtobuf } err = copyDag(childpb, from, to) if err != nil { return err } } return nil }
func ExportTar(ctx context.Context, root *dag.ProtoNode, ds dag.DAGService) (io.Reader, error) { if string(root.Data()) != "ipfs/tar" { return nil, errors.New("not an IPFS tarchive") } return &tarReader{ links: root.Links(), ds: ds, ctx: ctx, }, nil }
func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.ProtoNode) []string { if len(n.Links()) == 0 { return []string{name} } var out []string for _, lnk := range n.Links() { child, err := lnk.GetNode(ipfs.Context(), ipfs.DAG) if err != nil { t.Fatal(err) } childpb, ok := child.(*dag.ProtoNode) if !ok { t.Fatal(dag.ErrNotProtobuf) } sub := getPaths(t, ipfs, path.Join(name, lnk.Name), childpb) out = append(out, sub...) } return out }
func PrintDag(nd *mdag.ProtoNode, ds mdag.DAGService, indent int) { pbd, err := ft.FromBytes(nd.Data()) if err != nil { panic(err) } for i := 0; i < indent; i++ { fmt.Print(" ") } fmt.Printf("{size = %d, type = %s, children = %d", pbd.GetFilesize(), pbd.GetType().String(), len(pbd.GetBlocksizes())) if len(nd.Links()) > 0 { fmt.Println() } for _, lnk := range nd.Links() { child, err := lnk.GetNode(context.Background(), ds) if err != nil { panic(err) } PrintDag(child.(*mdag.ProtoNode), ds, indent+1) } if len(nd.Links()) > 0 { for i := 0; i < indent; i++ { fmt.Print(" ") } } fmt.Println("}") }
func (w *Writer) writeDir(nd *mdag.ProtoNode, fpath string) error { if err := writeDirHeader(w.TarW, fpath); err != nil { return err } for i, ng := range mdag.GetDAG(w.ctx, w.Dag, nd) { child, err := ng.Get(w.ctx) if err != nil { return err } childpb, ok := child.(*mdag.ProtoNode) if !ok { return mdag.ErrNotProtobuf } npath := path.Join(fpath, nd.Links()[i].Name) if err := w.WriteNode(childpb, npath); err != nil { return err } } return nil }
func Diff(ctx context.Context, ds dag.DAGService, a, b *dag.ProtoNode) ([]*Change, error) { if len(a.Links()) == 0 && len(b.Links()) == 0 { return []*Change{ &Change{ Type: Mod, Before: a.Cid(), After: b.Cid(), }, }, nil } var out []*Change clean_a := a.Copy().(*dag.ProtoNode) clean_b := b.Copy().(*dag.ProtoNode) // strip out unchanged stuff for _, lnk := range a.Links() { l, err := b.GetNodeLink(lnk.Name) if err == nil { if l.Cid.Equals(lnk.Cid) { // no change... ignore it } else { anode, err := lnk.GetNode(ctx, ds) if err != nil { return nil, err } bnode, err := l.GetNode(ctx, ds) if err != nil { return nil, err } anodepb, ok := anode.(*dag.ProtoNode) if !ok { return nil, dag.ErrNotProtobuf } bnodepb, ok := bnode.(*dag.ProtoNode) if !ok { return nil, dag.ErrNotProtobuf } sub, err := Diff(ctx, ds, anodepb, bnodepb) if err != nil { return nil, err } for _, subc := range sub { subc.Path = path.Join(lnk.Name, subc.Path) out = append(out, subc) } } clean_a.RemoveNodeLink(l.Name) clean_b.RemoveNodeLink(l.Name) } } for _, lnk := range clean_a.Links() { out = append(out, &Change{ Type: Remove, Path: lnk.Name, Before: lnk.Cid, }) } for _, lnk := range clean_b.Links() { out = append(out, &Change{ Type: Add, Path: lnk.Name, After: lnk.Cid, }) } return out, nil }
// dagTruncate truncates the given node to 'size' and returns the modified Node func dagTruncate(ctx context.Context, nd *mdag.ProtoNode, size uint64, ds mdag.DAGService) (*mdag.ProtoNode, error) { if len(nd.Links()) == 0 { // TODO: this can likely be done without marshaling and remarshaling pbn, err := ft.FromBytes(nd.Data()) if err != nil { return nil, err } nd.SetData(ft.WrapData(pbn.Data[:size])) return nd, nil } var cur uint64 end := 0 var modified *mdag.ProtoNode ndata := new(ft.FSNode) for i, lnk := range nd.Links() { child, err := lnk.GetNode(ctx, ds) if err != nil { return nil, err } childpb, ok := child.(*mdag.ProtoNode) if !ok { return nil, err } childsize, err := ft.DataSize(childpb.Data()) if err != nil { return nil, err } // found the child we want to cut if size < cur+childsize { nchild, err := dagTruncate(ctx, childpb, size-cur, ds) if err != nil { return nil, err } ndata.AddBlockSize(size - cur) modified = nchild end = i break } cur += childsize ndata.AddBlockSize(childsize) } _, err := ds.Add(modified) if err != nil { return nil, err } nd.SetLinks(nd.Links()[:end]) err = nd.AddNodeLinkClean("", modified) if err != nil { return nil, err } d, err := ndata.GetBytes() if err != nil { return nil, err } nd.SetData(d) // invalidate cache and recompute serialized data _, err = nd.EncodeProtobuf(true) if err != nil { return nil, err } return nd, nil }
// modifyDag writes the data in 'data' over the data in 'node' starting at 'offset' // returns the new key of the passed in node and whether or not all the data in the reader // has been consumed. func (dm *DagModifier) modifyDag(node *mdag.ProtoNode, offset uint64, data io.Reader) (*cid.Cid, bool, error) { f, err := ft.FromBytes(node.Data()) if err != nil { return nil, false, err } // If we've reached a leaf node. if len(node.Links()) == 0 { n, err := data.Read(f.Data[offset:]) if err != nil && err != io.EOF { return nil, false, err } // Update newly written node.. b, err := proto.Marshal(f) if err != nil { return nil, false, err } nd := new(mdag.ProtoNode) nd.SetData(b) k, err := dm.dagserv.Add(nd) if err != nil { return nil, false, err } // Hey look! we're done! var done bool if n < len(f.Data[offset:]) { done = true } return k, done, nil } var cur uint64 var done bool for i, bs := range f.GetBlocksizes() { // We found the correct child to write into if cur+bs > offset { child, err := node.Links()[i].GetNode(dm.ctx, dm.dagserv) if err != nil { return nil, false, err } childpb, ok := child.(*mdag.ProtoNode) if !ok { return nil, false, mdag.ErrNotProtobuf } k, sdone, err := dm.modifyDag(childpb, offset-cur, data) if err != nil { return nil, false, err } offset += bs node.Links()[i].Cid = k // Recache serialized node _, err = node.EncodeProtobuf(true) if err != nil { return nil, false, err } if sdone { // No more bytes to write! done = true break } offset = cur + bs } cur += bs } k, err := dm.dagserv.Add(node) return k, done, err }
// Recursive call for verifying the structure of a trickledag func verifyTDagRec(nd *dag.ProtoNode, depth, direct, layerRepeat int, ds dag.DAGService) error { if depth == 0 { // zero depth dag is raw data block if len(nd.Links()) > 0 { return errors.New("expected direct block") } pbn, err := ft.FromBytes(nd.Data()) if err != nil { return err } if pbn.GetType() != ft.TRaw { return errors.New("Expected raw block") } return nil } // Verify this is a branch node pbn, err := ft.FromBytes(nd.Data()) if err != nil { return err } if pbn.GetType() != ft.TFile { return fmt.Errorf("expected file as branch node, got: %s", pbn.GetType()) } if len(pbn.Data) > 0 { return errors.New("branch node should not have data") } for i := 0; i < len(nd.Links()); i++ { childi, err := nd.Links()[i].GetNode(context.TODO(), ds) if err != nil { return err } childpb, ok := childi.(*dag.ProtoNode) if !ok { return fmt.Errorf("cannot operate on non-protobuf nodes") } if i < direct { // Direct blocks err := verifyTDagRec(childpb, 0, direct, layerRepeat, ds) if err != nil { return err } } else { // Recursive trickle dags rdepth := ((i - direct) / layerRepeat) + 1 if rdepth >= depth && depth > 0 { return errors.New("Child dag was too deep!") } err := verifyTDagRec(childpb, rdepth, direct, layerRepeat, ds) if err != nil { return err } } } return nil }