func walkItems(ctx context.Context, dag merkledag.DAGService, n *merkledag.ProtoNode, fn walkerFunc, children keyObserver) error {
hdr, err := readHdr(n)
if err != nil {
return err
}
// readHdr guarantees fanout is a safe value
fanout := hdr.GetFanout()
for i, l := range n.Links()[fanout:] {
if err := fn(i, l); err != nil {
return err
}
}
for _, l := range n.Links()[:fanout] {
c := l.Cid
children(c)
if c.Equals(emptyKey) {
continue
}
subtree, err := l.GetNode(ctx, dag)
if err != nil {
return err
}
stpb, ok := subtree.(*merkledag.ProtoNode)
if !ok {
return merkledag.ErrNotProtobuf
}
if err := walkItems(ctx, dag, stpb, fn, children); err != nil {
return err
}
}
return nil
}
// converts the Node object into a real dag.ProtoNode
func deserializeNode(nd *Node, dataFieldEncoding string) (*dag.ProtoNode, error) {
dagnode := new(dag.ProtoNode)
switch dataFieldEncoding {
case "text":
dagnode.SetData([]byte(nd.Data))
case "base64":
data, _ := base64.StdEncoding.DecodeString(nd.Data)
dagnode.SetData(data)
default:
return nil, fmt.Errorf("Unkown data field encoding")
}
dagnode.SetLinks(make([]*node.Link, len(nd.Links)))
for i, link := range nd.Links {
c, err := cid.Decode(link.Hash)
if err != nil {
return nil, err
}
dagnode.Links()[i] = &node.Link{
Name: link.Name,
Size: link.Size,
Cid: c,
}
}
return dagnode, nil
}
func readHdr(n *merkledag.ProtoNode) (*pb.Set, error) {
hdrLenRaw, consumed := binary.Uvarint(n.Data())
if consumed <= 0 {
return nil, errors.New("invalid Set header length")
}
pbdata := n.Data()[consumed:]
if hdrLenRaw > uint64(len(pbdata)) {
return nil, errors.New("impossibly large Set header length")
}
// as hdrLenRaw was <= an int, we now know it fits in an int
hdrLen := int(hdrLenRaw)
var hdr pb.Set
if err := proto.Unmarshal(pbdata[:hdrLen], &hdr); err != nil {
return nil, err
}
if v := hdr.GetVersion(); v != 1 {
return nil, fmt.Errorf("unsupported Set version: %d", v)
}
if uint64(hdr.GetFanout()) > uint64(len(n.Links())) {
return nil, errors.New("impossibly large Fanout")
}
return &hdr, nil
}
func copyDag(nd *dag.ProtoNode, from, to dag.DAGService) error {
_, err := to.Add(nd)
if err != nil {
return err
}
for _, lnk := range nd.Links() {
child, err := lnk.GetNode(context.Background(), from)
if err != nil {
if err == dag.ErrNotFound {
// not found means we didnt modify it, and it should
// already be in the target datastore
continue
}
return err
}
childpb, ok := child.(*dag.ProtoNode)
if !ok {
return dag.ErrNotProtobuf
}
err = copyDag(childpb, from, to)
if err != nil {
return err
}
}
return nil
}
func ExportTar(ctx context.Context, root *dag.ProtoNode, ds dag.DAGService) (io.Reader, error) {
if string(root.Data()) != "ipfs/tar" {
return nil, errors.New("not an IPFS tarchive")
}
return &tarReader{
links: root.Links(),
ds: ds,
ctx: ctx,
}, nil
}
func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.ProtoNode) []string {
if len(n.Links()) == 0 {
return []string{name}
}
var out []string
for _, lnk := range n.Links() {
child, err := lnk.GetNode(ipfs.Context(), ipfs.DAG)
if err != nil {
t.Fatal(err)
}
childpb, ok := child.(*dag.ProtoNode)
if !ok {
t.Fatal(dag.ErrNotProtobuf)
}
sub := getPaths(t, ipfs, path.Join(name, lnk.Name), childpb)
out = append(out, sub...)
}
return out
}
func PrintDag(nd *mdag.ProtoNode, ds mdag.DAGService, indent int) {
pbd, err := ft.FromBytes(nd.Data())
if err != nil {
panic(err)
}
for i := 0; i < indent; i++ {
fmt.Print(" ")
}
fmt.Printf("{size = %d, type = %s, children = %d", pbd.GetFilesize(), pbd.GetType().String(), len(pbd.GetBlocksizes()))
if len(nd.Links()) > 0 {
fmt.Println()
}
for _, lnk := range nd.Links() {
child, err := lnk.GetNode(context.Background(), ds)
if err != nil {
panic(err)
}
PrintDag(child.(*mdag.ProtoNode), ds, indent+1)
}
if len(nd.Links()) > 0 {
for i := 0; i < indent; i++ {
fmt.Print(" ")
}
}
fmt.Println("}")
}
func (w *Writer) writeDir(nd *mdag.ProtoNode, fpath string) error {
if err := writeDirHeader(w.TarW, fpath); err != nil {
return err
}
for i, ng := range mdag.GetDAG(w.ctx, w.Dag, nd) {
child, err := ng.Get(w.ctx)
if err != nil {
return err
}
childpb, ok := child.(*mdag.ProtoNode)
if !ok {
return mdag.ErrNotProtobuf
}
npath := path.Join(fpath, nd.Links()[i].Name)
if err := w.WriteNode(childpb, npath); err != nil {
return err
}
}
return nil
}
func Diff(ctx context.Context, ds dag.DAGService, a, b *dag.ProtoNode) ([]*Change, error) {
if len(a.Links()) == 0 && len(b.Links()) == 0 {
return []*Change{
&Change{
Type: Mod,
Before: a.Cid(),
After: b.Cid(),
},
}, nil
}
var out []*Change
clean_a := a.Copy().(*dag.ProtoNode)
clean_b := b.Copy().(*dag.ProtoNode)
// strip out unchanged stuff
for _, lnk := range a.Links() {
l, err := b.GetNodeLink(lnk.Name)
if err == nil {
if l.Cid.Equals(lnk.Cid) {
// no change... ignore it
} else {
anode, err := lnk.GetNode(ctx, ds)
if err != nil {
return nil, err
}
bnode, err := l.GetNode(ctx, ds)
if err != nil {
return nil, err
}
anodepb, ok := anode.(*dag.ProtoNode)
if !ok {
return nil, dag.ErrNotProtobuf
}
bnodepb, ok := bnode.(*dag.ProtoNode)
if !ok {
return nil, dag.ErrNotProtobuf
}
sub, err := Diff(ctx, ds, anodepb, bnodepb)
if err != nil {
return nil, err
}
for _, subc := range sub {
subc.Path = path.Join(lnk.Name, subc.Path)
out = append(out, subc)
}
}
clean_a.RemoveNodeLink(l.Name)
clean_b.RemoveNodeLink(l.Name)
}
}
for _, lnk := range clean_a.Links() {
out = append(out, &Change{
Type: Remove,
Path: lnk.Name,
Before: lnk.Cid,
})
}
for _, lnk := range clean_b.Links() {
out = append(out, &Change{
Type: Add,
Path: lnk.Name,
After: lnk.Cid,
})
}
return out, nil
}
// dagTruncate truncates the given node to 'size' and returns the modified Node
func dagTruncate(ctx context.Context, nd *mdag.ProtoNode, size uint64, ds mdag.DAGService) (*mdag.ProtoNode, error) {
if len(nd.Links()) == 0 {
// TODO: this can likely be done without marshaling and remarshaling
pbn, err := ft.FromBytes(nd.Data())
if err != nil {
return nil, err
}
nd.SetData(ft.WrapData(pbn.Data[:size]))
return nd, nil
}
var cur uint64
end := 0
var modified *mdag.ProtoNode
ndata := new(ft.FSNode)
for i, lnk := range nd.Links() {
child, err := lnk.GetNode(ctx, ds)
if err != nil {
return nil, err
}
childpb, ok := child.(*mdag.ProtoNode)
if !ok {
return nil, err
}
childsize, err := ft.DataSize(childpb.Data())
if err != nil {
return nil, err
}
// found the child we want to cut
if size < cur+childsize {
nchild, err := dagTruncate(ctx, childpb, size-cur, ds)
if err != nil {
return nil, err
}
ndata.AddBlockSize(size - cur)
modified = nchild
end = i
break
}
cur += childsize
ndata.AddBlockSize(childsize)
}
_, err := ds.Add(modified)
if err != nil {
return nil, err
}
nd.SetLinks(nd.Links()[:end])
err = nd.AddNodeLinkClean("", modified)
if err != nil {
return nil, err
}
d, err := ndata.GetBytes()
if err != nil {
return nil, err
}
nd.SetData(d)
// invalidate cache and recompute serialized data
_, err = nd.EncodeProtobuf(true)
if err != nil {
return nil, err
}
return nd, nil
}
// modifyDag writes the data in 'data' over the data in 'node' starting at 'offset'
// returns the new key of the passed in node and whether or not all the data in the reader
// has been consumed.
func (dm *DagModifier) modifyDag(node *mdag.ProtoNode, offset uint64, data io.Reader) (*cid.Cid, bool, error) {
f, err := ft.FromBytes(node.Data())
if err != nil {
return nil, false, err
}
// If we've reached a leaf node.
if len(node.Links()) == 0 {
n, err := data.Read(f.Data[offset:])
if err != nil && err != io.EOF {
return nil, false, err
}
// Update newly written node..
b, err := proto.Marshal(f)
if err != nil {
return nil, false, err
}
nd := new(mdag.ProtoNode)
nd.SetData(b)
k, err := dm.dagserv.Add(nd)
if err != nil {
return nil, false, err
}
// Hey look! we're done!
var done bool
if n < len(f.Data[offset:]) {
done = true
}
return k, done, nil
}
var cur uint64
var done bool
for i, bs := range f.GetBlocksizes() {
// We found the correct child to write into
if cur+bs > offset {
child, err := node.Links()[i].GetNode(dm.ctx, dm.dagserv)
if err != nil {
return nil, false, err
}
childpb, ok := child.(*mdag.ProtoNode)
if !ok {
return nil, false, mdag.ErrNotProtobuf
}
k, sdone, err := dm.modifyDag(childpb, offset-cur, data)
if err != nil {
return nil, false, err
}
offset += bs
node.Links()[i].Cid = k
// Recache serialized node
_, err = node.EncodeProtobuf(true)
if err != nil {
return nil, false, err
}
if sdone {
// No more bytes to write!
done = true
break
}
offset = cur + bs
}
cur += bs
}
k, err := dm.dagserv.Add(node)
return k, done, err
}
// Recursive call for verifying the structure of a trickledag
func verifyTDagRec(nd *dag.ProtoNode, depth, direct, layerRepeat int, ds dag.DAGService) error {
if depth == 0 {
// zero depth dag is raw data block
if len(nd.Links()) > 0 {
return errors.New("expected direct block")
}
pbn, err := ft.FromBytes(nd.Data())
if err != nil {
return err
}
if pbn.GetType() != ft.TRaw {
return errors.New("Expected raw block")
}
return nil
}
// Verify this is a branch node
pbn, err := ft.FromBytes(nd.Data())
if err != nil {
return err
}
if pbn.GetType() != ft.TFile {
return fmt.Errorf("expected file as branch node, got: %s", pbn.GetType())
}
if len(pbn.Data) > 0 {
return errors.New("branch node should not have data")
}
for i := 0; i < len(nd.Links()); i++ {
childi, err := nd.Links()[i].GetNode(context.TODO(), ds)
if err != nil {
return err
}
childpb, ok := childi.(*dag.ProtoNode)
if !ok {
return fmt.Errorf("cannot operate on non-protobuf nodes")
}
if i < direct {
// Direct blocks
err := verifyTDagRec(childpb, 0, direct, layerRepeat, ds)
if err != nil {
return err
}
} else {
// Recursive trickle dags
rdepth := ((i - direct) / layerRepeat) + 1
if rdepth >= depth && depth > 0 {
return errors.New("Child dag was too deep!")
}
err := verifyTDagRec(childpb, rdepth, direct, layerRepeat, ds)
if err != nil {
return err
}
}
}
return nil
}