// NewDagReader creates a new reader object that reads the data represented by the given
// node, using the passed in DAGService for data retreival
func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*DagReader, error) {
pb := new(ftpb.Data)
if err := proto.Unmarshal(n.Data, pb); err != nil {
return nil, err
}
switch pb.GetType() {
case ftpb.Data_Directory:
// Dont allow reading directories
return nil, ErrIsDir
case ftpb.Data_Raw:
fallthrough
case ftpb.Data_File:
return NewDataFileReader(ctx, n, pb, serv), nil
case ftpb.Data_Metadata:
if len(n.Links) == 0 {
return nil, errors.New("incorrectly formatted metadata object")
}
child, err := n.Links[0].GetNode(ctx, serv)
if err != nil {
return nil, err
}
return NewDagReader(ctx, child, serv)
case ftpb.Data_Symlink:
return nil, ErrCantReadSymlinks
default:
return nil, ft.ErrUnrecognizedType
}
}
// precalcNextBuf follows the next link in line and loads it from the DAGService,
// setting the next buffer to read from
func (dr *DagReader) precalcNextBuf(ctx context.Context) error {
dr.buf.Close() // Just to make sure
if dr.linkPosition >= len(dr.promises) {
return io.EOF
}
nxt, err := dr.promises[dr.linkPosition].Get(ctx)
if err != nil {
return err
}
dr.linkPosition++
pb := new(ftpb.Data)
err = proto.Unmarshal(nxt.Data, pb)
if err != nil {
return fmt.Errorf("incorrectly formatted protobuf: %s", err)
}
switch pb.GetType() {
case ftpb.Data_Directory:
// A directory should not exist within a file
return ft.ErrInvalidDirLocation
case ftpb.Data_File:
dr.buf = newDataFileReader(dr.ctx, nxt, pb, dr.serv)
return nil
case ftpb.Data_Raw:
dr.buf = NewRSNCFromBytes(pb.GetData())
return nil
case ftpb.Data_Metadata:
return errors.New("Shouldnt have had metadata object inside file")
default:
return ft.ErrUnrecognizedType
}
}
func UnwrapData(data []byte) ([]byte, error) {
pbdata := new(pb.Data)
err := proto.Unmarshal(data, pbdata)
if err != nil {
return nil, err
}
return pbdata.GetData(), nil
}
func (n *FSNode) GetBytes() ([]byte, error) {
pbn := new(pb.Data)
pbn.Type = &n.Type
pbn.Filesize = proto.Uint64(uint64(len(n.Data)) + n.subtotal)
pbn.Blocksizes = n.blocksizes
pbn.Data = n.Data
return proto.Marshal(pbn)
}
func (w *Writer) writeFile(nd *mdag.Node, pb *upb.Data, fpath string) error {
if err := writeFileHeader(w.TarW, fpath, pb.GetFilesize()); err != nil {
return err
}
dagr := uio.NewDataFileReader(w.ctx, nd, pb, w.Dag)
_, err := dagr.WriteTo(w.TarW)
return err
}
// Returns Bytes that represent a Directory
func FolderPBData() []byte {
pbfile := new(pb.Data)
typ := pb.Data_Directory
pbfile.Type = &typ
data, err := proto.Marshal(pbfile)
if err != nil {
//this really shouldnt happen, i promise
panic(err)
}
return data
}
func BytesForMetadata(m *Metadata) ([]byte, error) {
pbd := new(pb.Data)
pbd.Filesize = proto.Uint64(m.Size)
typ := pb.Data_Metadata
pbd.Type = &typ
mdd, err := m.Bytes()
if err != nil {
return nil, err
}
pbd.Data = mdd
return proto.Marshal(pbd)
}
func SymlinkData(path string) ([]byte, error) {
pbdata := new(pb.Data)
typ := pb.Data_Symlink
pbdata.Data = []byte(path)
pbdata.Type = &typ
out, err := proto.Marshal(pbdata)
if err != nil {
return nil, err
}
return out, nil
}
func NewPBFileReader(ctx context.Context, n *mdag.ProtoNode, pb *ftpb.Data, serv mdag.DAGService) *pbDagReader {
fctx, cancel := context.WithCancel(ctx)
promises := mdag.GetDAG(fctx, serv, n)
return &pbDagReader{
node: n,
serv: serv,
buf: NewBufDagReader(pb.GetData()),
promises: promises,
ctx: fctx,
cancel: cancel,
pbdata: pb,
}
}
func NewDataFileReader(ctx context.Context, n *mdag.Node, pb *ftpb.Data, serv mdag.DAGService) *DagReader {
fctx, cancel := context.WithCancel(ctx)
promises := serv.GetDAG(fctx, n)
return &DagReader{
node: n,
serv: serv,
buf: NewRSNCFromBytes(pb.GetData()),
promises: promises,
ctx: fctx,
cancel: cancel,
pbdata: pb,
}
}
func WrapData(b []byte) []byte {
pbdata := new(pb.Data)
typ := pb.Data_Raw
pbdata.Data = b
pbdata.Type = &typ
out, err := proto.Marshal(pbdata)
if err != nil {
// This shouldnt happen. seriously.
panic(err)
}
return out
}
func FSNodeFromBytes(b []byte) (*FSNode, error) {
pbn := new(pb.Data)
err := proto.Unmarshal(b, pbn)
if err != nil {
return nil, err
}
n := new(FSNode)
n.Data = pbn.Data
n.blocksizes = pbn.Blocksizes
n.subtotal = pbn.GetFilesize() - uint64(len(n.Data))
n.Type = pbn.GetType()
return n, nil
}
func (w *Writer) WriteNode(ctx cxt.Context, nd *mdag.Node, fpath string) error {
pb := new(upb.Data)
if err := proto.Unmarshal(nd.Data, pb); err != nil {
return err
}
switch pb.GetType() {
case upb.Data_Directory:
return w.WriteDir(ctx, nd, fpath)
case upb.Data_File:
return w.writeFile(ctx, nd, pb, fpath)
default:
return fmt.Errorf("unixfs type not supported: %s", pb.GetType())
}
}
func (w *Writer) writeFile(ctx cxt.Context, nd *mdag.Node, pb *upb.Data, fpath string) error {
if err := writeFileHeader(w.TarW, fpath, pb.GetFilesize()); err != nil {
return err
}
dagr, err := uio.NewDagReader(ctx, nd, w.Dag)
if err != nil {
return err
}
_, err = io.Copy(w.TarW, dagr)
if err != nil && err != io.EOF {
return err
}
return nil
}
func FilePBData(data []byte, totalsize uint64) []byte {
pbfile := new(pb.Data)
typ := pb.Data_File
pbfile.Type = &typ
pbfile.Data = data
pbfile.Filesize = proto.Uint64(totalsize)
data, err := proto.Marshal(pbfile)
if err != nil {
// This really shouldnt happen, i promise
// The only failure case for marshal is if required fields
// are not filled out, and they all are. If the proto object
// gets changed and nobody updates this function, the code
// should panic due to programmer error
panic(err)
}
return data
}
func MetadataFromBytes(b []byte) (*Metadata, error) {
pbd := new(pb.Data)
err := proto.Unmarshal(b, pbd)
if err != nil {
return nil, err
}
if pbd.GetType() != pb.Data_Metadata {
return nil, errors.New("incorrect node type")
}
pbm := new(pb.Metadata)
err = proto.Unmarshal(pbd.Data, pbm)
if err != nil {
return nil, err
}
md := new(Metadata)
md.MimeType = pbm.GetMimeType()
return md, nil
}
// NewDagReader creates a new reader object that reads the data represented by
// the given node, using the passed in DAGService for data retreival
func NewDagReader(ctx context.Context, n node.Node, serv mdag.DAGService) (*DagReader, error) {
switch n := n.(type) {
case *mdag.RawNode:
return &DagReader{
buf: NewRSNCFromBytes(n.RawData()),
}, nil
case *mdag.ProtoNode:
pb := new(ftpb.Data)
if err := proto.Unmarshal(n.Data(), pb); err != nil {
return nil, err
}
switch pb.GetType() {
case ftpb.Data_Directory:
// Dont allow reading directories
return nil, ErrIsDir
case ftpb.Data_File, ftpb.Data_Raw:
return NewDataFileReader(ctx, n, pb, serv), nil
case ftpb.Data_Metadata:
if len(n.Links()) == 0 {
return nil, errors.New("incorrectly formatted metadata object")
}
child, err := n.Links()[0].GetNode(ctx, serv)
if err != nil {
return nil, err
}
childpb, ok := child.(*mdag.ProtoNode)
if !ok {
return nil, mdag.ErrNotProtobuf
}
return NewDagReader(ctx, childpb, serv)
case ftpb.Data_Symlink:
return nil, ErrCantReadSymlinks
default:
return nil, ft.ErrUnrecognizedType
}
default:
return nil, fmt.Errorf("unrecognized node type")
}
}
func (w *Writer) WriteNode(nd *mdag.ProtoNode, fpath string) error {
pb := new(upb.Data)
if err := proto.Unmarshal(nd.Data(), pb); err != nil {
return err
}
switch pb.GetType() {
case upb.Data_Metadata:
fallthrough
case upb.Data_Directory:
return w.writeDir(nd, fpath)
case upb.Data_Raw:
fallthrough
case upb.Data_File:
return w.writeFile(nd, pb, fpath)
case upb.Data_Symlink:
return writeSymlinkHeader(w.TarW, string(pb.GetData()), fpath)
default:
return ft.ErrUnrecognizedType
}
}
func DataSize(data []byte) (uint64, error) {
pbdata := new(pb.Data)
err := proto.Unmarshal(data, pbdata)
if err != nil {
return 0, err
}
switch pbdata.GetType() {
case pb.Data_Directory:
return 0, errors.New("Cant get data size of directory!")
case pb.Data_File:
return pbdata.GetFilesize(), nil
case pb.Data_Raw:
return uint64(len(pbdata.GetData())), nil
default:
return 0, errors.New("Unrecognized node data type!")
}
}
func (r *Reader) writeToBuf(ctx cxt.Context, dagnode *mdag.Node, path string, depth int) {
pb := new(upb.Data)
err := proto.Unmarshal(dagnode.Data, pb)
if err != nil {
r.emitError(err)
return
}
if depth == 0 {
defer r.close()
}
if pb.GetType() == upb.Data_Directory {
err = r.writer.WriteHeader(&tar.Header{
Name: path,
Typeflag: tar.TypeDir,
Mode: 0777,
ModTime: time.Now(),
// TODO: set mode, dates, etc. when added to unixFS
})
if err != nil {
r.emitError(err)
return
}
r.flush()
for i, ng := range r.dag.GetDAG(ctx, dagnode) {
childNode, err := ng.Get(ctx)
if err != nil {
r.emitError(err)
return
}
r.writeToBuf(ctx, childNode, gopath.Join(path, dagnode.Links[i].Name), depth+1)
}
return
}
err = r.writer.WriteHeader(&tar.Header{
Name: path,
Size: int64(pb.GetFilesize()),
Typeflag: tar.TypeReg,
Mode: 0644,
ModTime: time.Now(),
// TODO: set mode, dates, etc. when added to unixFS
})
if err != nil {
r.emitError(err)
return
}
r.flush()
reader, err := uio.NewDagReader(ctx, dagnode, r.dag)
if err != nil {
r.emitError(err)
return
}
err = r.syncCopy(reader)
if err != nil {
r.emitError(err)
return
}
}