// Creates an in memory DAG from data in the given reader
func NewDagFromReaderWithSplitter(r io.Reader, spl chunk.BlockSplitter) (*dag.Node, error) {
blkChan := spl.Split(r)
first := <-blkChan
root := &dag.Node{}
mbf := new(ft.MultiBlock)
for blk := range blkChan {
log.Debugf("created block, size %d", len(blk))
mbf.AddBlockSize(uint64(len(blk)))
child := &dag.Node{Data: ft.WrapData(blk)}
err := root.AddNodeLink("", child)
if err != nil {
return nil, err
}
}
mbf.Data = first
data, err := mbf.GetBytes()
if err != nil {
return nil, err
}
root.Data = data
return root, nil
}
// splitBytes uses a splitterFunc to turn a large array of bytes
// into many smaller arrays of bytes
func splitBytes(b []byte, spl chunk.BlockSplitter) [][]byte {
out := spl.Split(bytes.NewReader(b))
var arr [][]byte
for blk := range out {
arr = append(arr, blk)
}
return arr
}
// Builds a DAG from the data in the given reader, writing created blocks to disk
// as they are created
func BuildDagFromReader(r io.Reader, ds dag.DAGService, mp pin.ManualPinner, spl chunk.BlockSplitter) (*dag.Node, error) {
blkChan := spl.Split(r)
// grab first block, it will go in the index MultiBlock (faster io)
first := <-blkChan
root := &dag.Node{}
mbf := new(ft.MultiBlock)
for blk := range blkChan {
// Store the block size in the root node
mbf.AddBlockSize(uint64(len(blk)))
node := &dag.Node{Data: ft.WrapData(blk)}
nk, err := ds.Add(node)
if err != nil {
return nil, err
}
if mp != nil {
mp.PinWithMode(nk, pin.Indirect)
}
// Add a link to this node without storing a reference to the memory
err = root.AddNodeLinkClean("", node)
if err != nil {
return nil, err
}
}
// Generate the root node data
mbf.Data = first
data, err := mbf.GetBytes()
if err != nil {
return nil, err
}
root.Data = data
// Add root node to the dagservice
rootk, err := ds.Add(root)
if err != nil {
return nil, err
}
if mp != nil {
mp.PinWithMode(rootk, pin.Recursive)
}
return root, nil
}