Ejemplo n.º 1
0
// Creates an in memory DAG from data in the given reader
func NewDagFromReaderWithSplitter(r io.Reader, spl chunk.BlockSplitter) (*dag.Node, error) {
	blkChan := spl.Split(r)
	first := <-blkChan
	root := &dag.Node{}

	mbf := new(ft.MultiBlock)
	for blk := range blkChan {
		log.Debugf("created block, size %d", len(blk))
		mbf.AddBlockSize(uint64(len(blk)))
		child := &dag.Node{Data: ft.WrapData(blk)}
		err := root.AddNodeLink("", child)
		if err != nil {
			return nil, err
		}
	}

	mbf.Data = first
	data, err := mbf.GetBytes()
	if err != nil {
		return nil, err
	}

	root.Data = data
	return root, nil
}
Ejemplo n.º 2
0
// splitBytes uses a splitterFunc to turn a large array of bytes
// into many smaller arrays of bytes
func splitBytes(b []byte, spl chunk.BlockSplitter) [][]byte {
	out := spl.Split(bytes.NewReader(b))
	var arr [][]byte
	for blk := range out {
		arr = append(arr, blk)
	}
	return arr
}
Ejemplo n.º 3
0
// Builds a DAG from the data in the given reader, writing created blocks to disk
// as they are created
func BuildDagFromReader(r io.Reader, ds dag.DAGService, mp pin.ManualPinner, spl chunk.BlockSplitter) (*dag.Node, error) {
	blkChan := spl.Split(r)

	// grab first block, it will go in the index MultiBlock (faster io)
	first := <-blkChan
	root := &dag.Node{}

	mbf := new(ft.MultiBlock)
	for blk := range blkChan {
		// Store the block size in the root node
		mbf.AddBlockSize(uint64(len(blk)))
		node := &dag.Node{Data: ft.WrapData(blk)}
		nk, err := ds.Add(node)
		if err != nil {
			return nil, err
		}

		if mp != nil {
			mp.PinWithMode(nk, pin.Indirect)
		}

		// Add a link to this node without storing a reference to the memory
		err = root.AddNodeLinkClean("", node)
		if err != nil {
			return nil, err
		}
	}

	// Generate the root node data
	mbf.Data = first
	data, err := mbf.GetBytes()
	if err != nil {
		return nil, err
	}
	root.Data = data

	// Add root node to the dagservice
	rootk, err := ds.Add(root)
	if err != nil {
		return nil, err
	}
	if mp != nil {
		mp.PinWithMode(rootk, pin.Recursive)
	}

	return root, nil
}