func (suite *GraphTestSuite) TestBlockWithOffset_findsCorrectBlock(t *C) {
child, _ := makeNode("child", suite.ng.RootNode.Id, time.Now(), suite.ng)
data := testutils.RandDat(graph.MEGABYTE)
t.Check(child.WriteData(data, 0), IsNil)
data2 := testutils.RandDat(graph.MEGABYTE)
t.Check(child.WriteData(data2, graph.MEGABYTE), IsNil)
foundBlock := child.BlockWithOffset(0)
t.Check(string(foundBlock), Equals, graph.Hash(data))
foundBlock2 := child.BlockWithOffset(graph.MEGABYTE)
t.Check(string(foundBlock2), Equals, graph.Hash(data2))
}
// PUT v1/node/{nodeId}/{offset}
func (restApi OlympusApi) WriteBlock(writer http.ResponseWriter, req *http.Request) {
node := restApi.graph.NodeWithId(paramFromRequest("nodeId", req))
if !node.Exists() {
writeNodeNotFoundError(node.Id, writer)
return
}
defer req.Body.Close()
var data []byte
var err error
if data, err = ioutil.ReadAll(req.Body); err != nil {
http.Error(writer, err.Error(), http.StatusBadRequest)
return
}
headerHash := req.Header.Get("Content-Hash")
dataHash := graph.Hash(data)
if dataHash != headerHash {
http.Error(writer, "Hash in header does not match data's hash", http.StatusBadRequest)
return
}
offsetString := paramFromRequest("offset", req)
if offset, err := strconv.ParseInt(offsetString, 10, 64); err != nil {
http.Error(writer, fmt.Sprintf("Invalid offset parameter: %s", offsetString), http.StatusBadRequest)
} else if err := node.WriteData(data, offset); err != nil {
http.Error(writer, err.Error(), http.StatusBadRequest)
} else {
writer.WriteHeader(http.StatusCreated)
}
}
func (suite *GraphTestSuite) TestBlocks_returnsCorrectBlocks(t *C) {
child, err := makeNode("child", suite.ng.RootNode.Id, time.Now(), suite.ng)
t.Check(err, IsNil)
block1 := testutils.RandDat(graph.MEGABYTE)
block2 := testutils.RandDat(graph.MEGABYTE)
t.Check(child.WriteData(block1, 0), IsNil)
t.Check(child.WriteData(block2, graph.MEGABYTE), IsNil)
blocks := child.Blocks()
t.Assert(blocks, HasLen, 2)
t.Check(blocks[0].Offset, Equals, int64(0))
t.Check(blocks[1].Offset, Equals, int64(graph.MEGABYTE))
t.Check(blocks[0].Hash, Equals, graph.Hash(block1))
t.Check(blocks[1].Hash, Equals, graph.Hash(block2))
}
func (suite *ApiClientTestSuite) TestApiClient_WriteBlock_writesBlock(t *C) {
node, err := suite.ng.NewNodeWithNodeInfo(graph.NodeInfo{
ParentId: graph.RootNodeId,
Name: "thing.txt",
Mode: 0755,
})
t.Check(err, IsNil)
dat := testutils.RandDat(graph.MEGABYTE)
buf := bytes.NewBuffer(dat)
err = suite.client.WriteBlock(node.Id, 0, graph.Hash(dat), buf)
t.Check(err, IsNil)
blocks := node.Blocks()
t.Assert(blocks, HasLen, 1)
t.Check(blocks[0].Hash, Equals, graph.Hash(dat))
t.Check(blocks[0].Offset, Equals, int64(0))
}
func (suite *GraphTestSuite) TestWriteData_writesDataToCorrectBlock(t *C) {
child, _ := makeNode("child", suite.ng.RootNode.Id, time.Now(), suite.ng)
dat := testutils.RandDat(1024)
fingerprint := graph.Hash(dat)
t.Check(child.WriteData(dat, 0), IsNil)
blocks := child.Blocks()
t.Check(blocks, HasLen, 1)
if len(blocks) > 0 {
t.Check(blocks[0].Hash, Matches, fingerprint)
}
}
func (suite *GraphTestSuite) TestWriteData_removesExistingFingerprintForOffset(t *C) {
child, _ := makeNode("child", suite.ng.RootNode.Id, time.Now(), suite.ng)
dat := testutils.RandDat(1024)
t.Check(child.WriteData(dat, 0), IsNil)
dat = testutils.RandDat(1024)
fingerprint := graph.Hash(dat)
t.Check(child.WriteData(dat, 0), IsNil)
it := cayley.StartPath(suite.ng, child.Id).Out("offset-0").BuildIterator()
t.Check(cayley.RawNext(it), Equals, true)
t.Check(suite.ng.NameOf(it.Result()), Equals, fingerprint)
}
func (suite *ApiClientTestSuite) TestApiClient_ListBlocks_listsBlocks(t *C) {
node, err := suite.ng.NewNodeWithNodeInfo(graph.NodeInfo{
ParentId: graph.RootNodeId,
Name: "thing.txt",
Mode: 0755,
})
t.Check(err, IsNil)
dat1 := testutils.RandDat(graph.MEGABYTE)
hash1 := graph.Hash(dat1)
dat2 := testutils.RandDat(graph.MEGABYTE)
hash2 := graph.Hash(dat2)
t.Check(node.WriteData(dat1, 0), IsNil)
t.Check(node.WriteData(dat2, graph.MEGABYTE), IsNil)
blocks, err := suite.client.ListBlocks(node.Id)
t.Check(err, IsNil)
t.Check(blocks, HasLen, 2)
t.Check(blocks[0].Hash, Equals, hash1)
t.Check(blocks[0].Offset, Equals, int64(0))
t.Check(blocks[1].Hash, Equals, hash2)
t.Check(blocks[1].Offset, Equals, int64(graph.MEGABYTE))
}
func (manager *Manager) UploadFile(parentId, localPath string, callback ProgressCallback) (*graph.Node, error) {
errorFmt := func(err error) error {
return fmt.Errorf("Error uploading file: %s", err.Error())
}
if fi, err := os.Stat(localPath); err != nil {
return nil, errorFmt(err)
} else if fi.IsDir() {
return nil, errors.New("Cannot upload a directory")
} else {
nodeInfo := graph.NodeInfo{
Name: filepath.Base(fi.Name()),
Size: fi.Size(),
Mode: 0700,
MTime: time.Now(),
ParentId: parentId,
Type: util.MimeType(fi.Name()),
}
if newNode, err := manager.api.CreateNode(nodeInfo); err != nil {
return nil, errorFmt(err)
} else if localFile, err := os.Open(localPath); err != nil {
return nil, errorFmt(err)
} else {
defer localFile.Close()
errChan := make(chan error)
uploadChan := make(chan heap, 5)
defer close(uploadChan)
var wg sync.WaitGroup
numBlocks := int(fi.Size() / graph.BLOCK_SIZE)
if fi.Size()%graph.BLOCK_SIZE > 0 {
numBlocks++
}
wg.Add(numBlocks)
var uploadedBytes int64
for i := 0; i < 5; i++ { // TODO: min(numblocks, 5)
go func() {
for h := range uploadChan {
payloadSize := int64(len(h.data))
rd := bytes.NewBuffer(h.data)
hash := graph.Hash(h.data)
if err = manager.api.WriteBlock(newNode.Id, h.offset, hash, rd); err != nil {
errChan <- err
}
uploadedBytes += payloadSize
callback(fi.Size(), uploadedBytes)
wg.Done()
}
}()
}
errChecker := func() error {
select {
case err := <-errChan:
return errorFmt(err)
default:
return nil
}
}
var offset int64
for offset = 0; offset < fi.Size(); offset += graph.BLOCK_SIZE {
buf := make([]byte, min(fi.Size()-offset, graph.BLOCK_SIZE))
if _, err = localFile.ReadAt(buf, offset); err != nil {
return nil, errorFmt(err)
}
uploadChan <- heap{offset, buf}
if err := errChecker(); err != nil {
return nil, errorFmt(err)
}
}
if err := errChecker(); err != nil {
return nil, errorFmt(err)
}
wg.Wait()
return manager.graph.NodeWithNodeInfo(newNode), err
}
}
}
func fileData(size int) (string, io.Reader) {
dat := testutils.RandDat(size)
return graph.Hash(dat), bytes.NewBuffer(dat)
}