Example #1
0
func (suite *GraphTestSuite) TestBlockWithOffset_findsCorrectBlock(t *C) {
	child, _ := makeNode("child", suite.ng.RootNode.Id, time.Now(), suite.ng)
	data := testutils.RandDat(graph.MEGABYTE)
	t.Check(child.WriteData(data, 0), IsNil)

	data2 := testutils.RandDat(graph.MEGABYTE)
	t.Check(child.WriteData(data2, graph.MEGABYTE), IsNil)

	foundBlock := child.BlockWithOffset(0)
	t.Check(string(foundBlock), Equals, graph.Hash(data))

	foundBlock2 := child.BlockWithOffset(graph.MEGABYTE)
	t.Check(string(foundBlock2), Equals, graph.Hash(data2))
}
Example #2
0
// PUT v1/node/{nodeId}/{offset}
func (restApi OlympusApi) WriteBlock(writer http.ResponseWriter, req *http.Request) {
	node := restApi.graph.NodeWithId(paramFromRequest("nodeId", req))
	if !node.Exists() {
		writeNodeNotFoundError(node.Id, writer)
		return
	}

	defer req.Body.Close()
	var data []byte
	var err error
	if data, err = ioutil.ReadAll(req.Body); err != nil {
		http.Error(writer, err.Error(), http.StatusBadRequest)
		return
	}

	headerHash := req.Header.Get("Content-Hash")
	dataHash := graph.Hash(data)

	if dataHash != headerHash {
		http.Error(writer, "Hash in header does not match data's hash", http.StatusBadRequest)
		return
	}

	offsetString := paramFromRequest("offset", req)
	if offset, err := strconv.ParseInt(offsetString, 10, 64); err != nil {
		http.Error(writer, fmt.Sprintf("Invalid offset parameter: %s", offsetString), http.StatusBadRequest)
	} else if err := node.WriteData(data, offset); err != nil {
		http.Error(writer, err.Error(), http.StatusBadRequest)
	} else {
		writer.WriteHeader(http.StatusCreated)
	}
}
Example #3
0
func (suite *GraphTestSuite) TestBlocks_returnsCorrectBlocks(t *C) {
	child, err := makeNode("child", suite.ng.RootNode.Id, time.Now(), suite.ng)
	t.Check(err, IsNil)

	block1 := testutils.RandDat(graph.MEGABYTE)
	block2 := testutils.RandDat(graph.MEGABYTE)

	t.Check(child.WriteData(block1, 0), IsNil)
	t.Check(child.WriteData(block2, graph.MEGABYTE), IsNil)

	blocks := child.Blocks()
	t.Assert(blocks, HasLen, 2)

	t.Check(blocks[0].Offset, Equals, int64(0))
	t.Check(blocks[1].Offset, Equals, int64(graph.MEGABYTE))

	t.Check(blocks[0].Hash, Equals, graph.Hash(block1))
	t.Check(blocks[1].Hash, Equals, graph.Hash(block2))
}
Example #4
0
func (suite *ApiClientTestSuite) TestApiClient_WriteBlock_writesBlock(t *C) {
	node, err := suite.ng.NewNodeWithNodeInfo(graph.NodeInfo{
		ParentId: graph.RootNodeId,
		Name:     "thing.txt",
		Mode:     0755,
	})
	t.Check(err, IsNil)

	dat := testutils.RandDat(graph.MEGABYTE)
	buf := bytes.NewBuffer(dat)

	err = suite.client.WriteBlock(node.Id, 0, graph.Hash(dat), buf)
	t.Check(err, IsNil)

	blocks := node.Blocks()
	t.Assert(blocks, HasLen, 1)
	t.Check(blocks[0].Hash, Equals, graph.Hash(dat))
	t.Check(blocks[0].Offset, Equals, int64(0))
}
Example #5
0
func (suite *GraphTestSuite) TestWriteData_writesDataToCorrectBlock(t *C) {
	child, _ := makeNode("child", suite.ng.RootNode.Id, time.Now(), suite.ng)
	dat := testutils.RandDat(1024)
	fingerprint := graph.Hash(dat)

	t.Check(child.WriteData(dat, 0), IsNil)

	blocks := child.Blocks()
	t.Check(blocks, HasLen, 1)
	if len(blocks) > 0 {
		t.Check(blocks[0].Hash, Matches, fingerprint)
	}
}
Example #6
0
func (suite *GraphTestSuite) TestWriteData_removesExistingFingerprintForOffset(t *C) {
	child, _ := makeNode("child", suite.ng.RootNode.Id, time.Now(), suite.ng)
	dat := testutils.RandDat(1024)

	t.Check(child.WriteData(dat, 0), IsNil)

	dat = testutils.RandDat(1024)
	fingerprint := graph.Hash(dat)
	t.Check(child.WriteData(dat, 0), IsNil)

	it := cayley.StartPath(suite.ng, child.Id).Out("offset-0").BuildIterator()
	t.Check(cayley.RawNext(it), Equals, true)
	t.Check(suite.ng.NameOf(it.Result()), Equals, fingerprint)
}
Example #7
0
func (suite *ApiClientTestSuite) TestApiClient_ListBlocks_listsBlocks(t *C) {
	node, err := suite.ng.NewNodeWithNodeInfo(graph.NodeInfo{
		ParentId: graph.RootNodeId,
		Name:     "thing.txt",
		Mode:     0755,
	})
	t.Check(err, IsNil)

	dat1 := testutils.RandDat(graph.MEGABYTE)
	hash1 := graph.Hash(dat1)
	dat2 := testutils.RandDat(graph.MEGABYTE)
	hash2 := graph.Hash(dat2)

	t.Check(node.WriteData(dat1, 0), IsNil)
	t.Check(node.WriteData(dat2, graph.MEGABYTE), IsNil)

	blocks, err := suite.client.ListBlocks(node.Id)
	t.Check(err, IsNil)
	t.Check(blocks, HasLen, 2)
	t.Check(blocks[0].Hash, Equals, hash1)
	t.Check(blocks[0].Offset, Equals, int64(0))
	t.Check(blocks[1].Hash, Equals, hash2)
	t.Check(blocks[1].Offset, Equals, int64(graph.MEGABYTE))
}
Example #8
0
func (manager *Manager) UploadFile(parentId, localPath string, callback ProgressCallback) (*graph.Node, error) {
	errorFmt := func(err error) error {
		return fmt.Errorf("Error uploading file: %s", err.Error())
	}

	if fi, err := os.Stat(localPath); err != nil {
		return nil, errorFmt(err)
	} else if fi.IsDir() {
		return nil, errors.New("Cannot upload a directory")
	} else {
		nodeInfo := graph.NodeInfo{
			Name:     filepath.Base(fi.Name()),
			Size:     fi.Size(),
			Mode:     0700,
			MTime:    time.Now(),
			ParentId: parentId,
			Type:     util.MimeType(fi.Name()),
		}
		if newNode, err := manager.api.CreateNode(nodeInfo); err != nil {
			return nil, errorFmt(err)
		} else if localFile, err := os.Open(localPath); err != nil {
			return nil, errorFmt(err)
		} else {
			defer localFile.Close()

			errChan := make(chan error)
			uploadChan := make(chan heap, 5)
			defer close(uploadChan)

			var wg sync.WaitGroup
			numBlocks := int(fi.Size() / graph.BLOCK_SIZE)
			if fi.Size()%graph.BLOCK_SIZE > 0 {
				numBlocks++
			}

			wg.Add(numBlocks)
			var uploadedBytes int64
			for i := 0; i < 5; i++ { // TODO: min(numblocks, 5)
				go func() {
					for h := range uploadChan {
						payloadSize := int64(len(h.data))

						rd := bytes.NewBuffer(h.data)
						hash := graph.Hash(h.data)
						if err = manager.api.WriteBlock(newNode.Id, h.offset, hash, rd); err != nil {
							errChan <- err
						}
						uploadedBytes += payloadSize
						callback(fi.Size(), uploadedBytes)
						wg.Done()
					}
				}()
			}

			errChecker := func() error {
				select {
				case err := <-errChan:
					return errorFmt(err)
				default:
					return nil
				}
			}

			var offset int64
			for offset = 0; offset < fi.Size(); offset += graph.BLOCK_SIZE {
				buf := make([]byte, min(fi.Size()-offset, graph.BLOCK_SIZE))
				if _, err = localFile.ReadAt(buf, offset); err != nil {
					return nil, errorFmt(err)
				}
				uploadChan <- heap{offset, buf}

				if err := errChecker(); err != nil {
					return nil, errorFmt(err)
				}
			}

			if err := errChecker(); err != nil {
				return nil, errorFmt(err)
			}
			wg.Wait()

			return manager.graph.NodeWithNodeInfo(newNode), err
		}
	}
}
Example #9
0
func fileData(size int) (string, io.Reader) {
	dat := testutils.RandDat(size)
	return graph.Hash(dat), bytes.NewBuffer(dat)
}