func GetFile(apiClient pfs.APIClient, repoName string, commitID string, path string, offset int64, size int64, shard *pfs.Shard, writer io.Writer) error {
if size == 0 {
size = math.MaxInt64
}
apiGetFileClient, err := apiClient.GetFile(
context.Background(),
&pfs.GetFileRequest{
File: &pfs.File{
Commit: &pfs.Commit{
Repo: &pfs.Repo{
Name: repoName,
},
Id: commitID,
},
Path: path,
},
Shard: shard,
OffsetBytes: offset,
SizeBytes: size,
},
)
if err != nil {
return err
}
if err := protostream.WriteFromStreamingBytesClient(apiGetFileClient, writer); err != nil {
return err
}
return nil
}
func MakeDirectory(apiClient pfs.APIClient, repoName string, commitID string, path string) (retErr error) {
putFileClient, err := apiClient.PutFile(context.Background())
if err != nil {
return err
}
defer func() {
if _, err := putFileClient.CloseAndRecv(); err != nil && retErr == nil {
retErr = err
}
}()
return putFileClient.Send(
&pfs.PutFileRequest{
File: &pfs.File{
Commit: &pfs.Commit{
Repo: &pfs.Repo{
Name: repoName,
},
Id: commitID,
},
Path: path,
},
FileType: pfs.FileType_FILE_TYPE_DIR,
},
)
}
func FinishCommit(apiClient pfs.APIClient, repoName string, commitID string) error {
_, err := apiClient.FinishCommit(
context.Background(),
&pfs.FinishCommitRequest{
Commit: NewCommit(repoName, commitID),
},
)
return err
}
func DeleteFile(apiClient pfs.APIClient, repoName string, commitID string, path string) error {
_, err := apiClient.DeleteFile(
context.Background(),
&pfs.DeleteFileRequest{
File: NewFile(repoName, commitID, path),
},
)
return err
}
func DeleteRepo(apiClient pfs.APIClient, repoName string) error {
_, err := apiClient.DeleteRepo(
context.Background(),
&pfs.DeleteRepoRequest{
Repo: NewRepo(repoName),
},
)
return err
}
func ListServer(apiClient pfs.APIClient) ([]*pfs.ServerInfo, error) {
serverInfos, err := apiClient.ListServer(
context.Background(),
&pfs.ListServerRequest{},
)
if err != nil {
return nil, err
}
return serverInfos.ServerInfo, nil
}
func ListRepo(apiClient pfs.APIClient) ([]*pfs.RepoInfo, error) {
repoInfos, err := apiClient.ListRepo(
context.Background(),
&pfs.ListRepoRequest{},
)
if err != nil {
return nil, err
}
return repoInfos.RepoInfo, nil
}
func InspectServer(apiClient pfs.APIClient, serverID string) (*pfs.ServerInfo, error) {
return apiClient.InspectServer(
context.Background(),
&pfs.InspectServerRequest{
Server: &pfs.Server{
Id: serverID,
},
},
)
}
func CreateRepo(apiClient pfs.APIClient, repoName string) error {
_, err := apiClient.CreateRepo(
context.Background(),
&pfs.CreateRepoRequest{
Repo: &pfs.Repo{
Name: repoName,
},
},
)
return err
}
func ListBlock(apiClient pfs.APIClient, shard *pfs.Shard) ([]*pfs.BlockInfo, error) {
blockInfos, err := apiClient.ListBlock(
context.Background(),
&pfs.ListBlockRequest{
Shard: shard,
},
)
if err != nil {
return nil, err
}
return blockInfos.BlockInfo, nil
}
func InspectRepo(apiClient pfs.APIClient, repoName string) (*pfs.RepoInfo, error) {
repoInfo, err := apiClient.InspectRepo(
context.Background(),
&pfs.InspectRepoRequest{
Repo: NewRepo(repoName),
},
)
if err != nil {
return nil, err
}
return repoInfo, nil
}
func InspectCommit(apiClient pfs.APIClient, repoName string, commitID string) (*pfs.CommitInfo, error) {
commitInfo, err := apiClient.InspectCommit(
context.Background(),
&pfs.InspectCommitRequest{
Commit: NewCommit(repoName, commitID),
},
)
if err != nil {
return nil, err
}
return commitInfo, nil
}
func StartCommit(apiClient pfs.APIClient, repoName string, parentCommit string) (*pfs.Commit, error) {
commit, err := apiClient.StartCommit(
context.Background(),
&pfs.StartCommitRequest{
Parent: NewCommit(repoName, parentCommit),
},
)
if err != nil {
return nil, err
}
return commit, nil
}
func setupPFSRepo(t *testing.T, pfsAPIClient pfs.APIClient, repoName string) *pfs.Repo {
repo := &pfs.Repo{
Name: repoName,
}
_, err := pfsAPIClient.CreateRepo(
context.Background(),
&pfs.CreateRepoRequest{
Repo: repo,
},
)
require.NoError(t, err)
return repo
}
func DeleteCommit(apiClient pfs.APIClient, repoName string, commitID string) error {
_, err := apiClient.DeleteCommit(
context.Background(),
&pfs.DeleteCommitRequest{
Commit: &pfs.Commit{
Repo: &pfs.Repo{
Name: repoName,
},
Id: commitID,
},
},
)
return err
}
func ListCommit(apiClient pfs.APIClient, repoName string) ([]*pfs.CommitInfo, error) {
commitInfos, err := apiClient.ListCommit(
context.Background(),
&pfs.ListCommitRequest{
Repo: &pfs.Repo{
Name: repoName,
},
},
)
if err != nil {
return nil, err
}
return commitInfos.CommitInfo, nil
}
func ListFile(apiClient pfs.APIClient, repoName string, commitID string, path string, fromCommitID string, shard *pfs.Shard) ([]*pfs.FileInfo, error) {
fileInfos, err := apiClient.ListFile(
context.Background(),
&pfs.ListFileRequest{
File: NewFile(repoName, commitID, path),
Shard: shard,
FromCommit: newFromCommit(repoName, fromCommitID),
},
)
if err != nil {
return nil, err
}
return fileInfos.FileInfo, nil
}
func InspectBlock(apiClient pfs.APIClient, hash string) (*pfs.BlockInfo, error) {
blockInfo, err := apiClient.InspectBlock(
context.Background(),
&pfs.InspectBlockRequest{
Block: &pfs.Block{
Hash: hash,
},
},
)
if err != nil {
return nil, err
}
return blockInfo, nil
}
func ListBlock(apiClient pfs.APIClient, shard uint64, modulus uint64) ([]*pfs.BlockInfo, error) {
blockInfos, err := apiClient.ListBlock(
context.Background(),
&pfs.ListBlockRequest{
Shard: &pfs.Shard{
Number: shard,
Modulo: modulus,
},
},
)
if err != nil {
return nil, err
}
return blockInfos.BlockInfo, nil
}
func ListCommit(apiClient pfs.APIClient, repoNames []string) ([]*pfs.CommitInfo, error) {
var repos []*pfs.Repo
for _, repoName := range repoNames {
repos = append(repos, &pfs.Repo{Name: repoName})
}
commitInfos, err := apiClient.ListCommit(
context.Background(),
&pfs.ListCommitRequest{
Repo: repos,
},
)
if err != nil {
return nil, err
}
return commitInfos.CommitInfo, nil
}
func DeleteFile(apiClient pfs.APIClient, repoName string, commitID string, path string) error {
_, err := apiClient.DeleteFile(
context.Background(),
&pfs.DeleteFileRequest{
File: &pfs.File{
Commit: &pfs.Commit{
Repo: &pfs.Repo{
Name: repoName,
},
Id: commitID,
},
Path: path,
},
},
)
return err
}
func GetBlock(apiClient pfs.APIClient, hash string, writer io.Writer) error {
apiGetBlockClient, err := apiClient.GetBlock(
context.Background(),
&pfs.GetBlockRequest{
Block: &pfs.Block{
Hash: hash,
},
},
)
if err != nil {
return err
}
if err := protostream.WriteFromStreamingBytesClient(apiGetBlockClient, writer); err != nil {
return err
}
return nil
}
func PutFile(apiClient pfs.APIClient, repoName string, commitID string, path string, offset int64, reader io.Reader) (_ int, retErr error) {
putFileClient, err := apiClient.PutFile(context.Background())
if err != nil {
return 0, err
}
defer func() {
if _, err := putFileClient.CloseAndRecv(); err != nil && retErr == nil {
retErr = err
}
}()
request := pfs.PutFileRequest{
File: &pfs.File{
Commit: &pfs.Commit{
Repo: &pfs.Repo{
Name: repoName,
},
Id: commitID,
},
Path: path,
},
FileType: pfs.FileType_FILE_TYPE_REGULAR,
OffsetBytes: offset,
}
var size int
eof := false
for !eof {
value := make([]byte, chunkSize)
iSize, err := reader.Read(value)
if err != nil && err != io.EOF {
return 0, err
}
if err == io.EOF {
eof = true
}
request.Value = value[0:iSize]
size += iSize
if err := putFileClient.Send(&request); err != nil {
return 0, err
}
}
if err != nil && err != io.EOF {
return 0, err
}
return size, err
}
func getPFSContent(pfsAPIClient pfs.APIClient, commit *pfs.Commit, filePath string) ([]byte, error) {
apiGetFileClient, err := pfsAPIClient.GetFile(
context.Background(),
&pfs.GetFileRequest{
File: &pfs.File{
Commit: commit,
Path: filePath,
},
},
)
if err != nil {
return nil, err
}
buffer := bytes.NewBuffer(nil)
if err := protostream.WriteFromStreamingBytesClient(apiGetFileClient, buffer); err != nil {
return nil, err
}
return buffer.Bytes(), nil
}
func InspectFile(apiClient pfs.APIClient, repoName string, commitID string, path string) (*pfs.FileInfo, error) {
fileInfo, err := apiClient.InspectFile(
context.Background(),
&pfs.InspectFileRequest{
File: &pfs.File{
Commit: &pfs.Commit{
Repo: &pfs.Repo{
Name: repoName,
},
Id: commitID,
},
Path: path,
},
},
)
if err != nil {
return nil, err
}
return fileInfo, nil
}
func ListFile(apiClient pfs.APIClient, repoName string, commitID string, path string, shard *pfs.Shard) ([]*pfs.FileInfo, error) {
fileInfos, err := apiClient.ListFile(
context.Background(),
&pfs.ListFileRequest{
File: &pfs.File{
Commit: &pfs.Commit{
Repo: &pfs.Repo{
Name: repoName,
},
Id: commitID,
},
Path: path,
},
Shard: shard,
},
)
if err != nil {
return nil, err
}
return fileInfos.FileInfo, nil
}
func PutBlock(apiClient pfs.APIClient, repoName string, commitID string, path string, reader io.Reader) (*pfs.Block, error) {
value, err := ioutil.ReadAll(reader)
if err != nil {
return nil, err
}
return apiClient.PutBlock(
context.Background(),
&pfs.PutBlockRequest{
File: &pfs.File{
Commit: &pfs.Commit{
Repo: &pfs.Repo{
Name: repoName,
},
Id: commitID,
},
Path: path,
},
Value: value,
},
)
}
func GetFile(apiClient pfs.APIClient, repoName string, commitID string, path string, offset int64, size int64, fromCommitID string, shard *pfs.Shard, writer io.Writer) error {
if size == 0 {
size = math.MaxInt64
}
apiGetFileClient, err := apiClient.GetFile(
context.Background(),
&pfs.GetFileRequest{
File: NewFile(repoName, commitID, path),
Shard: shard,
OffsetBytes: offset,
SizeBytes: size,
FromCommit: newFromCommit(repoName, fromCommitID),
},
)
if err != nil {
return err
}
if err := protostream.WriteFromStreamingBytesClient(apiGetFileClient, writer); err != nil {
return err
}
return nil
}
func ListChange(apiClient pfs.APIClient, repoName string, commitID string, path string, shard uint64, modulus uint64) ([]*pfs.Change, error) {
changes, err := apiClient.ListChange(
context.Background(),
&pfs.ListChangeRequest{
File: &pfs.File{
Commit: &pfs.Commit{
Repo: &pfs.Repo{
Name: repoName,
},
Id: commitID,
},
Path: path,
},
Shard: &pfs.Shard{
Number: shard,
Modulo: modulus,
},
},
)
if err != nil {
return nil, err
}
return changes.Change, nil
}
// TODO: handle directories in filePathToContent
func setupPFSInputCommit(t *testing.T, pfsAPIClient pfs.APIClient, repo *pfs.Repo, filePathToContent map[string][]byte) *pfs.Commit {
commit, err := pfsAPIClient.StartCommit(
context.Background(),
&pfs.StartCommitRequest{
Parent: &pfs.Commit{
Repo: repo,
Id: pfs.InitialCommitID,
},
},
)
require.NoError(t, err)
for filePath, content := range filePathToContent {
apiPutFileClient, err := pfsAPIClient.PutFile(
context.Background(),
)
require.NoError(t, err)
err = apiPutFileClient.Send(
&pfs.PutFileRequest{
File: &pfs.File{
Commit: commit,
Path: filePath,
},
FileType: pfs.FileType_FILE_TYPE_REGULAR,
Value: content,
},
)
require.NoError(t, err)
_, _ = apiPutFileClient.CloseAndRecv()
}
_, err = pfsAPIClient.FinishCommit(
context.Background(),
&pfs.FinishCommitRequest{
Commit: commit,
},
)
require.NoError(t, err)
return commit
}