func RunBench(
b *testing.B,
f func(*testing.B, pfs.ApiClient),
) {
discoveryClient, err := getEtcdClient()
require.NoError(b, err)
var cluster Cluster
prototest.RunB(
b,
testNumServers,
func(servers map[string]*grpc.Server) {
cluster = registerFunc(b, discoveryClient, servers)
},
func(b *testing.B, clientConns map[string]*grpc.ClientConn) {
var clientConn *grpc.ClientConn
for _, c := range clientConns {
clientConn = c
break
}
f(
b,
pfs.NewApiClient(
clientConn,
),
)
},
)
cluster.Shutdown()
}
func (a *combinedAPIServer) GetFile(getFileRequest *pfs.GetFileRequest, apiGetFileServer pfs.Api_GetFileServer) (retErr error) {
shard, clientConn, err := a.getShardAndClientConnIfNecessary(getFileRequest.Path, true)
if err != nil {
return err
}
if clientConn != nil {
apiGetFileClient, err := pfs.NewApiClient(clientConn).GetFile(context.Background(), getFileRequest)
if err != nil {
return err
}
return protoutil.RelayFromStreamingBytesClient(apiGetFileClient, apiGetFileServer)
}
file, err := a.driver.GetFile(getFileRequest.Path, shard)
if err != nil {
return err
}
defer func() {
if err := file.Close(); err != nil && retErr == nil {
retErr = err
}
}()
return protoutil.WriteToStreamingBytesServer(
io.NewSectionReader(file, getFileRequest.OffsetBytes, getFileRequest.SizeBytes),
apiGetFileServer,
)
}
func (a *combinedAPIServer) MakeDirectory(ctx context.Context, makeDirectoryRequest *pfs.MakeDirectoryRequest) (*google_protobuf.Empty, error) {
shards, err := a.getAllShards(true)
if err != nil {
return nil, err
}
if err := a.driver.MakeDirectory(makeDirectoryRequest.Path, shards); err != nil {
return nil, err
}
if !makeDirectoryRequest.Redirect {
clientConns, err := a.router.GetAllClientConns()
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
if _, err := pfs.NewApiClient(clientConn).MakeDirectory(
ctx,
&pfs.MakeDirectoryRequest{
Path: makeDirectoryRequest.Path,
Redirect: true,
},
); err != nil {
return nil, err
}
}
}
return emptyInstance, nil
}
func RunBench(
b *testing.B,
f func(b *testing.B, apiClient pfs.ApiClient),
) {
discoveryClient, err := getEtcdClient()
require.NoError(b, err)
grpctest.RunB(
b,
testNumServers,
func(servers map[string]*grpc.Server) {
registerFunc(getDriver(b), discoveryClient, servers)
},
func(b *testing.B, clientConns map[string]*grpc.ClientConn) {
var clientConn *grpc.ClientConn
for _, c := range clientConns {
clientConn = c
break
}
f(
b,
pfs.NewApiClient(
clientConn,
),
)
},
)
}
func RunTest(
t *testing.T,
f func(*testing.T, pfs.ApiClient, pfs.InternalApiClient, Cluster),
) {
discoveryClient, err := getEtcdClient()
require.NoError(t, err)
var cluster Cluster
prototest.RunT(
t,
testNumServers,
func(servers map[string]*grpc.Server) {
cluster = registerFunc(t, discoveryClient, servers)
},
func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
var clientConn *grpc.ClientConn
for _, c := range clientConns {
clientConn = c
break
}
f(
t,
pfs.NewApiClient(
clientConn,
),
pfs.NewInternalApiClient(
clientConn,
),
cluster,
)
},
)
cluster.Shutdown()
}
func runTest(
t *testing.T,
driver drive.Driver,
f func(t *testing.T, apiClient pfs.ApiClient),
) {
grpctest.Run(
t,
testNumServers,
func(servers map[string]*grpc.Server) {
discoveryClient := discovery.NewMockClient()
i := 0
addresses := make([]string, testNumServers)
for address := range servers {
shards := make([]string, testShardsPerServer)
for j := 0; j < testShardsPerServer; j++ {
shards[j] = fmt.Sprintf("%d", (i*testShardsPerServer)+j)
}
_ = discoveryClient.Set(address+"-master", strings.Join(shards, ","))
addresses[i] = address
i++
}
_ = discoveryClient.Set("all-addresses", strings.Join(addresses, ","))
for address, server := range servers {
combinedAPIServer := NewCombinedAPIServer(
route.NewSharder(
testShardsPerServer*testNumServers,
),
route.NewRouter(
route.NewDiscoveryAddresser(
discoveryClient,
),
route.NewDialer(),
address,
),
driver,
)
pfs.RegisterApiServer(server, combinedAPIServer)
pfs.RegisterInternalApiServer(server, combinedAPIServer)
}
},
func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
var clientConn *grpc.ClientConn
for _, c := range clientConns {
clientConn = c
break
}
for _, c := range clientConns {
if c != clientConn {
_ = c.Close()
}
}
f(
t,
pfs.NewApiClient(
clientConn,
),
)
},
)
}
func (a *combinedAPIServer) Commit(ctx context.Context, commitRequest *pfs.CommitRequest) (*google_protobuf.Empty, error) {
shards, err := a.getAllShards(false)
if err != nil {
return nil, err
}
if err := a.driver.Commit(commitRequest.Commit, shards); err != nil {
return nil, err
}
if !commitRequest.Redirect {
clientConns, err := a.router.GetAllClientConns()
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
if _, err := pfs.NewApiClient(clientConn).Commit(
ctx,
&pfs.CommitRequest{
Commit: commitRequest.Commit,
Redirect: true,
},
); err != nil {
return nil, err
}
}
}
return emptyInstance, nil
}
func RunTest(
t *testing.T,
f func(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient),
) {
discoveryClient, err := getEtcdClient()
require.NoError(t, err)
grpctest.Run(
t,
testNumServers,
func(servers map[string]*grpc.Server) {
registerFunc(getDriver(t), discoveryClient, servers)
},
func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
var clientConn *grpc.ClientConn
for _, c := range clientConns {
clientConn = c
break
}
f(
t,
pfs.NewApiClient(
clientConn,
),
pfs.NewInternalApiClient(
clientConn,
),
)
},
)
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
logrus.Register()
address := appEnv.PachydermPfsd1Port
if address == "" {
address = appEnv.PfsAddress
} else {
address = strings.Replace(address, "tcp://", "", -1)
}
return dockervolume.Serve(
dockervolume.NewVolumeDriverHandler(
newVolumeDriver(
func() (fuse.Mounter, error) {
clientConn, err := grpc.Dial(address, grpc.WithInsecure())
if err != nil {
return nil, err
}
return fuse.NewMounter(
pfs.NewApiClient(
clientConn,
),
), nil
},
appEnv.BaseMountpoint,
),
dockervolume.VolumeDriverHandlerOptions{},
),
dockervolume.ProtocolUnix,
volumeDriverName,
volumeDriverGroup,
)
}
func (a *combinedAPIServer) InitRepository(ctx context.Context, initRepositoryRequest *pfs.InitRepositoryRequest) (*google_protobuf.Empty, error) {
shards, err := a.getAllShards(true)
if err != nil {
return nil, err
}
if err := a.driver.InitRepository(initRepositoryRequest.Repository, initRepositoryRequest.Replica, shards); err != nil {
return nil, err
}
if !initRepositoryRequest.Redirect {
clientConns, err := a.router.GetAllClientConns()
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
if _, err := pfs.NewApiClient(clientConn).InitRepository(
ctx,
&pfs.InitRepositoryRequest{
Repository: initRepositoryRequest.Repository,
Replica: initRepositoryRequest.Replica,
Redirect: true,
},
); err != nil {
return nil, err
}
}
}
return emptyInstance, nil
}
func (a *combinedAPIServer) Branch(ctx context.Context, branchRequest *pfs.BranchRequest) (*pfs.BranchResponse, error) {
if branchRequest.Redirect && branchRequest.NewCommit == nil {
return nil, fmt.Errorf("must set a new commit for redirect %+v", branchRequest)
}
shards, err := a.getAllShards(false)
if err != nil {
return nil, err
}
newCommit, err := a.driver.Branch(branchRequest.Commit, branchRequest.NewCommit, shards)
if err != nil {
return nil, err
}
if !branchRequest.Redirect {
clientConns, err := a.router.GetAllClientConns()
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
if _, err := pfs.NewApiClient(clientConn).Branch(
ctx,
&pfs.BranchRequest{
Commit: branchRequest.Commit,
Redirect: true,
NewCommit: newCommit,
},
); err != nil {
return nil, err
}
}
}
return &pfs.BranchResponse{
Commit: newCommit,
}, nil
}
func (m *mounterProvider) getOnce() (Mounter, error) {
clientConn, err := grpc.Dial(m.pfsAddress, grpc.WithInsecure())
if err != nil {
return nil, err
}
return NewMounter(
pfs.NewApiClient(
clientConn,
),
), nil
}
func (a *combinedAPIServer) InitRepository(ctx context.Context, initRepositoryRequest *pfs.InitRepositoryRequest) (*google_protobuf.Empty, error) {
masterShards, err := a.router.GetMasterShards()
if err != nil {
return nil, err
}
if err := a.driver.InitRepository(initRepositoryRequest.Repository, masterShards); err != nil {
return nil, err
}
replicaShards, err := a.router.GetReplicaShards()
if err != nil {
return nil, err
}
if err := a.driver.InitRepository(initRepositoryRequest.Repository, replicaShards); err != nil {
return nil, err
}
if !initRepositoryRequest.Redirect {
clientConns, err := a.router.GetAllClientConns()
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
if _, err := pfs.NewApiClient(clientConn).InitRepository(
ctx,
&pfs.InitRepositoryRequest{
Repository: initRepositoryRequest.Repository,
Redirect: true,
},
); err != nil {
return nil, err
}
}
// Create the initial commit
if _, err = a.Branch(ctx, &pfs.BranchRequest{
Commit: nil,
NewCommit: &pfs.Commit{
Repository: initRepositoryRequest.Repository,
Id: InitialCommitID,
},
Redirect: false,
}); err != nil {
return nil, err
}
if _, err = a.Commit(ctx, &pfs.CommitRequest{
Commit: &pfs.Commit{
Repository: initRepositoryRequest.Repository,
Id: InitialCommitID,
},
Redirect: false,
}); err != nil {
return nil, err
}
}
return emptyInstance, nil
}
func (a *combinedAPIServer) PutFile(ctx context.Context, putFileRequest *pfs.PutFileRequest) (*google_protobuf.Empty, error) {
shard, clientConn, err := a.getShardAndClientConnIfNecessary(putFileRequest.Path, false)
if err != nil {
return nil, err
}
if clientConn != nil {
return pfs.NewApiClient(clientConn).PutFile(ctx, putFileRequest)
}
if err := a.driver.PutFile(putFileRequest.Path, shard, bytes.NewReader(putFileRequest.Value)); err != nil {
return nil, err
}
return emptyInstance, nil
}
func getPfsAPIClient(address string) (pfs.ApiClient, error) {
var err error
if address == "" {
address, err = getPfsAddress()
if err != nil {
return nil, err
}
}
clientConn, err := grpc.Dial(address)
if err != nil {
return nil, err
}
return pfs.NewApiClient(clientConn), nil
}
func (a *combinedAPIServer) Master(shard int) error {
clientConns, err := a.router.GetReplicaClientConns(shard)
if err != nil {
return err
}
for _, clientConn := range clientConns {
apiClient := pfs.NewApiClient(clientConn)
response, err := apiClient.ListRepositories(context.Background(), &pfs.ListRepositoriesRequest{})
if err != nil {
return err
}
for _, repository := range response.Repository {
if err := a.driver.InitRepository(repository, map[int]bool{shard: true}); err != nil {
return err
}
response, err := apiClient.ListCommits(context.Background(), &pfs.ListCommitsRequest{Repository: repository})
if err != nil {
return err
}
localCommitInfo, err := a.driver.ListCommits(repository, shard)
if err != nil {
return err
}
for i, commitInfo := range response.CommitInfo {
if i < len(localCommitInfo) {
if *commitInfo != *localCommitInfo[i] {
return fmt.Errorf("divergent data")
}
continue
}
pullDiffClient, err := pfs.NewInternalApiClient(clientConn).PullDiff(
context.Background(),
&pfs.PullDiffRequest{
Commit: commitInfo.Commit,
Shard: uint64(shard),
},
)
if err != nil {
return err
}
if err := a.driver.PushDiff(commitInfo.Commit, protostream.NewStreamingBytesReader(pullDiffClient)); err != nil {
return err
}
}
}
}
return nil
}
func (a *apiServer) DeleteCommit(ctx context.Context, request *pfs.DeleteCommitRequest) (*google_protobuf.Empty, error) {
version, ctx, err := a.versionAndCtx(ctx)
if err != nil {
return nil, err
}
clientConns, err := a.router.GetAllClientConns(version)
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
if _, err := pfs.NewApiClient(clientConn).DeleteCommit(ctx, request); err != nil {
return nil, err
}
}
return emptyInstance, nil
}
// TODO(pedge): race on Branch
func (a *combinedAPIServer) GetCommitInfo(ctx context.Context, getCommitInfoRequest *pfs.GetCommitInfoRequest) (*pfs.GetCommitInfoResponse, error) {
shard, clientConn, err := a.getMasterShardOrMasterClientConnIfNecessary()
if err != nil {
return nil, err
}
if clientConn != nil {
return pfs.NewApiClient(clientConn).GetCommitInfo(ctx, getCommitInfoRequest)
}
commitInfo, err := a.driver.GetCommitInfo(getCommitInfoRequest.Commit, shard)
if err != nil {
return nil, err
}
return &pfs.GetCommitInfoResponse{
CommitInfo: commitInfo,
}, nil
}
func (a *combinedAPIServer) ListCommits(ctx context.Context, listCommitsRequest *pfs.ListCommitsRequest) (*pfs.ListCommitsResponse, error) {
shard, clientConn, err := a.getMasterShardOrMasterClientConnIfNecessary()
if err != nil {
return nil, err
}
if clientConn != nil {
return pfs.NewApiClient(clientConn).ListCommits(ctx, listCommitsRequest)
}
commitInfos, err := a.driver.ListCommits(listCommitsRequest.Repository, shard)
if err != nil {
return nil, err
}
return &pfs.ListCommitsResponse{
CommitInfo: commitInfos,
}, nil
}
func (a *combinedAPIServer) ListFiles(ctx context.Context, listFilesRequest *pfs.ListFilesRequest) (*pfs.ListFilesResponse, error) {
shards, err := a.getAllShards(false)
if err != nil {
return nil, err
}
dynamicShard := listFilesRequest.Shard
if dynamicShard == nil {
dynamicShard = &pfs.Shard{Number: 0, Modulo: 1}
}
filteredShards := make(map[int]bool)
for shard := range shards {
if uint64(shard)%dynamicShard.Modulo == dynamicShard.Number {
filteredShards[shard] = true
}
}
var fileInfos []*pfs.FileInfo
for shard := range filteredShards {
subFileInfos, err := a.driver.ListFiles(listFilesRequest.Path, shard)
if err != nil {
return nil, err
}
fileInfos = append(fileInfos, subFileInfos...)
}
if !listFilesRequest.Redirect {
clientConns, err := a.router.GetAllClientConns()
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
listFilesResponse, err := pfs.NewApiClient(clientConn).ListFiles(
ctx,
&pfs.ListFilesRequest{
Path: listFilesRequest.Path,
Shard: listFilesRequest.Shard,
Redirect: true,
},
)
if err != nil {
return nil, err
}
fileInfos = append(fileInfos, listFilesResponse.FileInfo...)
}
}
return &pfs.ListFilesResponse{
FileInfo: fileInfos,
}, nil
}
func (a *combinedAPIServer) GetFileInfo(ctx context.Context, getFileInfoRequest *pfs.GetFileInfoRequest) (*pfs.FileInfo, error) {
shard, clientConn, err := a.getShardAndClientConnIfNecessary(getFileInfoRequest.Path, false)
if err != nil {
return nil, err
}
if clientConn != nil {
return pfs.NewApiClient(clientConn).GetFileInfo(context.Background(), getFileInfoRequest)
}
fileInfo, ok, err := a.driver.GetFileInfo(getFileInfoRequest.Path, shard)
if err != nil {
return nil, err
}
if !ok {
return &pfs.FileInfo{}, nil
}
return fileInfo, nil
}
func (a *combinedAPIServer) PutFile(ctx context.Context, putFileRequest *pfs.PutFileRequest) (*google_protobuf.Empty, error) {
if strings.HasPrefix(putFileRequest.Path.Path, "/") {
// This is a subtle error case, the paths foo and /foo will hash to
// different shards but will produce the same change once they get to
// those shards due to how path.Join. This can go wrong in a number of
// ways so we forbid leading slashes.
return nil, fmt.Errorf("pachyderm: leading slash in path: %s", putFileRequest.Path.Path)
}
shard, clientConn, err := a.getShardAndClientConnIfNecessary(putFileRequest.Path, false)
if err != nil {
return nil, err
}
if clientConn != nil {
return pfs.NewApiClient(clientConn).PutFile(ctx, putFileRequest)
}
if err := a.driver.PutFile(putFileRequest.Path, shard, putFileRequest.OffsetBytes, bytes.NewReader(putFileRequest.Value)); err != nil {
return nil, err
}
return emptyInstance, nil
}
func (a *combinedAPIServer) ListRepositories(ctx context.Context, listRepositoriesRequest *pfs.ListRepositoriesRequest) (*pfs.Repositories, error) {
masterShards, err := a.router.GetMasterShards()
if err != nil {
return nil, err
}
for shard := range masterShards {
repositories, err := a.driver.ListRepositories(shard)
if err != nil {
return nil, err
}
return &pfs.Repositories{Repository: repositories}, nil
}
if !listRepositoriesRequest.Redirect {
clientConns, err := a.router.GetAllClientConns()
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
return pfs.NewApiClient(clientConn).ListRepositories(ctx, &pfs.ListRepositoriesRequest{Redirect: true})
}
}
return nil, fmt.Errorf("pachyderm: no available masters")
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
logrus.Register()
containerClient, err := getContainerClient(appEnv.DockerHost)
if err != nil {
return err
}
rethinkClient, err := getRethinkClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
if err != nil {
return err
}
pfsAddress := appEnv.PachydermPfsd1Port
if pfsAddress == "" {
pfsAddress = appEnv.PfsAddress
} else {
pfsAddress = strings.Replace(pfsAddress, "tcp://", "", -1)
}
clientConn, err := grpc.Dial(pfsAddress, grpc.WithInsecure())
if err != nil {
return err
}
// TODO(pedge): no!
trace.AuthRequest = func(_ *http.Request) (bool, bool) {
return true, true
}
return protoserver.Serve(
uint16(appEnv.Port),
func(s *grpc.Server) {
pps.RegisterApiServer(s, server.NewAPIServer(pfs.NewApiClient(clientConn), containerClient, rethinkClient, timing.NewSystemTimer()))
},
protoserver.ServeOptions{
DebugPort: uint16(appEnv.DebugPort),
Version: pachyderm.Version,
},
)
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
clientConn, err := grpc.Dial(appEnv.Address)
if err != nil {
return err
}
apiClient := pfs.NewApiClient(clientConn)
var shard int
var modulus int
initCmd := cobramainutil.Command{
Use: "init repository-name",
Long: "Initalize a repository.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.InitRepository(apiClient, args[0], false)
},
}.ToCobraCommand()
mkdirCmd := cobramainutil.Command{
Use: "mkdir repository-name commit-id path/to/dir",
Long: "Make a directory. Sub directories must already exist.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.MakeDirectory(apiClient, args[0], args[1], args[2])
},
}.ToCobraCommand()
putCmd := cobramainutil.Command{
Use: "put repository-name branch-id path/to/file",
Long: "Put a file from stdin. Directories must exist. branch-id must be a writeable commit.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
_, err := pfsutil.PutFile(apiClient, args[0], args[1], args[2], 0, os.Stdin)
return err
},
}.ToCobraCommand()
getCmd := cobramainutil.Command{
Use: "get repository-name commit-id path/to/file",
Long: "Get a file from stdout. commit-id must be a readable commit.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.GetFile(apiClient, args[0], args[1], args[2], 0, pfsutil.GetAll, os.Stdout)
},
}.ToCobraCommand()
lsCmd := cobramainutil.Command{
Use: "ls repository-name branch-id path/to/dir",
Long: "List a directory. Directory must exist.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
listFilesResponse, err := pfsutil.ListFiles(apiClient, args[0], args[1], args[2], uint64(shard), uint64(modulus))
if err != nil {
return err
}
for _, fileInfo := range listFilesResponse.FileInfo {
fmt.Printf("%+v\n", fileInfo)
}
return nil
},
}.ToCobraCommand()
lsCmd.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
lsCmd.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
branchCmd := cobramainutil.Command{
Use: "branch repository-name commit-id",
Long: "Branch a commit. commit-id must be a readable commit.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
branchResponse, err := pfsutil.Branch(apiClient, args[0], args[1])
if err != nil {
return err
}
fmt.Println(branchResponse.Commit.Id)
return nil
},
}.ToCobraCommand()
commitCmd := cobramainutil.Command{
Use: "commit repository-name branch-id",
Long: "Commit a branch. branch-id must be a writeable commit.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.Commit(apiClient, args[0], args[1])
},
}.ToCobraCommand()
commitInfoCmd := cobramainutil.Command{
Use: "commit-info repository-name commit-id",
Long: "Get info for a commit.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
commitInfoResponse, err := pfsutil.GetCommitInfo(apiClient, args[0], args[1])
if err != nil {
return err
}
fmt.Printf("%+v\n", commitInfoResponse.CommitInfo)
return nil
},
}.ToCobraCommand()
listCommitsCmd := cobramainutil.Command{
Use: "list-commits repository-name",
Long: "List commits on the repository.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
listCommitsResponse, err := pfsutil.ListCommits(apiClient, args[0])
if err != nil {
return err
}
for _, commitInfo := range listCommitsResponse.CommitInfo {
fmt.Printf("%+v\n", commitInfo)
}
return nil
},
}.ToCobraCommand()
mountCmd := cobramainutil.Command{
Use: "mount repository-name",
Long: "Mount a repository as a local file system.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
return fuse.NewMounter().Mount(apiClient, args[0], args[0], uint64(shard), uint64(modulus))
},
}.ToCobraCommand()
mountCmd.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
mountCmd.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
rootCmd := &cobra.Command{
Use: "pfs",
Long: `Access the PFS API.
Note that this CLI is experimental and does not even check for common errors.
The environment variable PFS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:650.`,
}
rootCmd.AddCommand(cobramainutil.NewVersionCommand(clientConn, pachyderm.Version))
rootCmd.AddCommand(initCmd)
rootCmd.AddCommand(mkdirCmd)
rootCmd.AddCommand(putCmd)
rootCmd.AddCommand(getCmd)
rootCmd.AddCommand(lsCmd)
rootCmd.AddCommand(branchCmd)
rootCmd.AddCommand(commitCmd)
rootCmd.AddCommand(commitInfoCmd)
rootCmd.AddCommand(listCommitsCmd)
rootCmd.AddCommand(mountCmd)
return rootCmd.Execute()
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
logrus.Register()
address := appEnv.PachydermPfsd1Port
if address == "" {
address = appEnv.Address
} else {
address = strings.Replace(address, "tcp://", "", -1)
}
clientConn, err := grpc.Dial(address, grpc.WithInsecure())
if err != nil {
return err
}
apiClient := pfs.NewApiClient(clientConn)
var shard int
var modulus int
createRepo := cobramainutil.Command{
Use: "create-repo repo-name",
Short: "Create a new repo.",
Long: "Create a new repo.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.CreateRepo(apiClient, args[0])
},
}.ToCobraCommand()
inspectRepo := cobramainutil.Command{
Use: "inspect-repo repo-name",
Short: "Return info about a repo.",
Long: "Return info about a repo.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
repoInfo, err := pfsutil.InspectRepo(apiClient, args[0])
if err != nil {
return err
}
if repoInfo == nil {
return fmt.Errorf("repo %s not found", args[0])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintRepoHeader(writer)
pretty.PrintRepoInfo(writer, repoInfo)
return writer.Flush()
},
}.ToCobraCommand()
listRepo := cobramainutil.Command{
Use: "list-repo",
Short: "Return all repos.",
Long: "Reutrn all repos.",
NumArgs: 0,
Run: func(cmd *cobra.Command, args []string) error {
repoInfos, err := pfsutil.ListRepo(apiClient)
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintRepoHeader(writer)
for _, repoInfo := range repoInfos {
pretty.PrintRepoInfo(writer, repoInfo)
}
return writer.Flush()
},
}.ToCobraCommand()
deleteRepo := cobramainutil.Command{
Use: "delete-repo repo-name",
Short: "Delete a repo.",
Long: "Delete a repo.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.DeleteRepo(apiClient, args[0])
},
}.ToCobraCommand()
startCommit := cobramainutil.Command{
Use: "start-commit repo-name parent-commit-id",
Short: "Start a new commit.",
Long: "Start a new commit with parent-commit-id as the parent.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
commit, err := pfsutil.StartCommit(apiClient, args[0], args[1])
if err != nil {
return err
}
fmt.Println(commit.Id)
return nil
},
}.ToCobraCommand()
finishCommit := cobramainutil.Command{
Use: "finish-commit repo-name commit-id",
Short: "Finish a started commit.",
Long: "Finish a started commit. Commit-id must be a writeable commit.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.FinishCommit(apiClient, args[0], args[1])
},
}.ToCobraCommand()
inspectCommit := cobramainutil.Command{
Use: "inspect-commit repo-name commit-id",
Short: "Return info about a commit.",
Long: "Return info about a commit.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
commitInfo, err := pfsutil.InspectCommit(apiClient, args[0], args[1])
if err != nil {
return err
}
if commitInfo == nil {
return fmt.Errorf("commit %s not found", args[1])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintCommitInfoHeader(writer)
pretty.PrintCommitInfo(writer, commitInfo)
return writer.Flush()
},
}.ToCobraCommand()
listCommit := cobramainutil.Command{
Use: "list-commit repo-name",
Short: "Return all commits on a repo.",
Long: "Return all commits on a repo.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
commitInfos, err := pfsutil.ListCommit(apiClient, args[0])
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintCommitInfoHeader(writer)
for _, commitInfo := range commitInfos {
pretty.PrintCommitInfo(writer, commitInfo)
}
return writer.Flush()
},
}.ToCobraCommand()
deleteCommit := cobramainutil.Command{
Use: "delete-commit repo-name commit-id",
Short: "Delete a commit.",
Long: "Delete a commit.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.DeleteCommit(apiClient, args[0], args[1])
},
}.ToCobraCommand()
putBlock := cobramainutil.Command{
Use: "put-block repo-name commit-id path/to/file",
Short: "Put a block from stdin",
Long: "Put a block from stdin. Directories must exist. commit-id must be a writeable commit.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
block, err := pfsutil.PutBlock(apiClient, args[0], args[1], args[2], os.Stdin)
if err != nil {
return err
}
fmt.Println(block.Hash)
return nil
},
}.ToCobraCommand()
getBlock := cobramainutil.Command{
Use: "get-block hash",
Short: "Return the contents of a block.",
Long: "Return the contents of a block.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.GetBlock(apiClient, args[0], os.Stdout)
},
}.ToCobraCommand()
inspectBlock := cobramainutil.Command{
Use: "inspect-block hash",
Short: "Return info about a block.",
Long: "Return info about a block.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
blockInfo, err := pfsutil.InspectBlock(apiClient, args[0])
if err != nil {
return err
}
if blockInfo == nil {
return fmt.Errorf("block %s not found", args[2])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintBlockInfoHeader(writer)
pretty.PrintBlockInfo(writer, blockInfo)
return writer.Flush()
},
}.ToCobraCommand()
listBlock := cobramainutil.Command{
Use: "list-block",
Short: "Return the blocks in a directory.",
Long: "Return the blocks in a directory.",
NumArgs: 0,
Run: func(cmd *cobra.Command, args []string) error {
blockInfos, err := pfsutil.ListBlock(apiClient, uint64(shard), uint64(modulus))
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintBlockInfoHeader(writer)
for _, blockInfo := range blockInfos {
pretty.PrintBlockInfo(writer, blockInfo)
}
return writer.Flush()
},
}.ToCobraCommand()
listBlock.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
listBlock.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
mkdir := cobramainutil.Command{
Use: "mkdir repo-name commit-id path/to/dir",
Short: "Make a directory.",
Long: "Make a directory. Parent directories need not exist.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.MakeDirectory(apiClient, args[0], args[1], args[2])
},
}.ToCobraCommand()
putFile := cobramainutil.Command{
Use: "put-file repo-name commit-id path/to/file",
Short: "Put a file from stdin",
Long: "Put a file from stdin. Directories must exist. commit-id must be a writeable commit.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
_, err := pfsutil.PutFile(apiClient, args[0], args[1], args[2], 0, os.Stdin)
return err
},
}.ToCobraCommand()
getFile := cobramainutil.Command{
Use: "get-file repo-name commit-id path/to/file",
Short: "Return the contents of a file.",
Long: "Return the contents of a file.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.GetFile(apiClient, args[0], args[1], args[2], 0, math.MaxInt64, os.Stdout)
},
}.ToCobraCommand()
inspectFile := cobramainutil.Command{
Use: "inspect-file repo-name commit-id path/to/file",
Short: "Return info about a file.",
Long: "Return info about a file.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
fileInfo, err := pfsutil.InspectFile(apiClient, args[0], args[1], args[2])
if err != nil {
return err
}
if fileInfo == nil {
return fmt.Errorf("file %s not found", args[2])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintFileInfoHeader(writer)
pretty.PrintFileInfo(writer, fileInfo)
return writer.Flush()
},
}.ToCobraCommand()
listFile := cobramainutil.Command{
Use: "list-file repo-name commit-id path/to/dir",
Short: "Return the files in a directory.",
Long: "Return the files in a directory.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
fileInfos, err := pfsutil.ListFile(apiClient, args[0], args[1], args[2], uint64(shard), uint64(modulus))
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintFileInfoHeader(writer)
for _, fileInfo := range fileInfos {
pretty.PrintFileInfo(writer, fileInfo)
}
return writer.Flush()
},
}.ToCobraCommand()
listFile.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
listFile.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
deleteFile := cobramainutil.Command{
Use: "delete-file repo-name commit-id path/to/file",
Short: "Delete a file.",
Long: "Delete a file.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.DeleteFile(apiClient, args[0], args[1], args[2])
},
}.ToCobraCommand()
listChange := cobramainutil.Command{
Use: "list-change repo-name commit-id path/to/dir",
Short: "Return the changes in a directory.",
Long: "Return the changes in a directory.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
changeInfos, err := pfsutil.ListChange(apiClient, args[0], args[1], args[2], uint64(shard), uint64(modulus))
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintChangeHeader(writer)
for _, changeInfo := range changeInfos {
pretty.PrintChange(writer, changeInfo)
}
return writer.Flush()
},
}.ToCobraCommand()
listChange.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
listChange.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
inspectServer := cobramainutil.Command{
Use: "inspect-server server-id",
Short: "Inspect a server.",
Long: "Inspect a server.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
serverInfo, err := pfsutil.InspectServer(apiClient, args[0])
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintServerInfoHeader(writer)
pretty.PrintServerInfo(writer, serverInfo)
return writer.Flush()
},
}.ToCobraCommand()
listServer := cobramainutil.Command{
Use: "list-server",
Short: "Return all servers in the cluster.",
Long: "Return all servers in the cluster.",
NumArgs: 0,
Run: func(cmd *cobra.Command, args []string) error {
serverInfos, err := pfsutil.ListServer(apiClient)
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintServerInfoHeader(writer)
for _, serverInfo := range serverInfos {
pretty.PrintServerInfo(writer, serverInfo)
}
return writer.Flush()
},
}.ToCobraCommand()
mount := cobramainutil.Command{
Use: "mount mountpoint repo-name [commit-id]",
Short: "Mount a repo as a local file system.",
Long: "Mount a repo as a local file system.",
MinNumArgs: 2,
MaxNumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
mountPoint := args[0]
repo := args[1]
commitID := ""
if len(args) == 3 {
commitID = args[2]
}
mounter := fuse.NewMounter(apiClient)
if err := mounter.Mount(repo, commitID, mountPoint, uint64(shard), uint64(modulus)); err != nil {
return err
}
return mounter.Wait(mountPoint)
},
}.ToCobraCommand()
mount.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
mount.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
rootCmd := &cobra.Command{
Use: "pfs",
Long: `Access the PFS API.
Note that this CLI is experimental and does not even check for common errors.
The environment variable PFS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:650.`,
}
rootCmd.AddCommand(protoclient.NewVersionCommand(clientConn, pachyderm.Version, nil))
rootCmd.AddCommand(createRepo)
rootCmd.AddCommand(inspectRepo)
rootCmd.AddCommand(listRepo)
rootCmd.AddCommand(deleteRepo)
rootCmd.AddCommand(startCommit)
rootCmd.AddCommand(finishCommit)
rootCmd.AddCommand(inspectCommit)
rootCmd.AddCommand(listCommit)
rootCmd.AddCommand(deleteCommit)
rootCmd.AddCommand(mkdir)
rootCmd.AddCommand(putBlock)
rootCmd.AddCommand(getBlock)
rootCmd.AddCommand(inspectBlock)
rootCmd.AddCommand(listBlock)
rootCmd.AddCommand(putFile)
rootCmd.AddCommand(getFile)
rootCmd.AddCommand(inspectFile)
rootCmd.AddCommand(listFile)
rootCmd.AddCommand(deleteFile)
rootCmd.AddCommand(listChange)
rootCmd.AddCommand(inspectServer)
rootCmd.AddCommand(listServer)
rootCmd.AddCommand(mount)
return rootCmd.Execute()
}