func getAPIClient(address string) (pfs.APIClient, error) {
clientConn, err := grpc.Dial(address, grpc.WithInsecure())
if err != nil {
return nil, err
}
return pfs.NewAPIClient(clientConn), nil
}
func RunBench(
b *testing.B,
f func(*testing.B, pfs.APIClient),
) {
discoveryClient, err := getEtcdClient()
require.NoError(b, err)
var cluster *cluster
prototest.RunB(
b,
testNumServers,
func(servers map[string]*grpc.Server) {
cluster = registerFunc(b, discoveryClient, servers)
},
func(b *testing.B, clientConns map[string]*grpc.ClientConn) {
var clientConn *grpc.ClientConn
for _, c := range clientConns {
clientConn = c
break
}
go func() {
require.Equal(b, cluster.realSharder.AssignRoles(cluster.cancel), shard.ErrCancelled)
}()
cluster.WaitForAvailability()
f(
b,
pfs.NewAPIClient(
clientConn,
),
)
},
)
cluster.Shutdown()
}
func getPfsClient(tb testing.TB) pfs.APIClient {
pfsdAddr := os.Getenv("PFSD_PORT_650_TCP_ADDR")
if pfsdAddr == "" {
tb.Error("PFSD_PORT_650_TCP_ADDR not set")
}
clientConn, err := grpc.Dial(fmt.Sprintf("%s:650", pfsdAddr), grpc.WithInsecure())
require.NoError(tb, err)
return pfs.NewAPIClient(clientConn)
}
func (m *mounterProvider) getOnce() (Mounter, error) {
clientConn, err := grpc.Dial(m.pfsAddress, grpc.WithInsecure())
if err != nil {
return nil, err
}
return NewMounter(
pfs.NewAPIClient(
clientConn,
),
), nil
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
containerClient, err := getContainerClient()
if err != nil {
return err
}
rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
if err != nil {
return err
}
pfsdAddress, err := getPfsdAddress()
if err != nil {
return err
}
clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
if err != nil {
return err
}
pfsAPIClient := pfs.NewAPIClient(clientConn)
jobAPIServer := jobserver.NewAPIServer(
pfsAPIClient,
rethinkAPIClient,
containerClient,
appEnv.PfsMountDir,
jobserverrun.JobRunnerOptions{
RemoveContainers: appEnv.RemoveContainers,
},
)
jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient)
errC := make(chan error)
go func() {
errC <- protoserver.Serve(
uint16(appEnv.Port),
func(s *grpc.Server) {
pps.RegisterJobAPIServer(s, jobAPIServer)
pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
},
protoserver.ServeOptions{
DebugPort: uint16(appEnv.DebugPort),
Version: pachyderm.Version,
},
)
}()
// TODO: pretty sure without this there is a problem, this is bad, we would
// prefer a callback for when the server is ready to accept requests
time.Sleep(1 * time.Second)
if err := pipelineAPIServer.Start(); err != nil {
return err
}
return <-errC
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
if err != nil {
return err
}
pfsdAddress, err := getPfsdAddress()
if err != nil {
return err
}
clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
if err != nil {
return err
}
pfsAPIClient := pfs.NewAPIClient(clientConn)
kubeAddr, err := getKubeAddress()
if err != nil {
return err
}
config := &kube.Config{
Host: kubeAddr,
Insecure: true,
}
kubeClient, err := kube.New(config)
if err != nil {
protolog.Printf("Error creating kubernetes client: %s", err.Error())
}
jobAPIServer := jobserver.NewAPIServer(
pfsAPIClient,
rethinkAPIClient,
kubeClient,
)
jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient)
if err := pipelineAPIServer.Start(); err != nil {
return err
}
return protoserver.Serve(
uint16(appEnv.Port),
func(s *grpc.Server) {
pps.RegisterJobAPIServer(s, jobAPIServer)
pps.RegisterInternalJobAPIServer(s, jobAPIServer)
pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
},
protoserver.ServeOptions{
DebugPort: uint16(appEnv.DebugPort),
Version: pachyderm.Version,
},
)
}
func (a *apiServer) DeleteCommit(ctx context.Context, request *pfs.DeleteCommitRequest) (*google_protobuf.Empty, error) {
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
if _, err := pfs.NewAPIClient(clientConn).DeleteCommit(ctx, request); err != nil {
return nil, err
}
}
return google_protobuf.EmptyInstance, nil
}
func (a *apiServer) getPfsClient() (pfs.APIClient, error) {
if a.pfsAPIClient == nil {
var onceErr error
a.pfsClientOnce.Do(func() {
clientConn, err := grpc.Dial(a.pfsAddress, grpc.WithInsecure())
if err != nil {
onceErr = err
}
a.pfsAPIClient = pfs.NewAPIClient(clientConn)
})
if onceErr != nil {
return nil, onceErr
}
}
return a.pfsAPIClient, nil
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
rethinkAPIServer, err := getRethinkAPIServer(appEnv.DatabaseAddress, appEnv.DatabaseName)
if err != nil {
return err
}
pfsdAddress, err := getPfsdAddress()
if err != nil {
return err
}
clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
if err != nil {
return err
}
pfsAPIClient := pfs.NewAPIClient(clientConn)
kubeClient, err := getKubeClient()
if err != nil {
return err
}
jobAPIServer := jobserver.NewAPIServer(
pfsAPIClient,
rethinkAPIServer,
kubeClient,
)
jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIServer)
if err := pipelineAPIServer.Start(); err != nil {
return err
}
return protoserver.Serve(
uint16(appEnv.Port),
func(s *grpc.Server) {
pps.RegisterJobAPIServer(s, jobAPIServer)
pps.RegisterInternalJobAPIServer(s, jobAPIServer)
pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
},
protoserver.ServeOptions{
DebugPort: uint16(appEnv.DebugPort),
Version: pachyderm.Version,
},
)
}
func TestWriteAndRead(t *testing.T) {
t.Parallel()
// don't leave goroutines running
var wg sync.WaitGroup
defer wg.Wait()
tmp, err := ioutil.TempDir("", "pachyderm-test-")
if err != nil {
t.Fatalf("tempdir: %v", err)
}
defer func() {
if err := os.RemoveAll(tmp); err != nil {
t.Errorf("cannot remove tempdir: %v", err)
}
}()
// closed on successful termination
quit := make(chan struct{})
defer close(quit)
listener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("cannot listen: %v", err)
}
defer func() {
_ = listener.Close()
}()
// TODO try to share more of this setup code with various main
// functions
localAddress := listener.Addr().String()
srv := grpc.NewServer()
const (
numShards = 1
)
sharder := shard.NewLocalSharder(localAddress, numShards)
hasher := pfs.NewHasher(numShards, 1)
router := shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
localAddress,
)
blockDir := filepath.Join(tmp, "blocks")
blockServer, err := server.NewLocalBlockAPIServer(blockDir)
if err != nil {
t.Fatalf("NewLocalBlockAPIServer: %v", err)
}
pfs.RegisterBlockAPIServer(srv, blockServer)
driver, err := drive.NewDriver(localAddress)
if err != nil {
t.Fatalf("NewDriver: %v", err)
}
apiServer := server.NewAPIServer(
hasher,
router,
)
pfs.RegisterAPIServer(srv, apiServer)
internalAPIServer := server.NewInternalAPIServer(
hasher,
router,
driver,
)
pfs.RegisterInternalAPIServer(srv, internalAPIServer)
wg.Add(1)
go func() {
defer wg.Done()
if err := srv.Serve(listener); err != nil {
select {
case <-quit:
// orderly shutdown
return
default:
t.Errorf("grpc serve: %v", err)
}
}
}()
clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
if err != nil {
t.Fatalf("grpc dial: %v", err)
}
apiClient := pfs.NewAPIClient(clientConn)
mounter := fuse.NewMounter(localAddress, apiClient)
mountpoint := filepath.Join(tmp, "mnt")
if err := os.Mkdir(mountpoint, 0700); err != nil {
t.Fatalf("mkdir mountpoint: %v", err)
}
ready := make(chan bool)
wg.Add(1)
go func() {
defer wg.Done()
if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
t.Errorf("mount and serve: %v", err)
}
}()
<-ready
defer func() {
if err := mounter.Unmount(mountpoint); err != nil {
t.Errorf("unmount: %v", err)
}
}()
const (
repoName = "foo"
)
if err := pfsutil.CreateRepo(apiClient, repoName); err != nil {
t.Fatalf("CreateRepo: %v", err)
}
commit, err := pfsutil.StartCommit(apiClient, repoName, "")
if err != nil {
t.Fatalf("StartCommit: %v", err)
}
const (
greeting = "Hello, world\n"
)
filePath := filepath.Join(mountpoint, repoName, commit.Id, "greeting")
if err := ioutil.WriteFile(filePath, []byte(greeting), 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
buf, err := ioutil.ReadFile(filePath)
if err != nil {
t.Fatalf("ReadFile: %v", err)
}
if g, e := string(buf), greeting; g != e {
t.Errorf("wrong content: %q != %q", g, e)
}
}
func TestCommitFinishedReadDir(t *testing.T) {
t.Parallel()
// don't leave goroutines running
var wg sync.WaitGroup
defer wg.Wait()
tmp, err := ioutil.TempDir("", "pachyderm-test-")
if err != nil {
t.Fatalf("tempdir: %v", err)
}
defer func() {
if err := os.RemoveAll(tmp); err != nil {
t.Errorf("cannot remove tempdir: %v", err)
}
}()
// closed on successful termination
quit := make(chan struct{})
defer close(quit)
listener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("cannot listen: %v", err)
}
defer func() {
_ = listener.Close()
}()
// TODO try to share more of this setup code with various main
// functions
localAddress := listener.Addr().String()
srv := grpc.NewServer()
const (
numShards = 1
)
sharder := shard.NewLocalSharder(localAddress, numShards)
hasher := pfs.NewHasher(numShards, 1)
router := shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
localAddress,
)
blockDir := filepath.Join(tmp, "blocks")
blockServer, err := server.NewLocalBlockAPIServer(blockDir)
if err != nil {
t.Fatalf("NewLocalBlockAPIServer: %v", err)
}
pfs.RegisterBlockAPIServer(srv, blockServer)
driver, err := drive.NewDriver(localAddress)
if err != nil {
t.Fatalf("NewDriver: %v", err)
}
apiServer := server.NewAPIServer(
hasher,
router,
)
pfs.RegisterAPIServer(srv, apiServer)
internalAPIServer := server.NewInternalAPIServer(
hasher,
router,
driver,
)
pfs.RegisterInternalAPIServer(srv, internalAPIServer)
wg.Add(1)
go func() {
defer wg.Done()
if err := srv.Serve(listener); err != nil {
select {
case <-quit:
// orderly shutdown
return
default:
t.Errorf("grpc serve: %v", err)
}
}
}()
clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
if err != nil {
t.Fatalf("grpc dial: %v", err)
}
apiClient := pfs.NewAPIClient(clientConn)
mounter := fuse.NewMounter(localAddress, apiClient)
mountpoint := filepath.Join(tmp, "mnt")
if err := os.Mkdir(mountpoint, 0700); err != nil {
t.Fatalf("mkdir mountpoint: %v", err)
}
ready := make(chan bool)
wg.Add(1)
go func() {
defer wg.Done()
if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
t.Errorf("mount and serve: %v", err)
}
}()
<-ready
defer func() {
if err := mounter.Unmount(mountpoint); err != nil {
t.Errorf("unmount: %v", err)
}
}()
const (
repoName = "foo"
)
if err := pfsutil.CreateRepo(apiClient, repoName); err != nil {
t.Fatalf("CreateRepo: %v", err)
}
commit, err := pfsutil.StartCommit(apiClient, repoName, "")
if err != nil {
t.Fatalf("StartCommit: %v", err)
}
t.Logf("open commit %v", commit.Id)
const (
greetingName = "greeting"
greeting = "Hello, world\n"
greetingPerm = 0644
)
if err := ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.Id, greetingName), []byte(greeting), greetingPerm); err != nil {
t.Fatalf("WriteFile: %v", err)
}
const (
scriptName = "script"
script = "#!/bin/sh\necho foo\n"
scriptPerm = 0750
)
if err := ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.Id, scriptName), []byte(script), scriptPerm); err != nil {
t.Fatalf("WriteFile: %v", err)
}
if err := pfsutil.FinishCommit(apiClient, repoName, commit.Id); err != nil {
t.Fatalf("FinishCommit: %v", err)
}
if err := fstestutil.CheckDir(filepath.Join(mountpoint, repoName, commit.Id), map[string]fstestutil.FileInfoCheck{
greetingName: func(fi os.FileInfo) error {
// TODO respect greetingPerm
if g, e := fi.Mode(), os.FileMode(0666); g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
if g, e := fi.Size(), int64(len(greeting)); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show fileModTime as mtime
// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
scriptName: func(fi os.FileInfo) error {
// TODO respect scriptPerm
if g, e := fi.Mode(), os.FileMode(0666); g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
if g, e := fi.Size(), int64(len(script)); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show fileModTime as mtime
// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
}); err != nil {
t.Errorf("wrong directory content: %v", err)
}
}
func TestRootReadDir(t *testing.T) {
t.Parallel()
// don't leave goroutines running
var wg sync.WaitGroup
defer wg.Wait()
tmp, err := ioutil.TempDir("", "pachyderm-test-")
if err != nil {
t.Fatalf("tempdir: %v", err)
}
defer func() {
if err := os.RemoveAll(tmp); err != nil {
t.Errorf("cannot remove tempdir: %v", err)
}
}()
// closed on successful termination
quit := make(chan struct{})
defer close(quit)
listener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("cannot listen: %v", err)
}
defer func() {
_ = listener.Close()
}()
// TODO try to share more of this setup code with various main
// functions
localAddress := listener.Addr().String()
srv := grpc.NewServer()
const (
numShards = 1
)
sharder := shard.NewLocalSharder(localAddress, numShards)
hasher := pfs.NewHasher(numShards, 1)
router := shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
localAddress,
)
blockDir := filepath.Join(tmp, "blocks")
blockServer, err := server.NewLocalBlockAPIServer(blockDir)
if err != nil {
t.Fatalf("NewLocalBlockAPIServer: %v", err)
}
pfs.RegisterBlockAPIServer(srv, blockServer)
driver, err := drive.NewDriver(localAddress)
if err != nil {
t.Fatalf("NewDriver: %v", err)
}
apiServer := server.NewAPIServer(
hasher,
router,
)
pfs.RegisterAPIServer(srv, apiServer)
internalAPIServer := server.NewInternalAPIServer(
hasher,
router,
driver,
)
pfs.RegisterInternalAPIServer(srv, internalAPIServer)
wg.Add(1)
go func() {
defer wg.Done()
if err := srv.Serve(listener); err != nil {
select {
case <-quit:
// orderly shutdown
return
default:
t.Errorf("grpc serve: %v", err)
}
}
}()
clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
if err != nil {
t.Fatalf("grpc dial: %v", err)
}
apiClient := pfs.NewAPIClient(clientConn)
mounter := fuse.NewMounter(localAddress, apiClient)
mountpoint := filepath.Join(tmp, "mnt")
if err := os.Mkdir(mountpoint, 0700); err != nil {
t.Fatalf("mkdir mountpoint: %v", err)
}
ready := make(chan bool)
wg.Add(1)
go func() {
defer wg.Done()
if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
t.Errorf("mount and serve: %v", err)
}
}()
<-ready
defer func() {
if err := mounter.Unmount(mountpoint); err != nil {
t.Errorf("unmount: %v", err)
}
}()
if err := pfsutil.CreateRepo(apiClient, "one"); err != nil {
t.Fatalf("CreateRepo: %v", err)
}
if err := pfsutil.CreateRepo(apiClient, "two"); err != nil {
t.Fatalf("CreateRepo: %v", err)
}
if err := fstestutil.CheckDir(mountpoint, map[string]fstestutil.FileInfoCheck{
"one": func(fi os.FileInfo) error {
if g, e := fi.Mode(), os.ModeDir|0555; g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
// TODO show repoSize in repo stat?
if g, e := fi.Size(), int64(0); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show RepoInfo.Created as time
// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
"two": func(fi os.FileInfo) error {
if g, e := fi.Mode(), os.ModeDir|0555; g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
// TODO show repoSize in repo stat?
if g, e := fi.Size(), int64(0); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show RepoInfo.Created as time
// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
}); err != nil {
t.Errorf("wrong directory content: %v", err)
}
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
address := appEnv.PachydermPfsd1Port
if address == "" {
address = appEnv.Address
} else {
address = strings.Replace(address, "tcp://", "", -1)
}
clientConn, err := grpc.Dial(address, grpc.WithInsecure())
if err != nil {
return err
}
apiClient := pfs.NewAPIClient(clientConn)
var shard int
var modulus int
createRepo := &cobra.Command{
Use: "create-repo repo-name",
Short: "Create a new repo.",
Long: "Create a new repo.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
return pfsutil.CreateRepo(apiClient, args[0])
}),
}
inspectRepo := &cobra.Command{
Use: "inspect-repo repo-name",
Short: "Return info about a repo.",
Long: "Return info about a repo.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
repoInfo, err := pfsutil.InspectRepo(apiClient, args[0])
if err != nil {
return err
}
if repoInfo == nil {
return fmt.Errorf("repo %s not found", args[0])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintRepoHeader(writer)
pretty.PrintRepoInfo(writer, repoInfo)
return writer.Flush()
}),
}
listRepo := &cobra.Command{
Use: "list-repo",
Short: "Return all repos.",
Long: "Reutrn all repos.",
Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
repoInfos, err := pfsutil.ListRepo(apiClient)
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintRepoHeader(writer)
for _, repoInfo := range repoInfos {
pretty.PrintRepoInfo(writer, repoInfo)
}
return writer.Flush()
}),
}
deleteRepo := &cobra.Command{
Use: "delete-repo repo-name",
Short: "Delete a repo.",
Long: "Delete a repo.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
return pfsutil.DeleteRepo(apiClient, args[0])
}),
}
startCommit := &cobra.Command{
Use: "start-commit repo-name parent-commit-id",
Short: "Start a new commit.",
Long: "Start a new commit with parent-commit-id as the parent.",
Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
commit, err := pfsutil.StartCommit(apiClient, args[0], args[1])
if err != nil {
return err
}
fmt.Println(commit.Id)
return nil
}),
}
finishCommit := &cobra.Command{
Use: "finish-commit repo-name commit-id",
Short: "Finish a started commit.",
Long: "Finish a started commit. Commit-id must be a writeable commit.",
Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
return pfsutil.FinishCommit(apiClient, args[0], args[1])
}),
}
inspectCommit := &cobra.Command{
Use: "inspect-commit repo-name commit-id",
Short: "Return info about a commit.",
Long: "Return info about a commit.",
Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
commitInfo, err := pfsutil.InspectCommit(apiClient, args[0], args[1])
if err != nil {
return err
}
if commitInfo == nil {
return fmt.Errorf("commit %s not found", args[1])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintCommitInfoHeader(writer)
pretty.PrintCommitInfo(writer, commitInfo)
return writer.Flush()
}),
}
listCommit := &cobra.Command{
Use: "list-commit repo-name",
Short: "Return all commits on a repo.",
Long: "Return all commits on a repo.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
commitInfos, err := pfsutil.ListCommit(apiClient, args[0])
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintCommitInfoHeader(writer)
for _, commitInfo := range commitInfos {
pretty.PrintCommitInfo(writer, commitInfo)
}
return writer.Flush()
}),
}
deleteCommit := &cobra.Command{
Use: "delete-commit repo-name commit-id",
Short: "Delete a commit.",
Long: "Delete a commit.",
Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
return pfsutil.DeleteCommit(apiClient, args[0], args[1])
}),
}
putBlock := &cobra.Command{
Use: "put-block repo-name commit-id path/to/file",
Short: "Put a block from stdin",
Long: "Put a block from stdin. Directories must exist. commit-id must be a writeable commit.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
block, err := pfsutil.PutBlock(apiClient, args[0], args[1], args[2], os.Stdin)
if err != nil {
return err
}
fmt.Println(block.Hash)
return nil
}),
}
getBlock := &cobra.Command{
Use: "get-block hash",
Short: "Return the contents of a block.",
Long: "Return the contents of a block.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
return pfsutil.GetBlock(apiClient, args[0], os.Stdout)
}),
}
inspectBlock := &cobra.Command{
Use: "inspect-block hash",
Short: "Return info about a block.",
Long: "Return info about a block.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
blockInfo, err := pfsutil.InspectBlock(apiClient, args[0])
if err != nil {
return err
}
if blockInfo == nil {
return fmt.Errorf("block %s not found", args[2])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintBlockInfoHeader(writer)
pretty.PrintBlockInfo(writer, blockInfo)
return writer.Flush()
}),
}
listBlock := &cobra.Command{
Use: "list-block",
Short: "Return the blocks in a directory.",
Long: "Return the blocks in a directory.",
Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
blockInfos, err := pfsutil.ListBlock(apiClient, uint64(shard), uint64(modulus))
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintBlockInfoHeader(writer)
for _, blockInfo := range blockInfos {
pretty.PrintBlockInfo(writer, blockInfo)
}
return writer.Flush()
}),
}
listBlock.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
listBlock.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
mkdir := &cobra.Command{
Use: "mkdir repo-name commit-id path/to/dir",
Short: "Make a directory.",
Long: "Make a directory. Parent directories need not exist.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
return pfsutil.MakeDirectory(apiClient, args[0], args[1], args[2])
}),
}
putFile := &cobra.Command{
Use: "put-file repo-name commit-id path/to/file",
Short: "Put a file from stdin",
Long: "Put a file from stdin. Directories must exist. commit-id must be a writeable commit.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
_, err := pfsutil.PutFile(apiClient, args[0], args[1], args[2], 0, os.Stdin)
return err
}),
}
getFile := &cobra.Command{
Use: "get-file repo-name commit-id path/to/file",
Short: "Return the contents of a file.",
Long: "Return the contents of a file.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
return pfsutil.GetFile(apiClient, args[0], args[1], args[2], 0, math.MaxInt64, os.Stdout)
}),
}
inspectFile := &cobra.Command{
Use: "inspect-file repo-name commit-id path/to/file",
Short: "Return info about a file.",
Long: "Return info about a file.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
fileInfo, err := pfsutil.InspectFile(apiClient, args[0], args[1], args[2])
if err != nil {
return err
}
if fileInfo == nil {
return fmt.Errorf("file %s not found", args[2])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintFileInfoHeader(writer)
pretty.PrintFileInfo(writer, fileInfo)
return writer.Flush()
}),
}
listFile := &cobra.Command{
Use: "list-file repo-name commit-id path/to/dir",
Short: "Return the files in a directory.",
Long: "Return the files in a directory.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
fileInfos, err := pfsutil.ListFile(apiClient, args[0], args[1], args[2], uint64(shard), uint64(modulus))
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintFileInfoHeader(writer)
for _, fileInfo := range fileInfos {
pretty.PrintFileInfo(writer, fileInfo)
}
return writer.Flush()
}),
}
listFile.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
listFile.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
deleteFile := &cobra.Command{
Use: "delete-file repo-name commit-id path/to/file",
Short: "Delete a file.",
Long: "Delete a file.",
Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
return pfsutil.DeleteFile(apiClient, args[0], args[1], args[2])
}),
}
listChange := &cobra.Command{
Use: "list-change repo-name commit-id path/to/dir",
Short: "Return the changes in a directory.",
Long: "Return the changes in a directory.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
changeInfos, err := pfsutil.ListChange(apiClient, args[0], args[1], args[2], uint64(shard), uint64(modulus))
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintChangeHeader(writer)
for _, changeInfo := range changeInfos {
pretty.PrintChange(writer, changeInfo)
}
return writer.Flush()
}),
}
listChange.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
listChange.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
inspectServer := &cobra.Command{
Use: "inspect-server server-id",
Short: "Inspect a server.",
Long: "Inspect a server.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
serverInfo, err := pfsutil.InspectServer(apiClient, args[0])
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintServerInfoHeader(writer)
pretty.PrintServerInfo(writer, serverInfo)
return writer.Flush()
}),
}
listServer := &cobra.Command{
Use: "list-server",
Short: "Return all servers in the cluster.",
Long: "Return all servers in the cluster.",
Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
serverInfos, err := pfsutil.ListServer(apiClient)
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintServerInfoHeader(writer)
for _, serverInfo := range serverInfos {
pretty.PrintServerInfo(writer, serverInfo)
}
return writer.Flush()
}),
}
mount := &cobra.Command{
Use: "mount [mountpoint]",
Short: "Mount pfs locally.",
Long: "Mount pfs locally.",
Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 0, Max: 1}, func(args []string) error {
mountPoint := "/pfs"
if len(args) > 0 {
mountPoint = args[0]
}
mounter := fuse.NewMounter(apiClient)
return mounter.Mount(address, mountPoint, uint64(shard), uint64(modulus))
}),
}
mount.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
mount.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
rootCmd := &cobra.Command{
Use: "pfs",
Long: `Access the PFS API.
Note that this CLI is experimental and does not even check for common errors.
The environment variable PFS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:650.`,
}
rootCmd.AddCommand(protoclient.NewVersionCommand(clientConn, pachyderm.Version, nil))
rootCmd.AddCommand(createRepo)
rootCmd.AddCommand(inspectRepo)
rootCmd.AddCommand(listRepo)
rootCmd.AddCommand(deleteRepo)
rootCmd.AddCommand(startCommit)
rootCmd.AddCommand(finishCommit)
rootCmd.AddCommand(inspectCommit)
rootCmd.AddCommand(listCommit)
rootCmd.AddCommand(deleteCommit)
rootCmd.AddCommand(mkdir)
rootCmd.AddCommand(putBlock)
rootCmd.AddCommand(getBlock)
rootCmd.AddCommand(inspectBlock)
rootCmd.AddCommand(listBlock)
rootCmd.AddCommand(putFile)
rootCmd.AddCommand(getFile)
rootCmd.AddCommand(inspectFile)
rootCmd.AddCommand(listFile)
rootCmd.AddCommand(deleteFile)
rootCmd.AddCommand(listChange)
rootCmd.AddCommand(inspectServer)
rootCmd.AddCommand(listServer)
rootCmd.AddCommand(mount)
return rootCmd.Execute()
}
func NewAPIClient(cc *grpc.ClientConn) *APIClient {
return &APIClient{
pfs.NewAPIClient(cc),
pps.NewAPIClient(cc),
}
}