func getAPIClient(address string) (pfs.APIClient, error) { clientConn, err := grpc.Dial(address, grpc.WithInsecure()) if err != nil { return nil, err } return pfs.NewAPIClient(clientConn), nil }
func RunBench( b *testing.B, f func(*testing.B, pfs.APIClient), ) { discoveryClient, err := getEtcdClient() require.NoError(b, err) var cluster *cluster prototest.RunB( b, testNumServers, func(servers map[string]*grpc.Server) { cluster = registerFunc(b, discoveryClient, servers) }, func(b *testing.B, clientConns map[string]*grpc.ClientConn) { var clientConn *grpc.ClientConn for _, c := range clientConns { clientConn = c break } go func() { require.Equal(b, cluster.realSharder.AssignRoles(cluster.cancel), shard.ErrCancelled) }() cluster.WaitForAvailability() f( b, pfs.NewAPIClient( clientConn, ), ) }, ) cluster.Shutdown() }
func getPfsClient(tb testing.TB) pfs.APIClient { pfsdAddr := os.Getenv("PFSD_PORT_650_TCP_ADDR") if pfsdAddr == "" { tb.Error("PFSD_PORT_650_TCP_ADDR not set") } clientConn, err := grpc.Dial(fmt.Sprintf("%s:650", pfsdAddr), grpc.WithInsecure()) require.NoError(tb, err) return pfs.NewAPIClient(clientConn) }
func (m *mounterProvider) getOnce() (Mounter, error) { clientConn, err := grpc.Dial(m.pfsAddress, grpc.WithInsecure()) if err != nil { return nil, err } return NewMounter( pfs.NewAPIClient( clientConn, ), ), nil }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) containerClient, err := getContainerClient() if err != nil { return err } rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName) if err != nil { return err } pfsdAddress, err := getPfsdAddress() if err != nil { return err } clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure()) if err != nil { return err } pfsAPIClient := pfs.NewAPIClient(clientConn) jobAPIServer := jobserver.NewAPIServer( pfsAPIClient, rethinkAPIClient, containerClient, appEnv.PfsMountDir, jobserverrun.JobRunnerOptions{ RemoveContainers: appEnv.RemoveContainers, }, ) jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer) pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient) errC := make(chan error) go func() { errC <- protoserver.Serve( uint16(appEnv.Port), func(s *grpc.Server) { pps.RegisterJobAPIServer(s, jobAPIServer) pps.RegisterPipelineAPIServer(s, pipelineAPIServer) }, protoserver.ServeOptions{ DebugPort: uint16(appEnv.DebugPort), Version: pachyderm.Version, }, ) }() // TODO: pretty sure without this there is a problem, this is bad, we would // prefer a callback for when the server is ready to accept requests time.Sleep(1 * time.Second) if err := pipelineAPIServer.Start(); err != nil { return err } return <-errC }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName) if err != nil { return err } pfsdAddress, err := getPfsdAddress() if err != nil { return err } clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure()) if err != nil { return err } pfsAPIClient := pfs.NewAPIClient(clientConn) kubeAddr, err := getKubeAddress() if err != nil { return err } config := &kube.Config{ Host: kubeAddr, Insecure: true, } kubeClient, err := kube.New(config) if err != nil { protolog.Printf("Error creating kubernetes client: %s", err.Error()) } jobAPIServer := jobserver.NewAPIServer( pfsAPIClient, rethinkAPIClient, kubeClient, ) jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer) pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient) if err := pipelineAPIServer.Start(); err != nil { return err } return protoserver.Serve( uint16(appEnv.Port), func(s *grpc.Server) { pps.RegisterJobAPIServer(s, jobAPIServer) pps.RegisterInternalJobAPIServer(s, jobAPIServer) pps.RegisterPipelineAPIServer(s, pipelineAPIServer) }, protoserver.ServeOptions{ DebugPort: uint16(appEnv.DebugPort), Version: pachyderm.Version, }, ) }
func (a *apiServer) DeleteCommit(ctx context.Context, request *pfs.DeleteCommitRequest) (*google_protobuf.Empty, error) { a.versionLock.RLock() defer a.versionLock.RUnlock() ctx = versionToContext(a.version, ctx) clientConns, err := a.router.GetAllClientConns(a.version) if err != nil { return nil, err } for _, clientConn := range clientConns { if _, err := pfs.NewAPIClient(clientConn).DeleteCommit(ctx, request); err != nil { return nil, err } } return google_protobuf.EmptyInstance, nil }
func (a *apiServer) getPfsClient() (pfs.APIClient, error) { if a.pfsAPIClient == nil { var onceErr error a.pfsClientOnce.Do(func() { clientConn, err := grpc.Dial(a.pfsAddress, grpc.WithInsecure()) if err != nil { onceErr = err } a.pfsAPIClient = pfs.NewAPIClient(clientConn) }) if onceErr != nil { return nil, onceErr } } return a.pfsAPIClient, nil }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) rethinkAPIServer, err := getRethinkAPIServer(appEnv.DatabaseAddress, appEnv.DatabaseName) if err != nil { return err } pfsdAddress, err := getPfsdAddress() if err != nil { return err } clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure()) if err != nil { return err } pfsAPIClient := pfs.NewAPIClient(clientConn) kubeClient, err := getKubeClient() if err != nil { return err } jobAPIServer := jobserver.NewAPIServer( pfsAPIClient, rethinkAPIServer, kubeClient, ) jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer) pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIServer) if err := pipelineAPIServer.Start(); err != nil { return err } return protoserver.Serve( uint16(appEnv.Port), func(s *grpc.Server) { pps.RegisterJobAPIServer(s, jobAPIServer) pps.RegisterInternalJobAPIServer(s, jobAPIServer) pps.RegisterPipelineAPIServer(s, pipelineAPIServer) }, protoserver.ServeOptions{ DebugPort: uint16(appEnv.DebugPort), Version: pachyderm.Version, }, ) }
func TestWriteAndRead(t *testing.T) { t.Parallel() // don't leave goroutines running var wg sync.WaitGroup defer wg.Wait() tmp, err := ioutil.TempDir("", "pachyderm-test-") if err != nil { t.Fatalf("tempdir: %v", err) } defer func() { if err := os.RemoveAll(tmp); err != nil { t.Errorf("cannot remove tempdir: %v", err) } }() // closed on successful termination quit := make(chan struct{}) defer close(quit) listener, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("cannot listen: %v", err) } defer func() { _ = listener.Close() }() // TODO try to share more of this setup code with various main // functions localAddress := listener.Addr().String() srv := grpc.NewServer() const ( numShards = 1 ) sharder := shard.NewLocalSharder(localAddress, numShards) hasher := pfs.NewHasher(numShards, 1) router := shard.NewRouter( sharder, grpcutil.NewDialer( grpc.WithInsecure(), ), localAddress, ) blockDir := filepath.Join(tmp, "blocks") blockServer, err := server.NewLocalBlockAPIServer(blockDir) if err != nil { t.Fatalf("NewLocalBlockAPIServer: %v", err) } pfs.RegisterBlockAPIServer(srv, blockServer) driver, err := drive.NewDriver(localAddress) if err != nil { t.Fatalf("NewDriver: %v", err) } apiServer := server.NewAPIServer( hasher, router, ) pfs.RegisterAPIServer(srv, apiServer) internalAPIServer := server.NewInternalAPIServer( hasher, router, driver, ) pfs.RegisterInternalAPIServer(srv, internalAPIServer) wg.Add(1) go func() { defer wg.Done() if err := srv.Serve(listener); err != nil { select { case <-quit: // orderly shutdown return default: t.Errorf("grpc serve: %v", err) } } }() clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure()) if err != nil { t.Fatalf("grpc dial: %v", err) } apiClient := pfs.NewAPIClient(clientConn) mounter := fuse.NewMounter(localAddress, apiClient) mountpoint := filepath.Join(tmp, "mnt") if err := os.Mkdir(mountpoint, 0700); err != nil { t.Fatalf("mkdir mountpoint: %v", err) } ready := make(chan bool) wg.Add(1) go func() { defer wg.Done() if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil { t.Errorf("mount and serve: %v", err) } }() <-ready defer func() { if err := mounter.Unmount(mountpoint); err != nil { t.Errorf("unmount: %v", err) } }() const ( repoName = "foo" ) if err := pfsutil.CreateRepo(apiClient, repoName); err != nil { t.Fatalf("CreateRepo: %v", err) } commit, err := pfsutil.StartCommit(apiClient, repoName, "") if err != nil { t.Fatalf("StartCommit: %v", err) } const ( greeting = "Hello, world\n" ) filePath := filepath.Join(mountpoint, repoName, commit.Id, "greeting") if err := ioutil.WriteFile(filePath, []byte(greeting), 0644); err != nil { t.Fatalf("WriteFile: %v", err) } buf, err := ioutil.ReadFile(filePath) if err != nil { t.Fatalf("ReadFile: %v", err) } if g, e := string(buf), greeting; g != e { t.Errorf("wrong content: %q != %q", g, e) } }
func TestCommitFinishedReadDir(t *testing.T) { t.Parallel() // don't leave goroutines running var wg sync.WaitGroup defer wg.Wait() tmp, err := ioutil.TempDir("", "pachyderm-test-") if err != nil { t.Fatalf("tempdir: %v", err) } defer func() { if err := os.RemoveAll(tmp); err != nil { t.Errorf("cannot remove tempdir: %v", err) } }() // closed on successful termination quit := make(chan struct{}) defer close(quit) listener, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("cannot listen: %v", err) } defer func() { _ = listener.Close() }() // TODO try to share more of this setup code with various main // functions localAddress := listener.Addr().String() srv := grpc.NewServer() const ( numShards = 1 ) sharder := shard.NewLocalSharder(localAddress, numShards) hasher := pfs.NewHasher(numShards, 1) router := shard.NewRouter( sharder, grpcutil.NewDialer( grpc.WithInsecure(), ), localAddress, ) blockDir := filepath.Join(tmp, "blocks") blockServer, err := server.NewLocalBlockAPIServer(blockDir) if err != nil { t.Fatalf("NewLocalBlockAPIServer: %v", err) } pfs.RegisterBlockAPIServer(srv, blockServer) driver, err := drive.NewDriver(localAddress) if err != nil { t.Fatalf("NewDriver: %v", err) } apiServer := server.NewAPIServer( hasher, router, ) pfs.RegisterAPIServer(srv, apiServer) internalAPIServer := server.NewInternalAPIServer( hasher, router, driver, ) pfs.RegisterInternalAPIServer(srv, internalAPIServer) wg.Add(1) go func() { defer wg.Done() if err := srv.Serve(listener); err != nil { select { case <-quit: // orderly shutdown return default: t.Errorf("grpc serve: %v", err) } } }() clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure()) if err != nil { t.Fatalf("grpc dial: %v", err) } apiClient := pfs.NewAPIClient(clientConn) mounter := fuse.NewMounter(localAddress, apiClient) mountpoint := filepath.Join(tmp, "mnt") if err := os.Mkdir(mountpoint, 0700); err != nil { t.Fatalf("mkdir mountpoint: %v", err) } ready := make(chan bool) wg.Add(1) go func() { defer wg.Done() if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil { t.Errorf("mount and serve: %v", err) } }() <-ready defer func() { if err := mounter.Unmount(mountpoint); err != nil { t.Errorf("unmount: %v", err) } }() const ( repoName = "foo" ) if err := pfsutil.CreateRepo(apiClient, repoName); err != nil { t.Fatalf("CreateRepo: %v", err) } commit, err := pfsutil.StartCommit(apiClient, repoName, "") if err != nil { t.Fatalf("StartCommit: %v", err) } t.Logf("open commit %v", commit.Id) const ( greetingName = "greeting" greeting = "Hello, world\n" greetingPerm = 0644 ) if err := ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.Id, greetingName), []byte(greeting), greetingPerm); err != nil { t.Fatalf("WriteFile: %v", err) } const ( scriptName = "script" script = "#!/bin/sh\necho foo\n" scriptPerm = 0750 ) if err := ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.Id, scriptName), []byte(script), scriptPerm); err != nil { t.Fatalf("WriteFile: %v", err) } if err := pfsutil.FinishCommit(apiClient, repoName, commit.Id); err != nil { t.Fatalf("FinishCommit: %v", err) } if err := fstestutil.CheckDir(filepath.Join(mountpoint, repoName, commit.Id), map[string]fstestutil.FileInfoCheck{ greetingName: func(fi os.FileInfo) error { // TODO respect greetingPerm if g, e := fi.Mode(), os.FileMode(0666); g != e { return fmt.Errorf("wrong mode: %v != %v", g, e) } if g, e := fi.Size(), int64(len(greeting)); g != e { t.Errorf("wrong size: %v != %v", g, e) } // TODO show fileModTime as mtime // if g, e := fi.ModTime().UTC(), fileModTime; g != e { // t.Errorf("wrong mtime: %v != %v", g, e) // } return nil }, scriptName: func(fi os.FileInfo) error { // TODO respect scriptPerm if g, e := fi.Mode(), os.FileMode(0666); g != e { return fmt.Errorf("wrong mode: %v != %v", g, e) } if g, e := fi.Size(), int64(len(script)); g != e { t.Errorf("wrong size: %v != %v", g, e) } // TODO show fileModTime as mtime // if g, e := fi.ModTime().UTC(), fileModTime; g != e { // t.Errorf("wrong mtime: %v != %v", g, e) // } return nil }, }); err != nil { t.Errorf("wrong directory content: %v", err) } }
func TestRootReadDir(t *testing.T) { t.Parallel() // don't leave goroutines running var wg sync.WaitGroup defer wg.Wait() tmp, err := ioutil.TempDir("", "pachyderm-test-") if err != nil { t.Fatalf("tempdir: %v", err) } defer func() { if err := os.RemoveAll(tmp); err != nil { t.Errorf("cannot remove tempdir: %v", err) } }() // closed on successful termination quit := make(chan struct{}) defer close(quit) listener, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("cannot listen: %v", err) } defer func() { _ = listener.Close() }() // TODO try to share more of this setup code with various main // functions localAddress := listener.Addr().String() srv := grpc.NewServer() const ( numShards = 1 ) sharder := shard.NewLocalSharder(localAddress, numShards) hasher := pfs.NewHasher(numShards, 1) router := shard.NewRouter( sharder, grpcutil.NewDialer( grpc.WithInsecure(), ), localAddress, ) blockDir := filepath.Join(tmp, "blocks") blockServer, err := server.NewLocalBlockAPIServer(blockDir) if err != nil { t.Fatalf("NewLocalBlockAPIServer: %v", err) } pfs.RegisterBlockAPIServer(srv, blockServer) driver, err := drive.NewDriver(localAddress) if err != nil { t.Fatalf("NewDriver: %v", err) } apiServer := server.NewAPIServer( hasher, router, ) pfs.RegisterAPIServer(srv, apiServer) internalAPIServer := server.NewInternalAPIServer( hasher, router, driver, ) pfs.RegisterInternalAPIServer(srv, internalAPIServer) wg.Add(1) go func() { defer wg.Done() if err := srv.Serve(listener); err != nil { select { case <-quit: // orderly shutdown return default: t.Errorf("grpc serve: %v", err) } } }() clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure()) if err != nil { t.Fatalf("grpc dial: %v", err) } apiClient := pfs.NewAPIClient(clientConn) mounter := fuse.NewMounter(localAddress, apiClient) mountpoint := filepath.Join(tmp, "mnt") if err := os.Mkdir(mountpoint, 0700); err != nil { t.Fatalf("mkdir mountpoint: %v", err) } ready := make(chan bool) wg.Add(1) go func() { defer wg.Done() if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil { t.Errorf("mount and serve: %v", err) } }() <-ready defer func() { if err := mounter.Unmount(mountpoint); err != nil { t.Errorf("unmount: %v", err) } }() if err := pfsutil.CreateRepo(apiClient, "one"); err != nil { t.Fatalf("CreateRepo: %v", err) } if err := pfsutil.CreateRepo(apiClient, "two"); err != nil { t.Fatalf("CreateRepo: %v", err) } if err := fstestutil.CheckDir(mountpoint, map[string]fstestutil.FileInfoCheck{ "one": func(fi os.FileInfo) error { if g, e := fi.Mode(), os.ModeDir|0555; g != e { return fmt.Errorf("wrong mode: %v != %v", g, e) } // TODO show repoSize in repo stat? if g, e := fi.Size(), int64(0); g != e { t.Errorf("wrong size: %v != %v", g, e) } // TODO show RepoInfo.Created as time // if g, e := fi.ModTime().UTC(), repoModTime; g != e { // t.Errorf("wrong mtime: %v != %v", g, e) // } return nil }, "two": func(fi os.FileInfo) error { if g, e := fi.Mode(), os.ModeDir|0555; g != e { return fmt.Errorf("wrong mode: %v != %v", g, e) } // TODO show repoSize in repo stat? if g, e := fi.Size(), int64(0); g != e { t.Errorf("wrong size: %v != %v", g, e) } // TODO show RepoInfo.Created as time // if g, e := fi.ModTime().UTC(), repoModTime; g != e { // t.Errorf("wrong mtime: %v != %v", g, e) // } return nil }, }); err != nil { t.Errorf("wrong directory content: %v", err) } }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) address := appEnv.PachydermPfsd1Port if address == "" { address = appEnv.Address } else { address = strings.Replace(address, "tcp://", "", -1) } clientConn, err := grpc.Dial(address, grpc.WithInsecure()) if err != nil { return err } apiClient := pfs.NewAPIClient(clientConn) var shard int var modulus int createRepo := &cobra.Command{ Use: "create-repo repo-name", Short: "Create a new repo.", Long: "Create a new repo.", Run: pkgcobra.RunFixedArgs(1, func(args []string) error { return pfsutil.CreateRepo(apiClient, args[0]) }), } inspectRepo := &cobra.Command{ Use: "inspect-repo repo-name", Short: "Return info about a repo.", Long: "Return info about a repo.", Run: pkgcobra.RunFixedArgs(1, func(args []string) error { repoInfo, err := pfsutil.InspectRepo(apiClient, args[0]) if err != nil { return err } if repoInfo == nil { return fmt.Errorf("repo %s not found", args[0]) } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintRepoHeader(writer) pretty.PrintRepoInfo(writer, repoInfo) return writer.Flush() }), } listRepo := &cobra.Command{ Use: "list-repo", Short: "Return all repos.", Long: "Reutrn all repos.", Run: pkgcobra.RunFixedArgs(0, func(args []string) error { repoInfos, err := pfsutil.ListRepo(apiClient) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintRepoHeader(writer) for _, repoInfo := range repoInfos { pretty.PrintRepoInfo(writer, repoInfo) } return writer.Flush() }), } deleteRepo := &cobra.Command{ Use: "delete-repo repo-name", Short: "Delete a repo.", Long: "Delete a repo.", Run: pkgcobra.RunFixedArgs(1, func(args []string) error { return pfsutil.DeleteRepo(apiClient, args[0]) }), } startCommit := &cobra.Command{ Use: "start-commit repo-name parent-commit-id", Short: "Start a new commit.", Long: "Start a new commit with parent-commit-id as the parent.", Run: pkgcobra.RunFixedArgs(2, func(args []string) error { commit, err := pfsutil.StartCommit(apiClient, args[0], args[1]) if err != nil { return err } fmt.Println(commit.Id) return nil }), } finishCommit := &cobra.Command{ Use: "finish-commit repo-name commit-id", Short: "Finish a started commit.", Long: "Finish a started commit. Commit-id must be a writeable commit.", Run: pkgcobra.RunFixedArgs(2, func(args []string) error { return pfsutil.FinishCommit(apiClient, args[0], args[1]) }), } inspectCommit := &cobra.Command{ Use: "inspect-commit repo-name commit-id", Short: "Return info about a commit.", Long: "Return info about a commit.", Run: pkgcobra.RunFixedArgs(2, func(args []string) error { commitInfo, err := pfsutil.InspectCommit(apiClient, args[0], args[1]) if err != nil { return err } if commitInfo == nil { return fmt.Errorf("commit %s not found", args[1]) } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintCommitInfoHeader(writer) pretty.PrintCommitInfo(writer, commitInfo) return writer.Flush() }), } listCommit := &cobra.Command{ Use: "list-commit repo-name", Short: "Return all commits on a repo.", Long: "Return all commits on a repo.", Run: pkgcobra.RunFixedArgs(1, func(args []string) error { commitInfos, err := pfsutil.ListCommit(apiClient, args[0]) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintCommitInfoHeader(writer) for _, commitInfo := range commitInfos { pretty.PrintCommitInfo(writer, commitInfo) } return writer.Flush() }), } deleteCommit := &cobra.Command{ Use: "delete-commit repo-name commit-id", Short: "Delete a commit.", Long: "Delete a commit.", Run: pkgcobra.RunFixedArgs(2, func(args []string) error { return pfsutil.DeleteCommit(apiClient, args[0], args[1]) }), } putBlock := &cobra.Command{ Use: "put-block repo-name commit-id path/to/file", Short: "Put a block from stdin", Long: "Put a block from stdin. Directories must exist. commit-id must be a writeable commit.", Run: pkgcobra.RunFixedArgs(3, func(args []string) error { block, err := pfsutil.PutBlock(apiClient, args[0], args[1], args[2], os.Stdin) if err != nil { return err } fmt.Println(block.Hash) return nil }), } getBlock := &cobra.Command{ Use: "get-block hash", Short: "Return the contents of a block.", Long: "Return the contents of a block.", Run: pkgcobra.RunFixedArgs(1, func(args []string) error { return pfsutil.GetBlock(apiClient, args[0], os.Stdout) }), } inspectBlock := &cobra.Command{ Use: "inspect-block hash", Short: "Return info about a block.", Long: "Return info about a block.", Run: pkgcobra.RunFixedArgs(1, func(args []string) error { blockInfo, err := pfsutil.InspectBlock(apiClient, args[0]) if err != nil { return err } if blockInfo == nil { return fmt.Errorf("block %s not found", args[2]) } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintBlockInfoHeader(writer) pretty.PrintBlockInfo(writer, blockInfo) return writer.Flush() }), } listBlock := &cobra.Command{ Use: "list-block", Short: "Return the blocks in a directory.", Long: "Return the blocks in a directory.", Run: pkgcobra.RunFixedArgs(0, func(args []string) error { blockInfos, err := pfsutil.ListBlock(apiClient, uint64(shard), uint64(modulus)) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintBlockInfoHeader(writer) for _, blockInfo := range blockInfos { pretty.PrintBlockInfo(writer, blockInfo) } return writer.Flush() }), } listBlock.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from") listBlock.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards") mkdir := &cobra.Command{ Use: "mkdir repo-name commit-id path/to/dir", Short: "Make a directory.", Long: "Make a directory. Parent directories need not exist.", Run: pkgcobra.RunFixedArgs(3, func(args []string) error { return pfsutil.MakeDirectory(apiClient, args[0], args[1], args[2]) }), } putFile := &cobra.Command{ Use: "put-file repo-name commit-id path/to/file", Short: "Put a file from stdin", Long: "Put a file from stdin. Directories must exist. commit-id must be a writeable commit.", Run: pkgcobra.RunFixedArgs(3, func(args []string) error { _, err := pfsutil.PutFile(apiClient, args[0], args[1], args[2], 0, os.Stdin) return err }), } getFile := &cobra.Command{ Use: "get-file repo-name commit-id path/to/file", Short: "Return the contents of a file.", Long: "Return the contents of a file.", Run: pkgcobra.RunFixedArgs(3, func(args []string) error { return pfsutil.GetFile(apiClient, args[0], args[1], args[2], 0, math.MaxInt64, os.Stdout) }), } inspectFile := &cobra.Command{ Use: "inspect-file repo-name commit-id path/to/file", Short: "Return info about a file.", Long: "Return info about a file.", Run: pkgcobra.RunFixedArgs(3, func(args []string) error { fileInfo, err := pfsutil.InspectFile(apiClient, args[0], args[1], args[2]) if err != nil { return err } if fileInfo == nil { return fmt.Errorf("file %s not found", args[2]) } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintFileInfoHeader(writer) pretty.PrintFileInfo(writer, fileInfo) return writer.Flush() }), } listFile := &cobra.Command{ Use: "list-file repo-name commit-id path/to/dir", Short: "Return the files in a directory.", Long: "Return the files in a directory.", Run: pkgcobra.RunFixedArgs(3, func(args []string) error { fileInfos, err := pfsutil.ListFile(apiClient, args[0], args[1], args[2], uint64(shard), uint64(modulus)) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintFileInfoHeader(writer) for _, fileInfo := range fileInfos { pretty.PrintFileInfo(writer, fileInfo) } return writer.Flush() }), } listFile.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from") listFile.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards") deleteFile := &cobra.Command{ Use: "delete-file repo-name commit-id path/to/file", Short: "Delete a file.", Long: "Delete a file.", Run: pkgcobra.RunFixedArgs(2, func(args []string) error { return pfsutil.DeleteFile(apiClient, args[0], args[1], args[2]) }), } listChange := &cobra.Command{ Use: "list-change repo-name commit-id path/to/dir", Short: "Return the changes in a directory.", Long: "Return the changes in a directory.", Run: pkgcobra.RunFixedArgs(3, func(args []string) error { changeInfos, err := pfsutil.ListChange(apiClient, args[0], args[1], args[2], uint64(shard), uint64(modulus)) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintChangeHeader(writer) for _, changeInfo := range changeInfos { pretty.PrintChange(writer, changeInfo) } return writer.Flush() }), } listChange.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from") listChange.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards") inspectServer := &cobra.Command{ Use: "inspect-server server-id", Short: "Inspect a server.", Long: "Inspect a server.", Run: pkgcobra.RunFixedArgs(1, func(args []string) error { serverInfo, err := pfsutil.InspectServer(apiClient, args[0]) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintServerInfoHeader(writer) pretty.PrintServerInfo(writer, serverInfo) return writer.Flush() }), } listServer := &cobra.Command{ Use: "list-server", Short: "Return all servers in the cluster.", Long: "Return all servers in the cluster.", Run: pkgcobra.RunFixedArgs(0, func(args []string) error { serverInfos, err := pfsutil.ListServer(apiClient) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintServerInfoHeader(writer) for _, serverInfo := range serverInfos { pretty.PrintServerInfo(writer, serverInfo) } return writer.Flush() }), } mount := &cobra.Command{ Use: "mount [mountpoint]", Short: "Mount pfs locally.", Long: "Mount pfs locally.", Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 0, Max: 1}, func(args []string) error { mountPoint := "/pfs" if len(args) > 0 { mountPoint = args[0] } mounter := fuse.NewMounter(apiClient) return mounter.Mount(address, mountPoint, uint64(shard), uint64(modulus)) }), } mount.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from") mount.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards") rootCmd := &cobra.Command{ Use: "pfs", Long: `Access the PFS API. Note that this CLI is experimental and does not even check for common errors. The environment variable PFS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:650.`, } rootCmd.AddCommand(protoclient.NewVersionCommand(clientConn, pachyderm.Version, nil)) rootCmd.AddCommand(createRepo) rootCmd.AddCommand(inspectRepo) rootCmd.AddCommand(listRepo) rootCmd.AddCommand(deleteRepo) rootCmd.AddCommand(startCommit) rootCmd.AddCommand(finishCommit) rootCmd.AddCommand(inspectCommit) rootCmd.AddCommand(listCommit) rootCmd.AddCommand(deleteCommit) rootCmd.AddCommand(mkdir) rootCmd.AddCommand(putBlock) rootCmd.AddCommand(getBlock) rootCmd.AddCommand(inspectBlock) rootCmd.AddCommand(listBlock) rootCmd.AddCommand(putFile) rootCmd.AddCommand(getFile) rootCmd.AddCommand(inspectFile) rootCmd.AddCommand(listFile) rootCmd.AddCommand(deleteFile) rootCmd.AddCommand(listChange) rootCmd.AddCommand(inspectServer) rootCmd.AddCommand(listServer) rootCmd.AddCommand(mount) return rootCmd.Execute() }
func NewAPIClient(cc *grpc.ClientConn) *APIClient { return &APIClient{ pfs.NewAPIClient(cc), pps.NewAPIClient(cc), } }