func main() { var commits int var lines int cmd := &cobra.Command{ Use: os.Args[0], Short: "Generate pfs traffic.", Long: "Generate pfs traffic.", Run: pkgcobra.RunFixedArgs(0, func(args []string) error { rand.Seed(time.Now().UnixNano()) client, err := client.NewFromAddress("0.0.0.0:30650") if err != nil { return err } for i := 0; i < commits; i++ { if _, err := client.StartCommit("data", "", "master"); err != nil { return err } if _, err := client.PutFile("data", "master", "sales", &reader{lines}); err != nil { return err } if err := client.FinishCommit("data", "master"); err != nil { return err } } return nil }), } cmd.Flags().IntVarP(&commits, "commits", "c", 100, "commits to write") cmd.Flags().IntVarP(&lines, "lines", "l", 100, "lines to write for each commit") cmd.Execute() }
func Example_pps() { c, err := client.NewFromAddress("0.0.0.0:30650") if err != nil { return // handle error } // we assume there's already a repo called "repo" // and that it already has some data in it // take a look at src/client/pfs_test.go for an example of how to get there. // Create a map pipeline if err := c.CreatePipeline( "map", // the name of the pipeline "pachyderm/test_image", // your docker image []string{"map"}, // the command run in your docker image nil, // no stdin 0, // let pachyderm decide the parallelism []*pps.PipelineInput{ // map over "repo" client.NewPipelineInput("repo", client.MapMethod), }, ); err != nil { return // handle error } if err := c.CreatePipeline( "reduce", // the name of the pipeline "pachyderm/test_image", // your docker image []string{"reduce"}, // the command run in your docker image nil, // no stdin 0, // let pachyderm decide the parallelism []*pps.PipelineInput{ // reduce over "map" client.NewPipelineInput("map", client.ReduceMethod), }, ); err != nil { return // handle error } commits, err := c.ListCommit( // List commits that are... []string{"reduce"}, // from the "reduce" repo (which the "reduce" pipeline outputs) nil, // starting at the beginning of time client.CommitTypeRead, // are readable true, // block until commits are available false, // ignore cancelled commits nil, // have no provenance ) if err != nil { return // handle error } for _, commitInfo := range commits { // Read output from the pipeline var buffer bytes.Buffer if err := c.GetFile("reduce", commitInfo.Commit.ID, "file", 0, 0, "", nil, &buffer); err != nil { return //handle error } } }
// getDataSet gets a previously stored dataset from pachyderm data versioning. func getDataSet(dataSet, branch, repoName string) (bytes.Buffer, error) { // Open a connection to pachyderm running on localhost. c, err := client.NewFromAddress("localhost:30650") if err != nil { return bytes.Buffer{}, errors.Wrap(err, "Could not connect to Pachyderm") } // Read the latest commit of filename to the given repoName. var buffer bytes.Buffer if err := c.GetFile(repoName, branch, dataSet, 0, 0, "", nil, &buffer); err != nil { return buffer, errors.Wrap(err, "Could not retrieve pachyderm file") } return buffer, nil }
// pushToPach reads in the given file, create a repo in PFS for the file, opens // a commit, pushes the file in the commit, and finishes the commit. func pushToPach(path, dataName, repoName string) error { // Read the contents of the given file. csvfile, err := ioutil.ReadFile(path) if err != nil { errors.Wrap(err, "Could not read input file") } // Open a connection to pachyderm running on localhost. c, err := client.NewFromAddress("localhost:30650") if err != nil { errors.Wrap(err, "Could not connect to Pachyderm") } // Create a repo. if err := c.CreateRepo(repoName); err != nil { errors.Wrap(err, "Could not create pachyderm repo") } // Start a commit on the "master" branch of the repo. if _, err = c.StartCommit(repoName, "", "master"); err != nil { errors.Wrap(err, "Could not start pachyderm repo commit") } // Put the given file into the repo after the commit is started. r := bytes.NewReader(csvfile) if _, err := c.PutFile(repoName, "master", dataName, r); err != nil { errors.Wrap(err, "Could not put file into pachyderm repo") } // Finish the commit. if err := c.FinishCommit(repoName, "master"); err != nil { errors.Wrap(err, "Could not finish Pachyderm commit") } return nil }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) rootCmd := &cobra.Command{ Use: os.Args[0] + " job-id", Short: `Pachyderm job-shim, coordinates with ppsd to create an output commit and run user work.`, Long: `Pachyderm job-shim, coordinates with ppsd to create an output commit and run user work.`, Run: func(_ *cobra.Command, args []string) { ppsClient, err := ppsserver.NewInternalJobAPIClientFromAddress(fmt.Sprintf("%v:650", appEnv.PachydermAddress)) if err != nil { errorAndExit(err.Error()) } response, err := ppsClient.StartJob( context.Background(), &ppsserver.StartJobRequest{ Job: &ppsclient.Job{ ID: args[0], }}) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err.Error()) os.Exit(0) } if response.Transform.Debug { lion.SetLevel(lion.LevelDebug) } // We want to make sure that we only send FinishJob once. // The most bulletproof way would be to check that on server side, // but this is easier. var finished bool // Make sure that we call FinishJob even if something caused a panic defer func() { if r := recover(); r != nil && !finished { fmt.Println("job shim crashed; this is like a bug in pachyderm") if _, err := ppsClient.FinishJob( context.Background(), &ppsserver.FinishJobRequest{ Job: &ppsclient.Job{ ID: args[0], }, Success: false, }, ); err != nil { errorAndExit(err.Error()) } } }() c, err := client.NewFromAddress(fmt.Sprintf("%v:650", appEnv.PachydermAddress)) if err != nil { errorAndExit(err.Error()) } mounter := fuse.NewMounter(appEnv.PachydermAddress, c.PfsAPIClient) ready := make(chan bool) go func() { if err := mounter.MountAndCreate( "/pfs", nil, response.CommitMounts, ready, response.Transform.Debug, ); err != nil { errorAndExit(err.Error()) } }() <-ready defer func() { if err := mounter.Unmount("/pfs"); err != nil { errorAndExit(err.Error()) } }() var readers []io.Reader for _, line := range response.Transform.Stdin { readers = append(readers, strings.NewReader(line+"\n")) } if len(response.Transform.Cmd) == 0 { fmt.Println("unable to run; a cmd needs to be provided") if _, err := ppsClient.FinishJob( context.Background(), &ppsserver.FinishJobRequest{ Job: client.NewJob(args[0]), Success: false, }, ); err != nil { errorAndExit(err.Error()) } finished = true return } cmd := exec.Command(response.Transform.Cmd[0], response.Transform.Cmd[1:]...) cmd.Stdin = io.MultiReader(readers...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr success := true if err := cmd.Run(); err != nil { success = false if exiterr, ok := err.(*exec.ExitError); ok { if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { for _, returnCode := range response.Transform.AcceptReturnCode { if int(returnCode) == status.ExitStatus() { success = true } } } } if !success { fmt.Fprintf(os.Stderr, "Error from exec: %s\n", err.Error()) } } if _, err := ppsClient.FinishJob( context.Background(), &ppsserver.FinishJobRequest{ Job: client.NewJob(args[0]), Success: success, }, ); err != nil { errorAndExit(err.Error()) } finished = true }, } return rootCmd.Execute() }
func Example_pfs() { c, err := client.NewFromAddress("0.0.0.0:30650") if err != nil { return // handle error } // Create a repo called "repo" if err := c.CreateRepo("repo"); err != nil { return // handle error } // Start a commit in our new repo on the "master" branch commit1, err := c.StartCommit("repo", "", "master") if err != nil { return // handle error } // Put a file called "file" in the newly created commit with the content "foo\n". if _, err := c.PutFile("repo", "master", "file", strings.NewReader("foo\n")); err != nil { return // handle error } // Finish the commit. if err := c.FinishCommit("repo", "master"); err != nil { return //handle error } // Read what we wrote. var buffer bytes.Buffer if err := c.GetFile("repo", "master", "file", 0, 0, "", nil, &buffer); err != nil { return //handle error } // buffer now contains "foo\n" // Start another commit with the previous commit as the parent. if _, err := c.StartCommit("repo", "", "master"); err != nil { return //handle error } // Extend "file" in the newly created commit with the content "bar\n". if _, err := c.PutFile("repo", "master", "file", strings.NewReader("bar\n")); err != nil { return // handle error } // Finish the commit. if err := c.FinishCommit("repo", "master"); err != nil { return //handle error } // Read what we wrote. buffer.Reset() if err := c.GetFile("repo", "master", "file", 0, 0, "", nil, &buffer); err != nil { return //handle error } // buffer now contains "foo\nbar\n" // We can still read the old version of the file though: buffer.Reset() if err := c.GetFile("repo", commit1.ID, "file", 0, 0, "", nil, &buffer); err != nil { return //handle error } // buffer now contains "foo\n" // We can also see the Diff between the most recent commit and the first one: buffer.Reset() if err := c.GetFile("repo", "master", "file", 0, 0, commit1.ID, nil, &buffer); err != nil { return //handle error } }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) etcdClient := getEtcdClient(appEnv) if appEnv.Init { if err := setClusterID(etcdClient); err != nil { return fmt.Errorf("error connecting to etcd, if this error persists it likely indicates that kubernetes services are not working correctly. See https://github.com/pachyderm/pachyderm/blob/master/SETUP.md#pachd-or-pachd-init-crash-loop-with-error-connecting-to-etcd for more info") } if err := persist_server.InitDBs(fmt.Sprintf("%s:28015", appEnv.DatabaseAddress), appEnv.DatabaseName); err != nil && !isDBCreated(err) { return err } return nil } if readinessCheck { //c, err := client.NewInCluster() c, err := client.NewFromAddress("127.0.0.1:650") if err != nil { return err } _, err = c.ListRepo(nil) if err != nil { return err } os.Exit(0) return nil } clusterID, err := getClusterID(etcdClient) if err != nil { return err } kubeClient, err := getKubeClient(appEnv) if err != nil { return err } if appEnv.Metrics { go metrics.ReportMetrics(clusterID, kubeClient) } rethinkAPIServer, err := getRethinkAPIServer(appEnv) if err != nil { return err } address, err := netutil.ExternalIP() if err != nil { return err } address = fmt.Sprintf("%s:%d", address, appEnv.Port) sharder := shard.NewSharder( etcdClient, appEnv.NumShards, appEnv.Namespace, ) go func() { if err := sharder.AssignRoles(address, nil); err != nil { protolion.Printf("Error from sharder.AssignRoles: %s", err.Error()) } }() driver, err := drive.NewDriver(address) if err != nil { return err } apiServer := pfs_server.NewAPIServer( pfsmodel.NewHasher( appEnv.NumShards, 1, ), shard.NewRouter( sharder, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ), ) go func() { if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{apiServer}); err != nil { protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error()) } }() internalAPIServer := pfs_server.NewInternalAPIServer( pfsmodel.NewHasher( appEnv.NumShards, 1, ), shard.NewRouter( sharder, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ), driver, ) ppsAPIServer := pps_server.NewAPIServer( ppsserver.NewHasher(appEnv.NumShards, appEnv.NumShards), address, kubeClient, getNamespace(), ) go func() { if err := sharder.Register(nil, address, []shard.Server{internalAPIServer, ppsAPIServer}); err != nil { protolion.Printf("Error from sharder.Register %s", err.Error()) } }() blockAPIServer, err := pfs_server.NewBlockAPIServer(appEnv.StorageRoot, appEnv.StorageBackend) if err != nil { return err } return protoserver.Serve( func(s *grpc.Server) { pfsclient.RegisterAPIServer(s, apiServer) pfsclient.RegisterInternalAPIServer(s, internalAPIServer) pfsclient.RegisterBlockAPIServer(s, blockAPIServer) ppsclient.RegisterAPIServer(s, ppsAPIServer) ppsserver.RegisterInternalJobAPIServer(s, ppsAPIServer) persist.RegisterAPIServer(s, rethinkAPIServer) }, protoserver.ServeOptions{ Version: version.Version, }, protoserver.ServeEnv{ GRPCPort: appEnv.Port, }, ) }
func Cmds(address string) ([]*cobra.Command, error) { marshaller := &jsonpb.Marshaler{Indent: " "} job := &cobra.Command{ Use: "job", Short: "Docs for jobs.", Long: `Jobs are the basic unit of computation in Pachyderm. Jobs run a containerized workload over a set of finished input commits. Creating a job will also create a new repo and a commit in that repo which contains the output of the job. Unless the job is created with another job as a parent. If the job is created with a parent it will use the same repo as its parent job and the commit it creates will use the parent job's commit as a parent. If the job fails the commit it creates will not be finished. The increase the throughput of a job increase the Shard paremeter. `, Run: pkgcmd.RunFixedArgs(0, func(args []string) error { return nil }), } exampleCreateJobRequest, err := marshaller.MarshalToString(example.CreateJobRequest()) if err != nil { return nil, err } exampleRunPipelineSpec, err := marshaller.MarshalToString(example.RunPipelineSpec()) if err != nil { return nil, err } pipelineSpec := string(pachyderm.MustAsset("doc/pipeline_spec.md")) var jobPath string createJob := &cobra.Command{ Use: "create-job -f job.json", Short: "Create a new job. Returns the id of the created job.", Long: fmt.Sprintf("Create a new job from a spec, the spec looks like this\n%s", exampleCreateJobRequest), Run: func(cmd *cobra.Command, args []string) { client, err := pach.NewFromAddress(address) if err != nil { pkgcmd.ErrorAndExit("Error connecting to pps: %v", err) } var buf bytes.Buffer var jobReader io.Reader if jobPath == "-" { jobReader = io.TeeReader(os.Stdin, &buf) fmt.Print("Reading from stdin.\n") } else { jobFile, err := os.Open(jobPath) if err != nil { pkgcmd.ErrorAndExit("Error opening %s: %v", jobPath, err) } defer func() { if err := jobFile.Close(); err != nil { pkgcmd.ErrorAndExit("Error closing%s: %v", jobPath, err) } }() jobReader = io.TeeReader(jobFile, &buf) } var request ppsclient.CreateJobRequest decoder := json.NewDecoder(jobReader) s, err := replaceMethodAliases(decoder) if err != nil { err = describeSyntaxError(err, buf) pkgcmd.ErrorAndExit("Error parsing job spec: %v", err) } if err := jsonpb.UnmarshalString(s, &request); err != nil { pkgcmd.ErrorAndExit("Error reading from stdin: %v", err) } job, err := client.PpsAPIClient.CreateJob( context.Background(), &request, ) if err != nil { pkgcmd.ErrorAndExit("Error from CreateJob: %v", err) } fmt.Println(job.ID) }, } createJob.Flags().StringVarP(&jobPath, "file", "f", "-", "The file containing the job, - reads from stdin.") var block bool inspectJob := &cobra.Command{ Use: "inspect-job job-id", Short: "Return info about a job.", Long: "Return info about a job.", Run: pkgcmd.RunFixedArgs(1, func(args []string) error { client, err := pach.NewFromAddress(address) if err != nil { return err } jobInfo, err := client.InspectJob(args[0], block) if err != nil { pkgcmd.ErrorAndExit("Error from InspectJob: %s", err.Error()) } if jobInfo == nil { pkgcmd.ErrorAndExit("Job %s not found.", args[0]) } pretty.PrintDetailedJobInfo(jobInfo) return nil }), } inspectJob.Flags().BoolVarP(&block, "block", "b", false, "block until the job has either succeeded or failed") var pipelineName string listJob := &cobra.Command{ Use: "list-job [-p pipeline-name] [commits]", Short: "Return info about jobs.", Long: `Return info about jobs. Examples: # return all jobs $ pachctl list-job # return all jobs in pipeline foo $ pachctl list-job -p foo # return all jobs whose input commits include foo/abc123 and bar/def456 $ pachctl list-job foo/abc123 bar/def456 # return all jobs in pipeline foo and whose input commits include bar/def456 $ pachctl list-job -p foo bar/def456 `, Run: func(cmd *cobra.Command, args []string) { client, err := pach.NewFromAddress(address) if err != nil { pkgcmd.ErrorAndExit("Error from InspectJob: %v", err) } commits, err := pkgcmd.ParseCommits(args) if err != nil { cmd.Usage() pkgcmd.ErrorAndExit("Error from InspectJob: %v", err) } jobInfos, err := client.ListJob(pipelineName, commits) if err != nil { pkgcmd.ErrorAndExit("Error from InspectJob: %v", err) } // Display newest jobs first sort.Sort(sort.Reverse(ByCreationTime(jobInfos))) writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintJobHeader(writer) for _, jobInfo := range jobInfos { pretty.PrintJobInfo(writer, jobInfo) } if err := writer.Flush(); err != nil { pkgcmd.ErrorAndExit("Error from InspectJob: %v", err) } }, } listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.") getLogs := &cobra.Command{ Use: "get-logs job-id", Short: "Return logs from a job.", Long: "Return logs from a job.", Run: pkgcmd.RunFixedArgs(1, func(args []string) error { client, err := pach.NewFromAddress(address) if err != nil { return err } return client.GetLogs(args[0], os.Stdout) }), } pipeline := &cobra.Command{ Use: "pipeline", Short: "Docs for pipelines.", Long: `Pipelines are a powerful abstraction for automating jobs. Pipelines take a set of repos as inputs, rather than the set of commits that jobs take. Pipelines then subscribe to commits on those repos and launches a job to process each incoming commit. Creating a pipeline will also create a repo of the same name. All jobs created by a pipeline will create commits in the pipeline's repo. `, Run: pkgcmd.RunFixedArgs(0, func(args []string) error { return nil }), } var pipelinePath string if err != nil { return nil, err } createPipeline := &cobra.Command{ Use: "create-pipeline -f pipeline.json", Short: "Create a new pipeline.", Long: fmt.Sprintf("Create a new pipeline from a spec\n\n%s", pipelineSpec), Run: func(cmd *cobra.Command, args []string) { client, err := pach.NewFromAddress(address) if err != nil { pkgcmd.ErrorAndExit("Error connecting to pps: %s", err.Error()) } var buf bytes.Buffer var pipelineReader io.Reader if pipelinePath == "-" { pipelineReader = io.TeeReader(os.Stdin, &buf) fmt.Print("Reading from stdin.\n") } else { rawBytes, err := ioutil.ReadFile(pipelinePath) if err != nil { pkgcmd.ErrorAndExit("Error reading file %s", pipelinePath) } pipelineReader = io.TeeReader(strings.NewReader(string(rawBytes)), &buf) } var request ppsclient.CreatePipelineRequest decoder := json.NewDecoder(pipelineReader) for { s, err := replaceMethodAliases(decoder) if err != nil { if err == io.EOF { break } err = describeSyntaxError(err, buf) pkgcmd.ErrorAndExit("Error parsing pipeline spec: %v", err) } if err := jsonpb.UnmarshalString(s, &request); err != nil { pkgcmd.ErrorAndExit("Error marshalling JSON into protobuf: %v", err) } if _, err := client.PpsAPIClient.CreatePipeline( context.Background(), &request, ); err != nil { pkgcmd.ErrorAndExit("Error from CreatePipeline: %v", err) } } }, } createPipeline.Flags().StringVarP(&pipelinePath, "file", "f", "-", "The file containing the pipeline, - reads from stdin.") inspectPipeline := &cobra.Command{ Use: "inspect-pipeline pipeline-name", Short: "Return info about a pipeline.", Long: "Return info about a pipeline.", Run: pkgcmd.RunFixedArgs(1, func(args []string) error { client, err := pach.NewFromAddress(address) if err != nil { return err } pipelineInfo, err := client.InspectPipeline(args[0]) if err != nil { pkgcmd.ErrorAndExit("Error from InspectPipeline: %s", err.Error()) } if pipelineInfo == nil { pkgcmd.ErrorAndExit("Pipeline %s not found.", args[0]) } pretty.PrintDetailedPipelineInfo(pipelineInfo) return nil }), } listPipeline := &cobra.Command{ Use: "list-pipeline", Short: "Return info about all pipelines.", Long: "Return info about all pipelines.", Run: pkgcmd.RunFixedArgs(0, func(args []string) error { client, err := pach.NewFromAddress(address) if err != nil { return err } pipelineInfos, err := client.ListPipeline() if err != nil { pkgcmd.ErrorAndExit("Error from ListPipeline: %s", err.Error()) } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintPipelineHeader(writer) for _, pipelineInfo := range pipelineInfos { pretty.PrintPipelineInfo(writer, pipelineInfo) } return writer.Flush() }), } deletePipeline := &cobra.Command{ Use: "delete-pipeline pipeline-name", Short: "Delete a pipeline.", Long: "Delete a pipeline.", Run: pkgcmd.RunFixedArgs(1, func(args []string) error { client, err := pach.NewFromAddress(address) if err != nil { return err } if err := client.DeletePipeline(args[0]); err != nil { pkgcmd.ErrorAndExit("Error from DeletePipeline: %s", err.Error()) } return nil }), } var specPath string runPipeline := &cobra.Command{ Use: "run-pipeline pipeline-name [-f job.json]", Short: "Run a pipeline once.", Long: fmt.Sprintf("Run a pipeline once, optionally overriding some pipeline options by providing a spec. The spec looks like this:\n%s", exampleRunPipelineSpec), Run: pkgcmd.RunFixedArgs(1, func(args []string) error { client, err := pach.NewFromAddress(address) if err != nil { return err } request := &ppsclient.CreateJobRequest{ Pipeline: &ppsclient.Pipeline{ Name: args[0], }, Force: true, } var buf bytes.Buffer var specReader io.Reader if specPath == "-" { specReader = io.TeeReader(os.Stdin, &buf) fmt.Print("Reading from stdin.\n") } else if specPath != "" { specFile, err := os.Open(specPath) if err != nil { pkgcmd.ErrorAndExit("Error opening %s: %v", specPath, err) } defer func() { if err := specFile.Close(); err != nil { pkgcmd.ErrorAndExit("Error closing%s: %v", specPath, err) } }() specReader = io.TeeReader(specFile, &buf) decoder := json.NewDecoder(specReader) s, err := replaceMethodAliases(decoder) if err != nil { err = describeSyntaxError(err, buf) pkgcmd.ErrorAndExit("Error parsing pipeline spec: %v", err) } if err := jsonpb.UnmarshalString(s, request); err != nil { pkgcmd.ErrorAndExit("Error reading from stdin: %v", err) } } job, err := client.PpsAPIClient.CreateJob( context.Background(), request, ) if err != nil { pkgcmd.ErrorAndExit("Error from RunPipeline: %v", err) } fmt.Println(job.ID) return nil }), } runPipeline.Flags().StringVarP(&specPath, "file", "f", "", "The file containing the run-pipeline spec, - reads from stdin.") var result []*cobra.Command result = append(result, job) result = append(result, createJob) result = append(result, inspectJob) result = append(result, getLogs) result = append(result, listJob) result = append(result, pipeline) result = append(result, createPipeline) result = append(result, inspectPipeline) result = append(result, listPipeline) result = append(result, deletePipeline) result = append(result, runPipeline) return result, nil }
func Cmds(address string) []*cobra.Command { var fileNumber int var fileModulus int var blockNumber int var blockModulus int shard := func() *pfsclient.Shard { return &pfsclient.Shard{ FileNumber: uint64(fileNumber), FileModulus: uint64(fileModulus), BlockNumber: uint64(blockNumber), BlockModulus: uint64(blockModulus), } } addShardFlags := func(cmd *cobra.Command) { cmd.Flags().IntVarP(&fileNumber, "file-shard", "s", 0, "file shard to read") cmd.Flags().IntVarP(&fileModulus, "file-modulus", "m", 1, "modulus of file shard") cmd.Flags().IntVarP(&blockNumber, "block-shard", "b", 0, "block shard to read") cmd.Flags().IntVarP(&blockModulus, "block-modulus", "n", 1, "modulus of block shard") } repo := &cobra.Command{ Use: "repo", Short: "Docs for repos.", Long: `Repos, short for repository, are the top level data object in Pachyderm. Repos are created with create-repo.`, Run: cmd.RunFixedArgs(0, func(args []string) error { return nil }), } createRepo := &cobra.Command{ Use: "create-repo repo-name", Short: "Create a new repo.", Long: "Create a new repo.", Run: cmd.RunFixedArgs(1, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } return client.CreateRepo(args[0]) }), } inspectRepo := &cobra.Command{ Use: "inspect-repo repo-name", Short: "Return info about a repo.", Long: "Return info about a repo.", Run: cmd.RunFixedArgs(1, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } repoInfo, err := client.InspectRepo(args[0]) if err != nil { return err } if repoInfo == nil { return fmt.Errorf("repo %s not found", args[0]) } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintRepoHeader(writer) pretty.PrintRepoInfo(writer, repoInfo) return writer.Flush() }), } var listRepoProvenance cmd.RepeatedStringArg listRepo := &cobra.Command{ Use: "list-repo", Short: "Return all repos.", Long: "Reutrn all repos.", Run: cmd.RunFixedArgs(0, func(args []string) error { c, err := client.NewFromAddress(address) if err != nil { return err } repoInfos, err := c.ListRepo(listRepoProvenance) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintRepoHeader(writer) for _, repoInfo := range repoInfos { pretty.PrintRepoInfo(writer, repoInfo) } return writer.Flush() }), } listRepo.Flags().VarP(&listRepoProvenance, "provenance", "p", "list only repos with the specified repos provenance") deleteRepo := &cobra.Command{ Use: "delete-repo repo-name", Short: "Delete a repo.", Long: "Delete a repo.", Run: cmd.RunFixedArgs(1, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } return client.DeleteRepo(args[0]) }), } commit := &cobra.Command{ Use: "commit", Short: "Docs for commits.", Long: `Commits are atomic transactions on the content of a repo. Creating a commit is a multistep process: - start a new commit with start-commit - write files to it through fuse or with put-file - finish the new commit with finish-commit Commits that have been started but not finished are NOT durable storage. Commits become reliable (and immutable) when they are finished. Commits can be created with another commit as a parent. This layers the data in the commit over the data in the parent.`, Run: cmd.RunFixedArgs(0, func(args []string) error { return nil }), } var parentCommitID string startCommit := &cobra.Command{ Use: "start-commit repo-name [branch]", Short: "Start a new commit.", Long: "Start a new commit with parent-commit-id as the parent.", Run: cmd.RunBoundedArgs(1, 2, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } branch := "" if len(args) == 2 { branch = args[1] } commit, err := client.StartCommit(args[0], parentCommitID, branch) if err != nil { return err } fmt.Println(commit.ID) return nil }), } startCommit.Flags().StringVarP(&parentCommitID, "parent", "p", "", "parent id") var cancel bool finishCommit := &cobra.Command{ Use: "finish-commit repo-name commit-id", Short: "Finish a started commit.", Long: "Finish a started commit. Commit-id must be a writeable commit.", Run: cmd.RunFixedArgs(2, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } if cancel { return client.CancelCommit(args[0], args[1]) } return client.FinishCommit(args[0], args[1]) }), } finishCommit.Flags().BoolVarP(&cancel, "cancel", "c", false, "cancel the commit") inspectCommit := &cobra.Command{ Use: "inspect-commit repo-name commit-id", Short: "Return info about a commit.", Long: "Return info about a commit.", Run: cmd.RunFixedArgs(2, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } commitInfo, err := client.InspectCommit(args[0], args[1]) if err != nil { return err } if commitInfo == nil { return fmt.Errorf("commit %s not found", args[1]) } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintCommitInfoHeader(writer) pretty.PrintCommitInfo(writer, commitInfo) return writer.Flush() }), } var all bool var block bool var listCommitProvenance cmd.RepeatedStringArg listCommit := &cobra.Command{ Use: "list-commit repo-name", Short: "Return all commits on a set of repos", Long: `Return all commits on a set of repos. Examples: # return commits in repo "foo" and repo "bar" $ pachctl list-commit foo bar # return commits in repo "foo" since commit abc123 and those in repo "bar" since commit def456 $ pachctl list-commit foo/abc123 bar/def456 # return commits in repo "foo" that have commits # "bar/abc123" and "baz/def456" as provenance $ pachctl list-commit foo -p bar/abc123 -p baz/def456 `, Run: pkgcobra.Run(func(args []string) error { commits, err := cmd.ParseCommits(args) if err != nil { return err } var repos []string var fromCommits []string for _, commit := range commits { repos = append(repos, commit.Repo.Name) fromCommits = append(fromCommits, commit.ID) } c, err := client.NewFromAddress(address) if err != nil { return err } provenance, err := cmd.ParseCommits(listCommitProvenance) if err != nil { return err } commitInfos, err := c.ListCommit(repos, fromCommits, client.CommitTypeNone, block, all, provenance) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintCommitInfoHeader(writer) for _, commitInfo := range commitInfos { pretty.PrintCommitInfo(writer, commitInfo) } return writer.Flush() }), } listCommit.Flags().BoolVarP(&all, "all", "a", false, "list all commits including cancelled commits") listCommit.Flags().BoolVarP(&block, "block", "b", false, "block until there are new commits since the from commits") listCommit.Flags().VarP(&listCommitProvenance, "provenance", "p", "list only commits with the specified `commit`s provenance, commits are specified as RepoName/CommitID") var repos cmd.RepeatedStringArg flushCommit := &cobra.Command{ Use: "flush-commit commit [commit ...]", Short: "Wait for all commits caused by the specified commits to finish and return them.", Long: `Wait for all commits caused by the specified commits to finish and return them. Examples: # return commits caused by foo/abc123 and bar/def456 $ pachctl flush-commit foo/abc123 bar/def456 # return commits caused by foo/abc123 leading to repos bar and baz $ pachctl flush-commit foo/abc123 -r bar -r baz `, Run: pkgcobra.Run(func(args []string) error { commits, err := cmd.ParseCommits(args) if err != nil { return err } c, err := client.NewFromAddress(address) if err != nil { return err } var toRepos []*pfsclient.Repo for _, repoName := range repos { toRepos = append(toRepos, client.NewRepo(repoName)) } commitInfos, err := c.FlushCommit(commits, toRepos) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintCommitInfoHeader(writer) for _, commitInfo := range commitInfos { pretty.PrintCommitInfo(writer, commitInfo) } return writer.Flush() }), } flushCommit.Flags().VarP(&repos, "repos", "r", "Wait only for commits leading to a specific set of repos") listBranch := &cobra.Command{ Use: "list-branch repo-name", Short: "Return all branches on a repo.", Long: "Return all branches on a repo.", Run: cmd.RunFixedArgs(1, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } commitInfos, err := client.ListBranch(args[0]) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintCommitInfoHeader(writer) for _, commitInfo := range commitInfos { pretty.PrintCommitInfo(writer, commitInfo) } return writer.Flush() }), } file := &cobra.Command{ Use: "file", Short: "Docs for files.", Long: `Files are the lowest level data object in Pachyderm. Files can be written to started (but not finished) commits with put-file. Files can be read from finished commits with get-file.`, Run: cmd.RunFixedArgs(0, func(args []string) error { return nil }), } putFile := &cobra.Command{ Use: "put-file repo-name commit-id path/to/file", Short: "Put a file from stdin", Long: "Put a file from stdin. commit-id must be a writeable commit.", Run: cmd.RunFixedArgs(3, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } _, err = client.PutFile(args[0], args[1], args[2], os.Stdin) return err }), } var fromCommitID string var unsafe bool getFile := &cobra.Command{ Use: "get-file repo-name commit-id path/to/file", Short: "Return the contents of a file.", Long: "Return the contents of a file.", Run: cmd.RunFixedArgs(3, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } return client.GetFile(args[0], args[1], args[2], 0, 0, fromCommitID, shard(), os.Stdout) }), } addShardFlags(getFile) getFile.Flags().StringVarP(&fromCommitID, "from", "f", "", "only consider data written since this commit") getFile.Flags().BoolVar(&unsafe, "unsafe", false, "use this flag if you need to read data written in the current commit; this operation will race with concurrent writes") inspectFile := &cobra.Command{ Use: "inspect-file repo-name commit-id path/to/file", Short: "Return info about a file.", Long: "Return info about a file.", Run: cmd.RunFixedArgs(3, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } fileInfo, err := client.InspectFile(args[0], args[1], args[2], "", shard()) if err != nil { return err } if fileInfo == nil { return fmt.Errorf("file %s not found", args[2]) } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintFileInfoHeader(writer) pretty.PrintFileInfo(writer, fileInfo) return writer.Flush() }), } addShardFlags(inspectFile) inspectFile.Flags().BoolVar(&unsafe, "unsafe", false, "use this flag if you need to inspect files written in the current commit; this operation will race with concurrent writes") listFile := &cobra.Command{ Use: "list-file repo-name commit-id path/to/dir", Short: "Return the files in a directory.", Long: "Return the files in a directory.", Run: cmd.RunBoundedArgs(2, 3, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } var path string if len(args) == 3 { path = args[2] } fileInfos, err := client.ListFile(args[0], args[1], path, fromCommitID, shard(), true) if err != nil { return err } writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) pretty.PrintFileInfoHeader(writer) for _, fileInfo := range fileInfos { pretty.PrintFileInfo(writer, fileInfo) } return writer.Flush() }), } addShardFlags(listFile) listFile.Flags().StringVarP(&fromCommitID, "from", "f", "", "only list files that are written since this commit") listFile.Flags().BoolVar(&unsafe, "unsafe", false, "use this flag if you need to list files written in the current commit; this operation will race with concurrent writes") deleteFile := &cobra.Command{ Use: "delete-file repo-name commit-id path/to/file", Short: "Delete a file.", Long: "Delete a file.", Run: cmd.RunFixedArgs(2, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } return client.DeleteFile(args[0], args[1], args[2], false, "") }), } var debug bool mount := &cobra.Command{ Use: "mount path/to/mount/point", Short: "Mount pfs locally.", Long: "Mount pfs locally.", Run: cmd.RunFixedArgs(1, func(args []string) error { client, err := client.NewFromAddress(address) if err != nil { return err } mounter := fuse.NewMounter(address, client.PfsAPIClient) mountPoint := args[0] err = mounter.Mount(mountPoint, shard(), nil, nil, debug) if err != nil { return err } return nil }), } addShardFlags(mount) finishCommit.Flags().BoolVarP(&debug, "debug", "d", false, "turn on debug messages") var result []*cobra.Command result = append(result, repo) result = append(result, createRepo) result = append(result, inspectRepo) result = append(result, listRepo) result = append(result, deleteRepo) result = append(result, commit) result = append(result, startCommit) result = append(result, finishCommit) result = append(result, inspectCommit) result = append(result, listCommit) result = append(result, flushCommit) result = append(result, listBranch) result = append(result, file) result = append(result, putFile) result = append(result, getFile) result = append(result, inspectFile) result = append(result, listFile) result = append(result, deleteFile) result = append(result, mount) return result }