func main() {
var commits int
var lines int
cmd := &cobra.Command{
Use: os.Args[0],
Short: "Generate pfs traffic.",
Long: "Generate pfs traffic.",
Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
rand.Seed(time.Now().UnixNano())
client, err := client.NewFromAddress("0.0.0.0:30650")
if err != nil {
return err
}
for i := 0; i < commits; i++ {
if _, err := client.StartCommit("data", "", "master"); err != nil {
return err
}
if _, err := client.PutFile("data", "master", "sales", &reader{lines}); err != nil {
return err
}
if err := client.FinishCommit("data", "master"); err != nil {
return err
}
}
return nil
}),
}
cmd.Flags().IntVarP(&commits, "commits", "c", 100, "commits to write")
cmd.Flags().IntVarP(&lines, "lines", "l", 100, "lines to write for each commit")
cmd.Execute()
}
func Example_pps() {
c, err := client.NewFromAddress("0.0.0.0:30650")
if err != nil {
return // handle error
}
// we assume there's already a repo called "repo"
// and that it already has some data in it
// take a look at src/client/pfs_test.go for an example of how to get there.
// Create a map pipeline
if err := c.CreatePipeline(
"map", // the name of the pipeline
"pachyderm/test_image", // your docker image
[]string{"map"}, // the command run in your docker image
nil, // no stdin
0, // let pachyderm decide the parallelism
[]*pps.PipelineInput{
// map over "repo"
client.NewPipelineInput("repo", client.MapMethod),
},
); err != nil {
return // handle error
}
if err := c.CreatePipeline(
"reduce", // the name of the pipeline
"pachyderm/test_image", // your docker image
[]string{"reduce"}, // the command run in your docker image
nil, // no stdin
0, // let pachyderm decide the parallelism
[]*pps.PipelineInput{
// reduce over "map"
client.NewPipelineInput("map", client.ReduceMethod),
},
); err != nil {
return // handle error
}
commits, err := c.ListCommit( // List commits that are...
[]string{"reduce"}, // from the "reduce" repo (which the "reduce" pipeline outputs)
nil, // starting at the beginning of time
client.CommitTypeRead, // are readable
true, // block until commits are available
false, // ignore cancelled commits
nil, // have no provenance
)
if err != nil {
return // handle error
}
for _, commitInfo := range commits {
// Read output from the pipeline
var buffer bytes.Buffer
if err := c.GetFile("reduce", commitInfo.Commit.ID, "file", 0, 0, "", nil, &buffer); err != nil {
return //handle error
}
}
}
// getDataSet gets a previously stored dataset from pachyderm data versioning.
func getDataSet(dataSet, branch, repoName string) (bytes.Buffer, error) {
// Open a connection to pachyderm running on localhost.
c, err := client.NewFromAddress("localhost:30650")
if err != nil {
return bytes.Buffer{}, errors.Wrap(err, "Could not connect to Pachyderm")
}
// Read the latest commit of filename to the given repoName.
var buffer bytes.Buffer
if err := c.GetFile(repoName, branch, dataSet, 0, 0, "", nil, &buffer); err != nil {
return buffer, errors.Wrap(err, "Could not retrieve pachyderm file")
}
return buffer, nil
}
// pushToPach reads in the given file, create a repo in PFS for the file, opens
// a commit, pushes the file in the commit, and finishes the commit.
func pushToPach(path, dataName, repoName string) error {
// Read the contents of the given file.
csvfile, err := ioutil.ReadFile(path)
if err != nil {
errors.Wrap(err, "Could not read input file")
}
// Open a connection to pachyderm running on localhost.
c, err := client.NewFromAddress("localhost:30650")
if err != nil {
errors.Wrap(err, "Could not connect to Pachyderm")
}
// Create a repo.
if err := c.CreateRepo(repoName); err != nil {
errors.Wrap(err, "Could not create pachyderm repo")
}
// Start a commit on the "master" branch of the repo.
if _, err = c.StartCommit(repoName, "", "master"); err != nil {
errors.Wrap(err, "Could not start pachyderm repo commit")
}
// Put the given file into the repo after the commit is started.
r := bytes.NewReader(csvfile)
if _, err := c.PutFile(repoName, "master", dataName, r); err != nil {
errors.Wrap(err, "Could not put file into pachyderm repo")
}
// Finish the commit.
if err := c.FinishCommit(repoName, "master"); err != nil {
errors.Wrap(err, "Could not finish Pachyderm commit")
}
return nil
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
rootCmd := &cobra.Command{
Use: os.Args[0] + " job-id",
Short: `Pachyderm job-shim, coordinates with ppsd to create an output commit and run user work.`,
Long: `Pachyderm job-shim, coordinates with ppsd to create an output commit and run user work.`,
Run: func(_ *cobra.Command, args []string) {
ppsClient, err := ppsserver.NewInternalJobAPIClientFromAddress(fmt.Sprintf("%v:650", appEnv.PachydermAddress))
if err != nil {
errorAndExit(err.Error())
}
response, err := ppsClient.StartJob(
context.Background(),
&ppsserver.StartJobRequest{
Job: &ppsclient.Job{
ID: args[0],
}})
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err.Error())
os.Exit(0)
}
if response.Transform.Debug {
lion.SetLevel(lion.LevelDebug)
}
// We want to make sure that we only send FinishJob once.
// The most bulletproof way would be to check that on server side,
// but this is easier.
var finished bool
// Make sure that we call FinishJob even if something caused a panic
defer func() {
if r := recover(); r != nil && !finished {
fmt.Println("job shim crashed; this is like a bug in pachyderm")
if _, err := ppsClient.FinishJob(
context.Background(),
&ppsserver.FinishJobRequest{
Job: &ppsclient.Job{
ID: args[0],
},
Success: false,
},
); err != nil {
errorAndExit(err.Error())
}
}
}()
c, err := client.NewFromAddress(fmt.Sprintf("%v:650", appEnv.PachydermAddress))
if err != nil {
errorAndExit(err.Error())
}
mounter := fuse.NewMounter(appEnv.PachydermAddress, c.PfsAPIClient)
ready := make(chan bool)
go func() {
if err := mounter.MountAndCreate(
"/pfs",
nil,
response.CommitMounts,
ready,
response.Transform.Debug,
); err != nil {
errorAndExit(err.Error())
}
}()
<-ready
defer func() {
if err := mounter.Unmount("/pfs"); err != nil {
errorAndExit(err.Error())
}
}()
var readers []io.Reader
for _, line := range response.Transform.Stdin {
readers = append(readers, strings.NewReader(line+"\n"))
}
if len(response.Transform.Cmd) == 0 {
fmt.Println("unable to run; a cmd needs to be provided")
if _, err := ppsClient.FinishJob(
context.Background(),
&ppsserver.FinishJobRequest{
Job: client.NewJob(args[0]),
Success: false,
},
); err != nil {
errorAndExit(err.Error())
}
finished = true
return
}
cmd := exec.Command(response.Transform.Cmd[0], response.Transform.Cmd[1:]...)
cmd.Stdin = io.MultiReader(readers...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
success := true
if err := cmd.Run(); err != nil {
success = false
if exiterr, ok := err.(*exec.ExitError); ok {
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
for _, returnCode := range response.Transform.AcceptReturnCode {
if int(returnCode) == status.ExitStatus() {
success = true
}
}
}
}
if !success {
fmt.Fprintf(os.Stderr, "Error from exec: %s\n", err.Error())
}
}
if _, err := ppsClient.FinishJob(
context.Background(),
&ppsserver.FinishJobRequest{
Job: client.NewJob(args[0]),
Success: success,
},
); err != nil {
errorAndExit(err.Error())
}
finished = true
},
}
return rootCmd.Execute()
}
func Example_pfs() {
c, err := client.NewFromAddress("0.0.0.0:30650")
if err != nil {
return // handle error
}
// Create a repo called "repo"
if err := c.CreateRepo("repo"); err != nil {
return // handle error
}
// Start a commit in our new repo on the "master" branch
commit1, err := c.StartCommit("repo", "", "master")
if err != nil {
return // handle error
}
// Put a file called "file" in the newly created commit with the content "foo\n".
if _, err := c.PutFile("repo", "master", "file", strings.NewReader("foo\n")); err != nil {
return // handle error
}
// Finish the commit.
if err := c.FinishCommit("repo", "master"); err != nil {
return //handle error
}
// Read what we wrote.
var buffer bytes.Buffer
if err := c.GetFile("repo", "master", "file", 0, 0, "", nil, &buffer); err != nil {
return //handle error
}
// buffer now contains "foo\n"
// Start another commit with the previous commit as the parent.
if _, err := c.StartCommit("repo", "", "master"); err != nil {
return //handle error
}
// Extend "file" in the newly created commit with the content "bar\n".
if _, err := c.PutFile("repo", "master", "file", strings.NewReader("bar\n")); err != nil {
return // handle error
}
// Finish the commit.
if err := c.FinishCommit("repo", "master"); err != nil {
return //handle error
}
// Read what we wrote.
buffer.Reset()
if err := c.GetFile("repo", "master", "file", 0, 0, "", nil, &buffer); err != nil {
return //handle error
}
// buffer now contains "foo\nbar\n"
// We can still read the old version of the file though:
buffer.Reset()
if err := c.GetFile("repo", commit1.ID, "file", 0, 0, "", nil, &buffer); err != nil {
return //handle error
}
// buffer now contains "foo\n"
// We can also see the Diff between the most recent commit and the first one:
buffer.Reset()
if err := c.GetFile("repo", "master", "file", 0, 0, commit1.ID, nil, &buffer); err != nil {
return //handle error
}
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
etcdClient := getEtcdClient(appEnv)
if appEnv.Init {
if err := setClusterID(etcdClient); err != nil {
return fmt.Errorf("error connecting to etcd, if this error persists it likely indicates that kubernetes services are not working correctly. See https://github.com/pachyderm/pachyderm/blob/master/SETUP.md#pachd-or-pachd-init-crash-loop-with-error-connecting-to-etcd for more info")
}
if err := persist_server.InitDBs(fmt.Sprintf("%s:28015", appEnv.DatabaseAddress), appEnv.DatabaseName); err != nil && !isDBCreated(err) {
return err
}
return nil
}
if readinessCheck {
//c, err := client.NewInCluster()
c, err := client.NewFromAddress("127.0.0.1:650")
if err != nil {
return err
}
_, err = c.ListRepo(nil)
if err != nil {
return err
}
os.Exit(0)
return nil
}
clusterID, err := getClusterID(etcdClient)
if err != nil {
return err
}
kubeClient, err := getKubeClient(appEnv)
if err != nil {
return err
}
if appEnv.Metrics {
go metrics.ReportMetrics(clusterID, kubeClient)
}
rethinkAPIServer, err := getRethinkAPIServer(appEnv)
if err != nil {
return err
}
address, err := netutil.ExternalIP()
if err != nil {
return err
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
sharder := shard.NewSharder(
etcdClient,
appEnv.NumShards,
appEnv.Namespace,
)
go func() {
if err := sharder.AssignRoles(address, nil); err != nil {
protolion.Printf("Error from sharder.AssignRoles: %s", err.Error())
}
}()
driver, err := drive.NewDriver(address)
if err != nil {
return err
}
apiServer := pfs_server.NewAPIServer(
pfsmodel.NewHasher(
appEnv.NumShards,
1,
),
shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
go func() {
if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{apiServer}); err != nil {
protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
}
}()
internalAPIServer := pfs_server.NewInternalAPIServer(
pfsmodel.NewHasher(
appEnv.NumShards,
1,
),
shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
ppsAPIServer := pps_server.NewAPIServer(
ppsserver.NewHasher(appEnv.NumShards, appEnv.NumShards),
address,
kubeClient,
getNamespace(),
)
go func() {
if err := sharder.Register(nil, address, []shard.Server{internalAPIServer, ppsAPIServer}); err != nil {
protolion.Printf("Error from sharder.Register %s", err.Error())
}
}()
blockAPIServer, err := pfs_server.NewBlockAPIServer(appEnv.StorageRoot, appEnv.StorageBackend)
if err != nil {
return err
}
return protoserver.Serve(
func(s *grpc.Server) {
pfsclient.RegisterAPIServer(s, apiServer)
pfsclient.RegisterInternalAPIServer(s, internalAPIServer)
pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
ppsclient.RegisterAPIServer(s, ppsAPIServer)
ppsserver.RegisterInternalJobAPIServer(s, ppsAPIServer)
persist.RegisterAPIServer(s, rethinkAPIServer)
},
protoserver.ServeOptions{
Version: version.Version,
},
protoserver.ServeEnv{
GRPCPort: appEnv.Port,
},
)
}
func Cmds(address string) ([]*cobra.Command, error) {
marshaller := &jsonpb.Marshaler{Indent: " "}
job := &cobra.Command{
Use: "job",
Short: "Docs for jobs.",
Long: `Jobs are the basic unit of computation in Pachyderm.
Jobs run a containerized workload over a set of finished input commits.
Creating a job will also create a new repo and a commit in that repo which
contains the output of the job. Unless the job is created with another job as a
parent. If the job is created with a parent it will use the same repo as its
parent job and the commit it creates will use the parent job's commit as a
parent.
If the job fails the commit it creates will not be finished.
The increase the throughput of a job increase the Shard paremeter.
`,
Run: pkgcmd.RunFixedArgs(0, func(args []string) error {
return nil
}),
}
exampleCreateJobRequest, err := marshaller.MarshalToString(example.CreateJobRequest())
if err != nil {
return nil, err
}
exampleRunPipelineSpec, err := marshaller.MarshalToString(example.RunPipelineSpec())
if err != nil {
return nil, err
}
pipelineSpec := string(pachyderm.MustAsset("doc/pipeline_spec.md"))
var jobPath string
createJob := &cobra.Command{
Use: "create-job -f job.json",
Short: "Create a new job. Returns the id of the created job.",
Long: fmt.Sprintf("Create a new job from a spec, the spec looks like this\n%s", exampleCreateJobRequest),
Run: func(cmd *cobra.Command, args []string) {
client, err := pach.NewFromAddress(address)
if err != nil {
pkgcmd.ErrorAndExit("Error connecting to pps: %v", err)
}
var buf bytes.Buffer
var jobReader io.Reader
if jobPath == "-" {
jobReader = io.TeeReader(os.Stdin, &buf)
fmt.Print("Reading from stdin.\n")
} else {
jobFile, err := os.Open(jobPath)
if err != nil {
pkgcmd.ErrorAndExit("Error opening %s: %v", jobPath, err)
}
defer func() {
if err := jobFile.Close(); err != nil {
pkgcmd.ErrorAndExit("Error closing%s: %v", jobPath, err)
}
}()
jobReader = io.TeeReader(jobFile, &buf)
}
var request ppsclient.CreateJobRequest
decoder := json.NewDecoder(jobReader)
s, err := replaceMethodAliases(decoder)
if err != nil {
err = describeSyntaxError(err, buf)
pkgcmd.ErrorAndExit("Error parsing job spec: %v", err)
}
if err := jsonpb.UnmarshalString(s, &request); err != nil {
pkgcmd.ErrorAndExit("Error reading from stdin: %v", err)
}
job, err := client.PpsAPIClient.CreateJob(
context.Background(),
&request,
)
if err != nil {
pkgcmd.ErrorAndExit("Error from CreateJob: %v", err)
}
fmt.Println(job.ID)
},
}
createJob.Flags().StringVarP(&jobPath, "file", "f", "-", "The file containing the job, - reads from stdin.")
var block bool
inspectJob := &cobra.Command{
Use: "inspect-job job-id",
Short: "Return info about a job.",
Long: "Return info about a job.",
Run: pkgcmd.RunFixedArgs(1, func(args []string) error {
client, err := pach.NewFromAddress(address)
if err != nil {
return err
}
jobInfo, err := client.InspectJob(args[0], block)
if err != nil {
pkgcmd.ErrorAndExit("Error from InspectJob: %s", err.Error())
}
if jobInfo == nil {
pkgcmd.ErrorAndExit("Job %s not found.", args[0])
}
pretty.PrintDetailedJobInfo(jobInfo)
return nil
}),
}
inspectJob.Flags().BoolVarP(&block, "block", "b", false, "block until the job has either succeeded or failed")
var pipelineName string
listJob := &cobra.Command{
Use: "list-job [-p pipeline-name] [commits]",
Short: "Return info about jobs.",
Long: `Return info about jobs.
Examples:
# return all jobs
$ pachctl list-job
# return all jobs in pipeline foo
$ pachctl list-job -p foo
# return all jobs whose input commits include foo/abc123 and bar/def456
$ pachctl list-job foo/abc123 bar/def456
# return all jobs in pipeline foo and whose input commits include bar/def456
$ pachctl list-job -p foo bar/def456
`,
Run: func(cmd *cobra.Command, args []string) {
client, err := pach.NewFromAddress(address)
if err != nil {
pkgcmd.ErrorAndExit("Error from InspectJob: %v", err)
}
commits, err := pkgcmd.ParseCommits(args)
if err != nil {
cmd.Usage()
pkgcmd.ErrorAndExit("Error from InspectJob: %v", err)
}
jobInfos, err := client.ListJob(pipelineName, commits)
if err != nil {
pkgcmd.ErrorAndExit("Error from InspectJob: %v", err)
}
// Display newest jobs first
sort.Sort(sort.Reverse(ByCreationTime(jobInfos)))
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintJobHeader(writer)
for _, jobInfo := range jobInfos {
pretty.PrintJobInfo(writer, jobInfo)
}
if err := writer.Flush(); err != nil {
pkgcmd.ErrorAndExit("Error from InspectJob: %v", err)
}
},
}
listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.")
getLogs := &cobra.Command{
Use: "get-logs job-id",
Short: "Return logs from a job.",
Long: "Return logs from a job.",
Run: pkgcmd.RunFixedArgs(1, func(args []string) error {
client, err := pach.NewFromAddress(address)
if err != nil {
return err
}
return client.GetLogs(args[0], os.Stdout)
}),
}
pipeline := &cobra.Command{
Use: "pipeline",
Short: "Docs for pipelines.",
Long: `Pipelines are a powerful abstraction for automating jobs.
Pipelines take a set of repos as inputs, rather than the set of commits that
jobs take. Pipelines then subscribe to commits on those repos and launches a job
to process each incoming commit.
Creating a pipeline will also create a repo of the same name.
All jobs created by a pipeline will create commits in the pipeline's repo.
`,
Run: pkgcmd.RunFixedArgs(0, func(args []string) error {
return nil
}),
}
var pipelinePath string
if err != nil {
return nil, err
}
createPipeline := &cobra.Command{
Use: "create-pipeline -f pipeline.json",
Short: "Create a new pipeline.",
Long: fmt.Sprintf("Create a new pipeline from a spec\n\n%s", pipelineSpec),
Run: func(cmd *cobra.Command, args []string) {
client, err := pach.NewFromAddress(address)
if err != nil {
pkgcmd.ErrorAndExit("Error connecting to pps: %s", err.Error())
}
var buf bytes.Buffer
var pipelineReader io.Reader
if pipelinePath == "-" {
pipelineReader = io.TeeReader(os.Stdin, &buf)
fmt.Print("Reading from stdin.\n")
} else {
rawBytes, err := ioutil.ReadFile(pipelinePath)
if err != nil {
pkgcmd.ErrorAndExit("Error reading file %s", pipelinePath)
}
pipelineReader = io.TeeReader(strings.NewReader(string(rawBytes)), &buf)
}
var request ppsclient.CreatePipelineRequest
decoder := json.NewDecoder(pipelineReader)
for {
s, err := replaceMethodAliases(decoder)
if err != nil {
if err == io.EOF {
break
}
err = describeSyntaxError(err, buf)
pkgcmd.ErrorAndExit("Error parsing pipeline spec: %v", err)
}
if err := jsonpb.UnmarshalString(s, &request); err != nil {
pkgcmd.ErrorAndExit("Error marshalling JSON into protobuf: %v", err)
}
if _, err := client.PpsAPIClient.CreatePipeline(
context.Background(),
&request,
); err != nil {
pkgcmd.ErrorAndExit("Error from CreatePipeline: %v", err)
}
}
},
}
createPipeline.Flags().StringVarP(&pipelinePath, "file", "f", "-", "The file containing the pipeline, - reads from stdin.")
inspectPipeline := &cobra.Command{
Use: "inspect-pipeline pipeline-name",
Short: "Return info about a pipeline.",
Long: "Return info about a pipeline.",
Run: pkgcmd.RunFixedArgs(1, func(args []string) error {
client, err := pach.NewFromAddress(address)
if err != nil {
return err
}
pipelineInfo, err := client.InspectPipeline(args[0])
if err != nil {
pkgcmd.ErrorAndExit("Error from InspectPipeline: %s", err.Error())
}
if pipelineInfo == nil {
pkgcmd.ErrorAndExit("Pipeline %s not found.", args[0])
}
pretty.PrintDetailedPipelineInfo(pipelineInfo)
return nil
}),
}
listPipeline := &cobra.Command{
Use: "list-pipeline",
Short: "Return info about all pipelines.",
Long: "Return info about all pipelines.",
Run: pkgcmd.RunFixedArgs(0, func(args []string) error {
client, err := pach.NewFromAddress(address)
if err != nil {
return err
}
pipelineInfos, err := client.ListPipeline()
if err != nil {
pkgcmd.ErrorAndExit("Error from ListPipeline: %s", err.Error())
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintPipelineHeader(writer)
for _, pipelineInfo := range pipelineInfos {
pretty.PrintPipelineInfo(writer, pipelineInfo)
}
return writer.Flush()
}),
}
deletePipeline := &cobra.Command{
Use: "delete-pipeline pipeline-name",
Short: "Delete a pipeline.",
Long: "Delete a pipeline.",
Run: pkgcmd.RunFixedArgs(1, func(args []string) error {
client, err := pach.NewFromAddress(address)
if err != nil {
return err
}
if err := client.DeletePipeline(args[0]); err != nil {
pkgcmd.ErrorAndExit("Error from DeletePipeline: %s", err.Error())
}
return nil
}),
}
var specPath string
runPipeline := &cobra.Command{
Use: "run-pipeline pipeline-name [-f job.json]",
Short: "Run a pipeline once.",
Long: fmt.Sprintf("Run a pipeline once, optionally overriding some pipeline options by providing a spec. The spec looks like this:\n%s", exampleRunPipelineSpec),
Run: pkgcmd.RunFixedArgs(1, func(args []string) error {
client, err := pach.NewFromAddress(address)
if err != nil {
return err
}
request := &ppsclient.CreateJobRequest{
Pipeline: &ppsclient.Pipeline{
Name: args[0],
},
Force: true,
}
var buf bytes.Buffer
var specReader io.Reader
if specPath == "-" {
specReader = io.TeeReader(os.Stdin, &buf)
fmt.Print("Reading from stdin.\n")
} else if specPath != "" {
specFile, err := os.Open(specPath)
if err != nil {
pkgcmd.ErrorAndExit("Error opening %s: %v", specPath, err)
}
defer func() {
if err := specFile.Close(); err != nil {
pkgcmd.ErrorAndExit("Error closing%s: %v", specPath, err)
}
}()
specReader = io.TeeReader(specFile, &buf)
decoder := json.NewDecoder(specReader)
s, err := replaceMethodAliases(decoder)
if err != nil {
err = describeSyntaxError(err, buf)
pkgcmd.ErrorAndExit("Error parsing pipeline spec: %v", err)
}
if err := jsonpb.UnmarshalString(s, request); err != nil {
pkgcmd.ErrorAndExit("Error reading from stdin: %v", err)
}
}
job, err := client.PpsAPIClient.CreateJob(
context.Background(),
request,
)
if err != nil {
pkgcmd.ErrorAndExit("Error from RunPipeline: %v", err)
}
fmt.Println(job.ID)
return nil
}),
}
runPipeline.Flags().StringVarP(&specPath, "file", "f", "", "The file containing the run-pipeline spec, - reads from stdin.")
var result []*cobra.Command
result = append(result, job)
result = append(result, createJob)
result = append(result, inspectJob)
result = append(result, getLogs)
result = append(result, listJob)
result = append(result, pipeline)
result = append(result, createPipeline)
result = append(result, inspectPipeline)
result = append(result, listPipeline)
result = append(result, deletePipeline)
result = append(result, runPipeline)
return result, nil
}
func Cmds(address string) []*cobra.Command {
var fileNumber int
var fileModulus int
var blockNumber int
var blockModulus int
shard := func() *pfsclient.Shard {
return &pfsclient.Shard{
FileNumber: uint64(fileNumber),
FileModulus: uint64(fileModulus),
BlockNumber: uint64(blockNumber),
BlockModulus: uint64(blockModulus),
}
}
addShardFlags := func(cmd *cobra.Command) {
cmd.Flags().IntVarP(&fileNumber, "file-shard", "s", 0, "file shard to read")
cmd.Flags().IntVarP(&fileModulus, "file-modulus", "m", 1, "modulus of file shard")
cmd.Flags().IntVarP(&blockNumber, "block-shard", "b", 0, "block shard to read")
cmd.Flags().IntVarP(&blockModulus, "block-modulus", "n", 1, "modulus of block shard")
}
repo := &cobra.Command{
Use: "repo",
Short: "Docs for repos.",
Long: `Repos, short for repository, are the top level data object in Pachyderm.
Repos are created with create-repo.`,
Run: cmd.RunFixedArgs(0, func(args []string) error {
return nil
}),
}
createRepo := &cobra.Command{
Use: "create-repo repo-name",
Short: "Create a new repo.",
Long: "Create a new repo.",
Run: cmd.RunFixedArgs(1, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
return client.CreateRepo(args[0])
}),
}
inspectRepo := &cobra.Command{
Use: "inspect-repo repo-name",
Short: "Return info about a repo.",
Long: "Return info about a repo.",
Run: cmd.RunFixedArgs(1, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
repoInfo, err := client.InspectRepo(args[0])
if err != nil {
return err
}
if repoInfo == nil {
return fmt.Errorf("repo %s not found", args[0])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintRepoHeader(writer)
pretty.PrintRepoInfo(writer, repoInfo)
return writer.Flush()
}),
}
var listRepoProvenance cmd.RepeatedStringArg
listRepo := &cobra.Command{
Use: "list-repo",
Short: "Return all repos.",
Long: "Reutrn all repos.",
Run: cmd.RunFixedArgs(0, func(args []string) error {
c, err := client.NewFromAddress(address)
if err != nil {
return err
}
repoInfos, err := c.ListRepo(listRepoProvenance)
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintRepoHeader(writer)
for _, repoInfo := range repoInfos {
pretty.PrintRepoInfo(writer, repoInfo)
}
return writer.Flush()
}),
}
listRepo.Flags().VarP(&listRepoProvenance, "provenance", "p", "list only repos with the specified repos provenance")
deleteRepo := &cobra.Command{
Use: "delete-repo repo-name",
Short: "Delete a repo.",
Long: "Delete a repo.",
Run: cmd.RunFixedArgs(1, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
return client.DeleteRepo(args[0])
}),
}
commit := &cobra.Command{
Use: "commit",
Short: "Docs for commits.",
Long: `Commits are atomic transactions on the content of a repo.
Creating a commit is a multistep process:
- start a new commit with start-commit
- write files to it through fuse or with put-file
- finish the new commit with finish-commit
Commits that have been started but not finished are NOT durable storage.
Commits become reliable (and immutable) when they are finished.
Commits can be created with another commit as a parent.
This layers the data in the commit over the data in the parent.`,
Run: cmd.RunFixedArgs(0, func(args []string) error {
return nil
}),
}
var parentCommitID string
startCommit := &cobra.Command{
Use: "start-commit repo-name [branch]",
Short: "Start a new commit.",
Long: "Start a new commit with parent-commit-id as the parent.",
Run: cmd.RunBoundedArgs(1, 2, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
branch := ""
if len(args) == 2 {
branch = args[1]
}
commit, err := client.StartCommit(args[0],
parentCommitID, branch)
if err != nil {
return err
}
fmt.Println(commit.ID)
return nil
}),
}
startCommit.Flags().StringVarP(&parentCommitID, "parent", "p", "", "parent id")
var cancel bool
finishCommit := &cobra.Command{
Use: "finish-commit repo-name commit-id",
Short: "Finish a started commit.",
Long: "Finish a started commit. Commit-id must be a writeable commit.",
Run: cmd.RunFixedArgs(2, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
if cancel {
return client.CancelCommit(args[0], args[1])
}
return client.FinishCommit(args[0], args[1])
}),
}
finishCommit.Flags().BoolVarP(&cancel, "cancel", "c", false, "cancel the commit")
inspectCommit := &cobra.Command{
Use: "inspect-commit repo-name commit-id",
Short: "Return info about a commit.",
Long: "Return info about a commit.",
Run: cmd.RunFixedArgs(2, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
commitInfo, err := client.InspectCommit(args[0], args[1])
if err != nil {
return err
}
if commitInfo == nil {
return fmt.Errorf("commit %s not found", args[1])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintCommitInfoHeader(writer)
pretty.PrintCommitInfo(writer, commitInfo)
return writer.Flush()
}),
}
var all bool
var block bool
var listCommitProvenance cmd.RepeatedStringArg
listCommit := &cobra.Command{
Use: "list-commit repo-name",
Short: "Return all commits on a set of repos",
Long: `Return all commits on a set of repos.
Examples:
# return commits in repo "foo" and repo "bar"
$ pachctl list-commit foo bar
# return commits in repo "foo" since commit abc123 and those in repo "bar" since commit def456
$ pachctl list-commit foo/abc123 bar/def456
# return commits in repo "foo" that have commits
# "bar/abc123" and "baz/def456" as provenance
$ pachctl list-commit foo -p bar/abc123 -p baz/def456
`,
Run: pkgcobra.Run(func(args []string) error {
commits, err := cmd.ParseCommits(args)
if err != nil {
return err
}
var repos []string
var fromCommits []string
for _, commit := range commits {
repos = append(repos, commit.Repo.Name)
fromCommits = append(fromCommits, commit.ID)
}
c, err := client.NewFromAddress(address)
if err != nil {
return err
}
provenance, err := cmd.ParseCommits(listCommitProvenance)
if err != nil {
return err
}
commitInfos, err := c.ListCommit(repos, fromCommits, client.CommitTypeNone, block, all, provenance)
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintCommitInfoHeader(writer)
for _, commitInfo := range commitInfos {
pretty.PrintCommitInfo(writer, commitInfo)
}
return writer.Flush()
}),
}
listCommit.Flags().BoolVarP(&all, "all", "a", false, "list all commits including cancelled commits")
listCommit.Flags().BoolVarP(&block, "block", "b", false, "block until there are new commits since the from commits")
listCommit.Flags().VarP(&listCommitProvenance, "provenance", "p",
"list only commits with the specified `commit`s provenance, commits are specified as RepoName/CommitID")
var repos cmd.RepeatedStringArg
flushCommit := &cobra.Command{
Use: "flush-commit commit [commit ...]",
Short: "Wait for all commits caused by the specified commits to finish and return them.",
Long: `Wait for all commits caused by the specified commits to finish and return them.
Examples:
# return commits caused by foo/abc123 and bar/def456
$ pachctl flush-commit foo/abc123 bar/def456
# return commits caused by foo/abc123 leading to repos bar and baz
$ pachctl flush-commit foo/abc123 -r bar -r baz
`,
Run: pkgcobra.Run(func(args []string) error {
commits, err := cmd.ParseCommits(args)
if err != nil {
return err
}
c, err := client.NewFromAddress(address)
if err != nil {
return err
}
var toRepos []*pfsclient.Repo
for _, repoName := range repos {
toRepos = append(toRepos, client.NewRepo(repoName))
}
commitInfos, err := c.FlushCommit(commits, toRepos)
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintCommitInfoHeader(writer)
for _, commitInfo := range commitInfos {
pretty.PrintCommitInfo(writer, commitInfo)
}
return writer.Flush()
}),
}
flushCommit.Flags().VarP(&repos, "repos", "r", "Wait only for commits leading to a specific set of repos")
listBranch := &cobra.Command{
Use: "list-branch repo-name",
Short: "Return all branches on a repo.",
Long: "Return all branches on a repo.",
Run: cmd.RunFixedArgs(1, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
commitInfos, err := client.ListBranch(args[0])
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintCommitInfoHeader(writer)
for _, commitInfo := range commitInfos {
pretty.PrintCommitInfo(writer, commitInfo)
}
return writer.Flush()
}),
}
file := &cobra.Command{
Use: "file",
Short: "Docs for files.",
Long: `Files are the lowest level data object in Pachyderm.
Files can be written to started (but not finished) commits with put-file.
Files can be read from finished commits with get-file.`,
Run: cmd.RunFixedArgs(0, func(args []string) error {
return nil
}),
}
putFile := &cobra.Command{
Use: "put-file repo-name commit-id path/to/file",
Short: "Put a file from stdin",
Long: "Put a file from stdin. commit-id must be a writeable commit.",
Run: cmd.RunFixedArgs(3, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
_, err = client.PutFile(args[0], args[1], args[2], os.Stdin)
return err
}),
}
var fromCommitID string
var unsafe bool
getFile := &cobra.Command{
Use: "get-file repo-name commit-id path/to/file",
Short: "Return the contents of a file.",
Long: "Return the contents of a file.",
Run: cmd.RunFixedArgs(3, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
return client.GetFile(args[0], args[1], args[2], 0, 0, fromCommitID, shard(), os.Stdout)
}),
}
addShardFlags(getFile)
getFile.Flags().StringVarP(&fromCommitID, "from", "f", "", "only consider data written since this commit")
getFile.Flags().BoolVar(&unsafe, "unsafe", false, "use this flag if you need to read data written in the current commit; this operation will race with concurrent writes")
inspectFile := &cobra.Command{
Use: "inspect-file repo-name commit-id path/to/file",
Short: "Return info about a file.",
Long: "Return info about a file.",
Run: cmd.RunFixedArgs(3, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
fileInfo, err := client.InspectFile(args[0], args[1], args[2], "", shard())
if err != nil {
return err
}
if fileInfo == nil {
return fmt.Errorf("file %s not found", args[2])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintFileInfoHeader(writer)
pretty.PrintFileInfo(writer, fileInfo)
return writer.Flush()
}),
}
addShardFlags(inspectFile)
inspectFile.Flags().BoolVar(&unsafe, "unsafe", false, "use this flag if you need to inspect files written in the current commit; this operation will race with concurrent writes")
listFile := &cobra.Command{
Use: "list-file repo-name commit-id path/to/dir",
Short: "Return the files in a directory.",
Long: "Return the files in a directory.",
Run: cmd.RunBoundedArgs(2, 3, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
var path string
if len(args) == 3 {
path = args[2]
}
fileInfos, err := client.ListFile(args[0], args[1], path, fromCommitID, shard(), true)
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintFileInfoHeader(writer)
for _, fileInfo := range fileInfos {
pretty.PrintFileInfo(writer, fileInfo)
}
return writer.Flush()
}),
}
addShardFlags(listFile)
listFile.Flags().StringVarP(&fromCommitID, "from", "f", "", "only list files that are written since this commit")
listFile.Flags().BoolVar(&unsafe, "unsafe", false, "use this flag if you need to list files written in the current commit; this operation will race with concurrent writes")
deleteFile := &cobra.Command{
Use: "delete-file repo-name commit-id path/to/file",
Short: "Delete a file.",
Long: "Delete a file.",
Run: cmd.RunFixedArgs(2, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
return client.DeleteFile(args[0], args[1], args[2], false, "")
}),
}
var debug bool
mount := &cobra.Command{
Use: "mount path/to/mount/point",
Short: "Mount pfs locally.",
Long: "Mount pfs locally.",
Run: cmd.RunFixedArgs(1, func(args []string) error {
client, err := client.NewFromAddress(address)
if err != nil {
return err
}
mounter := fuse.NewMounter(address, client.PfsAPIClient)
mountPoint := args[0]
err = mounter.Mount(mountPoint, shard(), nil, nil, debug)
if err != nil {
return err
}
return nil
}),
}
addShardFlags(mount)
finishCommit.Flags().BoolVarP(&debug, "debug", "d", false, "turn on debug messages")
var result []*cobra.Command
result = append(result, repo)
result = append(result, createRepo)
result = append(result, inspectRepo)
result = append(result, listRepo)
result = append(result, deleteRepo)
result = append(result, commit)
result = append(result, startCommit)
result = append(result, finishCommit)
result = append(result, inspectCommit)
result = append(result, listCommit)
result = append(result, flushCommit)
result = append(result, listBranch)
result = append(result, file)
result = append(result, putFile)
result = append(result, getFile)
result = append(result, inspectFile)
result = append(result, listFile)
result = append(result, deleteFile)
result = append(result, mount)
return result
}