Example #1
0
func main() {
	var commits int
	var lines int
	cmd := &cobra.Command{
		Use:   os.Args[0],
		Short: "Generate pfs traffic.",
		Long:  "Generate pfs traffic.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			rand.Seed(time.Now().UnixNano())
			client, err := client.NewFromAddress("0.0.0.0:30650")
			if err != nil {
				return err
			}
			for i := 0; i < commits; i++ {
				if _, err := client.StartCommit("data", "", "master"); err != nil {
					return err
				}
				if _, err := client.PutFile("data", "master", "sales", &reader{lines}); err != nil {
					return err
				}
				if err := client.FinishCommit("data", "master"); err != nil {
					return err
				}
			}
			return nil
		}),
	}
	cmd.Flags().IntVarP(&commits, "commits", "c", 100, "commits to write")
	cmd.Flags().IntVarP(&lines, "lines", "l", 100, "lines to write for each commit")
	cmd.Execute()
}
Example #2
0
func Example_pps() {
	c, err := client.NewFromAddress("0.0.0.0:30650")
	if err != nil {
		return // handle error
	}

	// we assume there's already a repo called "repo"
	// and that it already has some data in it
	// take a look at src/client/pfs_test.go for an example of how to get there.

	// Create a map pipeline
	if err := c.CreatePipeline(
		"map", // the name of the pipeline
		"pachyderm/test_image", // your docker image
		[]string{"map"},        // the command run in your docker image
		nil,                    // no stdin
		0,                      // let pachyderm decide the parallelism
		[]*pps.PipelineInput{
			// map over "repo"
			client.NewPipelineInput("repo", client.MapMethod),
		},
	); err != nil {
		return // handle error
	}
	if err := c.CreatePipeline(
		"reduce",               // the name of the pipeline
		"pachyderm/test_image", // your docker image
		[]string{"reduce"},     // the command run in your docker image
		nil,                    // no stdin
		0,                      // let pachyderm decide the parallelism
		[]*pps.PipelineInput{
			// reduce over "map"
			client.NewPipelineInput("map", client.ReduceMethod),
		},
	); err != nil {
		return // handle error
	}

	commits, err := c.ListCommit( // List commits that are...
		[]string{"reduce"}, // from the "reduce" repo (which the "reduce" pipeline outputs)
		nil,                // starting at the beginning of time
		client.CommitTypeRead, // are readable
		true,  // block until commits are available
		false, // ignore cancelled commits
		nil,   // have no provenance
	)
	if err != nil {
		return // handle error
	}
	for _, commitInfo := range commits {
		// Read output from the pipeline
		var buffer bytes.Buffer
		if err := c.GetFile("reduce", commitInfo.Commit.ID, "file", 0, 0, "", nil, &buffer); err != nil {
			return //handle error
		}
	}
}
Example #3
0
// getDataSet gets a previously stored dataset from pachyderm data versioning.
func getDataSet(dataSet, branch, repoName string) (bytes.Buffer, error) {

	// Open a connection to pachyderm running on localhost.
	c, err := client.NewFromAddress("localhost:30650")
	if err != nil {
		return bytes.Buffer{}, errors.Wrap(err, "Could not connect to Pachyderm")
	}

	// Read the latest commit of filename to the given repoName.
	var buffer bytes.Buffer
	if err := c.GetFile(repoName, branch, dataSet, 0, 0, "", nil, &buffer); err != nil {
		return buffer, errors.Wrap(err, "Could not retrieve pachyderm file")
	}

	return buffer, nil
}
Example #4
0
// pushToPach reads in the given file, create a repo in PFS for the file, opens
// a commit, pushes the file in the commit, and finishes the commit.
func pushToPach(path, dataName, repoName string) error {

	// Read the contents of the given file.
	csvfile, err := ioutil.ReadFile(path)
	if err != nil {
		errors.Wrap(err, "Could not read input file")
	}

	// Open a connection to pachyderm running on localhost.
	c, err := client.NewFromAddress("localhost:30650")
	if err != nil {
		errors.Wrap(err, "Could not connect to Pachyderm")
	}

	// Create a repo.
	if err := c.CreateRepo(repoName); err != nil {
		errors.Wrap(err, "Could not create pachyderm repo")
	}

	// Start a commit on the "master" branch of the repo.
	if _, err = c.StartCommit(repoName, "", "master"); err != nil {
		errors.Wrap(err, "Could not start pachyderm repo commit")
	}

	// Put the given file into the repo after the commit is started.
	r := bytes.NewReader(csvfile)
	if _, err := c.PutFile(repoName, "master", dataName, r); err != nil {
		errors.Wrap(err, "Could not put file into pachyderm repo")
	}

	// Finish the commit.
	if err := c.FinishCommit(repoName, "master"); err != nil {
		errors.Wrap(err, "Could not finish Pachyderm commit")
	}

	return nil
}
Example #5
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	rootCmd := &cobra.Command{
		Use:   os.Args[0] + " job-id",
		Short: `Pachyderm job-shim, coordinates with ppsd to create an output commit and run user work.`,
		Long:  `Pachyderm job-shim, coordinates with ppsd to create an output commit and run user work.`,
		Run: func(_ *cobra.Command, args []string) {
			ppsClient, err := ppsserver.NewInternalJobAPIClientFromAddress(fmt.Sprintf("%v:650", appEnv.PachydermAddress))
			if err != nil {
				errorAndExit(err.Error())
			}
			response, err := ppsClient.StartJob(
				context.Background(),
				&ppsserver.StartJobRequest{
					Job: &ppsclient.Job{
						ID: args[0],
					}})
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s\n", err.Error())
				os.Exit(0)
			}

			if response.Transform.Debug {
				lion.SetLevel(lion.LevelDebug)
			}

			// We want to make sure that we only send FinishJob once.
			// The most bulletproof way would be to check that on server side,
			// but this is easier.
			var finished bool
			// Make sure that we call FinishJob even if something caused a panic
			defer func() {
				if r := recover(); r != nil && !finished {
					fmt.Println("job shim crashed; this is like a bug in pachyderm")
					if _, err := ppsClient.FinishJob(
						context.Background(),
						&ppsserver.FinishJobRequest{
							Job: &ppsclient.Job{
								ID: args[0],
							},
							Success: false,
						},
					); err != nil {
						errorAndExit(err.Error())
					}
				}
			}()

			c, err := client.NewFromAddress(fmt.Sprintf("%v:650", appEnv.PachydermAddress))
			if err != nil {
				errorAndExit(err.Error())
			}

			mounter := fuse.NewMounter(appEnv.PachydermAddress, c.PfsAPIClient)
			ready := make(chan bool)
			go func() {
				if err := mounter.MountAndCreate(
					"/pfs",
					nil,
					response.CommitMounts,
					ready,
					response.Transform.Debug,
				); err != nil {
					errorAndExit(err.Error())
				}
			}()
			<-ready
			defer func() {
				if err := mounter.Unmount("/pfs"); err != nil {
					errorAndExit(err.Error())
				}
			}()
			var readers []io.Reader
			for _, line := range response.Transform.Stdin {
				readers = append(readers, strings.NewReader(line+"\n"))
			}
			if len(response.Transform.Cmd) == 0 {
				fmt.Println("unable to run; a cmd needs to be provided")
				if _, err := ppsClient.FinishJob(
					context.Background(),
					&ppsserver.FinishJobRequest{
						Job:     client.NewJob(args[0]),
						Success: false,
					},
				); err != nil {
					errorAndExit(err.Error())
				}
				finished = true
				return
			}
			cmd := exec.Command(response.Transform.Cmd[0], response.Transform.Cmd[1:]...)
			cmd.Stdin = io.MultiReader(readers...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			success := true
			if err := cmd.Run(); err != nil {
				success = false
				if exiterr, ok := err.(*exec.ExitError); ok {
					if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
						for _, returnCode := range response.Transform.AcceptReturnCode {
							if int(returnCode) == status.ExitStatus() {
								success = true
							}
						}
					}
				}
				if !success {
					fmt.Fprintf(os.Stderr, "Error from exec: %s\n", err.Error())
				}
			}
			if _, err := ppsClient.FinishJob(
				context.Background(),
				&ppsserver.FinishJobRequest{
					Job:     client.NewJob(args[0]),
					Success: success,
				},
			); err != nil {
				errorAndExit(err.Error())
			}
			finished = true
		},
	}

	return rootCmd.Execute()
}
Example #6
0
func Example_pfs() {
	c, err := client.NewFromAddress("0.0.0.0:30650")
	if err != nil {
		return // handle error
	}
	// Create a repo called "repo"
	if err := c.CreateRepo("repo"); err != nil {
		return // handle error
	}
	// Start a commit in our new repo on the "master" branch
	commit1, err := c.StartCommit("repo", "", "master")
	if err != nil {
		return // handle error
	}
	// Put a file called "file" in the newly created commit with the content "foo\n".
	if _, err := c.PutFile("repo", "master", "file", strings.NewReader("foo\n")); err != nil {
		return // handle error
	}
	// Finish the commit.
	if err := c.FinishCommit("repo", "master"); err != nil {
		return //handle error
	}
	// Read what we wrote.
	var buffer bytes.Buffer
	if err := c.GetFile("repo", "master", "file", 0, 0, "", nil, &buffer); err != nil {
		return //handle error
	}
	// buffer now contains "foo\n"

	// Start another commit with the previous commit as the parent.
	if _, err := c.StartCommit("repo", "", "master"); err != nil {
		return //handle error
	}
	// Extend "file" in the newly created commit with the content "bar\n".
	if _, err := c.PutFile("repo", "master", "file", strings.NewReader("bar\n")); err != nil {
		return // handle error
	}
	// Finish the commit.
	if err := c.FinishCommit("repo", "master"); err != nil {
		return //handle error
	}
	// Read what we wrote.
	buffer.Reset()
	if err := c.GetFile("repo", "master", "file", 0, 0, "", nil, &buffer); err != nil {
		return //handle error
	}
	// buffer now contains "foo\nbar\n"

	// We can still read the old version of the file though:
	buffer.Reset()
	if err := c.GetFile("repo", commit1.ID, "file", 0, 0, "", nil, &buffer); err != nil {
		return //handle error
	}
	// buffer now contains "foo\n"

	// We can also see the Diff between the most recent commit and the first one:
	buffer.Reset()
	if err := c.GetFile("repo", "master", "file", 0, 0, commit1.ID, nil, &buffer); err != nil {
		return //handle error
	}
}
Example #7
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	etcdClient := getEtcdClient(appEnv)
	if appEnv.Init {
		if err := setClusterID(etcdClient); err != nil {
			return fmt.Errorf("error connecting to etcd, if this error persists it likely indicates that kubernetes services are not working correctly. See https://github.com/pachyderm/pachyderm/blob/master/SETUP.md#pachd-or-pachd-init-crash-loop-with-error-connecting-to-etcd for more info")
		}
		if err := persist_server.InitDBs(fmt.Sprintf("%s:28015", appEnv.DatabaseAddress), appEnv.DatabaseName); err != nil && !isDBCreated(err) {
			return err
		}
		return nil
	}
	if readinessCheck {
		//c, err := client.NewInCluster()
		c, err := client.NewFromAddress("127.0.0.1:650")
		if err != nil {
			return err
		}

		_, err = c.ListRepo(nil)
		if err != nil {
			return err
		}

		os.Exit(0)

		return nil
	}

	clusterID, err := getClusterID(etcdClient)
	if err != nil {
		return err
	}
	kubeClient, err := getKubeClient(appEnv)
	if err != nil {
		return err
	}
	if appEnv.Metrics {
		go metrics.ReportMetrics(clusterID, kubeClient)
	}
	rethinkAPIServer, err := getRethinkAPIServer(appEnv)
	if err != nil {
		return err
	}
	address, err := netutil.ExternalIP()
	if err != nil {
		return err
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	sharder := shard.NewSharder(
		etcdClient,
		appEnv.NumShards,
		appEnv.Namespace,
	)
	go func() {
		if err := sharder.AssignRoles(address, nil); err != nil {
			protolion.Printf("Error from sharder.AssignRoles: %s", err.Error())
		}
	}()
	driver, err := drive.NewDriver(address)
	if err != nil {
		return err
	}
	apiServer := pfs_server.NewAPIServer(
		pfsmodel.NewHasher(
			appEnv.NumShards,
			1,
		),
		shard.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
	)
	go func() {
		if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{apiServer}); err != nil {
			protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
		}
	}()
	internalAPIServer := pfs_server.NewInternalAPIServer(
		pfsmodel.NewHasher(
			appEnv.NumShards,
			1,
		),
		shard.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		driver,
	)
	ppsAPIServer := pps_server.NewAPIServer(
		ppsserver.NewHasher(appEnv.NumShards, appEnv.NumShards),
		address,
		kubeClient,
		getNamespace(),
	)
	go func() {
		if err := sharder.Register(nil, address, []shard.Server{internalAPIServer, ppsAPIServer}); err != nil {
			protolion.Printf("Error from sharder.Register %s", err.Error())
		}
	}()
	blockAPIServer, err := pfs_server.NewBlockAPIServer(appEnv.StorageRoot, appEnv.StorageBackend)
	if err != nil {
		return err
	}
	return protoserver.Serve(
		func(s *grpc.Server) {
			pfsclient.RegisterAPIServer(s, apiServer)
			pfsclient.RegisterInternalAPIServer(s, internalAPIServer)
			pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
			ppsclient.RegisterAPIServer(s, ppsAPIServer)
			ppsserver.RegisterInternalJobAPIServer(s, ppsAPIServer)
			persist.RegisterAPIServer(s, rethinkAPIServer)
		},
		protoserver.ServeOptions{
			Version: version.Version,
		},
		protoserver.ServeEnv{
			GRPCPort: appEnv.Port,
		},
	)
}
Example #8
0
func Cmds(address string) ([]*cobra.Command, error) {
	marshaller := &jsonpb.Marshaler{Indent: "  "}

	job := &cobra.Command{
		Use:   "job",
		Short: "Docs for jobs.",
		Long: `Jobs are the basic unit of computation in Pachyderm.

Jobs run a containerized workload over a set of finished input commits.
Creating a job will also create a new repo and a commit in that repo which
contains the output of the job. Unless the job is created with another job as a
parent. If the job is created with a parent it will use the same repo as its
parent job and the commit it creates will use the parent job's commit as a
parent.
If the job fails the commit it creates will not be finished.
The increase the throughput of a job increase the Shard paremeter.
`,
		Run: pkgcmd.RunFixedArgs(0, func(args []string) error {
			return nil
		}),
	}

	exampleCreateJobRequest, err := marshaller.MarshalToString(example.CreateJobRequest())
	if err != nil {
		return nil, err
	}

	exampleRunPipelineSpec, err := marshaller.MarshalToString(example.RunPipelineSpec())
	if err != nil {
		return nil, err
	}

	pipelineSpec := string(pachyderm.MustAsset("doc/pipeline_spec.md"))

	var jobPath string
	createJob := &cobra.Command{
		Use:   "create-job -f job.json",
		Short: "Create a new job. Returns the id of the created job.",
		Long:  fmt.Sprintf("Create a new job from a spec, the spec looks like this\n%s", exampleCreateJobRequest),
		Run: func(cmd *cobra.Command, args []string) {
			client, err := pach.NewFromAddress(address)
			if err != nil {
				pkgcmd.ErrorAndExit("Error connecting to pps: %v", err)
			}
			var buf bytes.Buffer
			var jobReader io.Reader
			if jobPath == "-" {
				jobReader = io.TeeReader(os.Stdin, &buf)
				fmt.Print("Reading from stdin.\n")
			} else {
				jobFile, err := os.Open(jobPath)
				if err != nil {
					pkgcmd.ErrorAndExit("Error opening %s: %v", jobPath, err)
				}
				defer func() {
					if err := jobFile.Close(); err != nil {
						pkgcmd.ErrorAndExit("Error closing%s: %v", jobPath, err)
					}
				}()
				jobReader = io.TeeReader(jobFile, &buf)
			}
			var request ppsclient.CreateJobRequest
			decoder := json.NewDecoder(jobReader)
			s, err := replaceMethodAliases(decoder)
			if err != nil {
				err = describeSyntaxError(err, buf)
				pkgcmd.ErrorAndExit("Error parsing job spec: %v", err)
			}
			if err := jsonpb.UnmarshalString(s, &request); err != nil {
				pkgcmd.ErrorAndExit("Error reading from stdin: %v", err)
			}
			job, err := client.PpsAPIClient.CreateJob(
				context.Background(),
				&request,
			)
			if err != nil {
				pkgcmd.ErrorAndExit("Error from CreateJob: %v", err)
			}
			fmt.Println(job.ID)
		},
	}
	createJob.Flags().StringVarP(&jobPath, "file", "f", "-", "The file containing the job, - reads from stdin.")

	var block bool
	inspectJob := &cobra.Command{
		Use:   "inspect-job job-id",
		Short: "Return info about a job.",
		Long:  "Return info about a job.",
		Run: pkgcmd.RunFixedArgs(1, func(args []string) error {
			client, err := pach.NewFromAddress(address)
			if err != nil {
				return err
			}
			jobInfo, err := client.InspectJob(args[0], block)
			if err != nil {
				pkgcmd.ErrorAndExit("Error from InspectJob: %s", err.Error())
			}
			if jobInfo == nil {
				pkgcmd.ErrorAndExit("Job %s not found.", args[0])
			}
			pretty.PrintDetailedJobInfo(jobInfo)
			return nil
		}),
	}
	inspectJob.Flags().BoolVarP(&block, "block", "b", false, "block until the job has either succeeded or failed")

	var pipelineName string
	listJob := &cobra.Command{
		Use:   "list-job [-p pipeline-name] [commits]",
		Short: "Return info about jobs.",
		Long: `Return info about jobs.

Examples:

	# return all jobs
	$ pachctl list-job

	# return all jobs in pipeline foo
	$ pachctl list-job -p foo

	# return all jobs whose input commits include foo/abc123 and bar/def456
	$ pachctl list-job foo/abc123 bar/def456

	# return all jobs in pipeline foo and whose input commits include bar/def456
	$ pachctl list-job -p foo bar/def456

`,
		Run: func(cmd *cobra.Command, args []string) {
			client, err := pach.NewFromAddress(address)
			if err != nil {
				pkgcmd.ErrorAndExit("Error from InspectJob: %v", err)
			}

			commits, err := pkgcmd.ParseCommits(args)
			if err != nil {
				cmd.Usage()
				pkgcmd.ErrorAndExit("Error from InspectJob: %v", err)
			}

			jobInfos, err := client.ListJob(pipelineName, commits)
			if err != nil {
				pkgcmd.ErrorAndExit("Error from InspectJob: %v", err)
			}

			// Display newest jobs first
			sort.Sort(sort.Reverse(ByCreationTime(jobInfos)))

			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			for _, jobInfo := range jobInfos {
				pretty.PrintJobInfo(writer, jobInfo)
			}

			if err := writer.Flush(); err != nil {
				pkgcmd.ErrorAndExit("Error from InspectJob: %v", err)
			}
		},
	}
	listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.")

	getLogs := &cobra.Command{
		Use:   "get-logs job-id",
		Short: "Return logs from a job.",
		Long:  "Return logs from a job.",
		Run: pkgcmd.RunFixedArgs(1, func(args []string) error {
			client, err := pach.NewFromAddress(address)
			if err != nil {
				return err
			}
			return client.GetLogs(args[0], os.Stdout)
		}),
	}

	pipeline := &cobra.Command{
		Use:   "pipeline",
		Short: "Docs for pipelines.",
		Long: `Pipelines are a powerful abstraction for automating jobs.

Pipelines take a set of repos as inputs, rather than the set of commits that
jobs take. Pipelines then subscribe to commits on those repos and launches a job
to process each incoming commit.
Creating a pipeline will also create a repo of the same name.
All jobs created by a pipeline will create commits in the pipeline's repo.
`,
		Run: pkgcmd.RunFixedArgs(0, func(args []string) error {
			return nil
		}),
	}

	var pipelinePath string
	if err != nil {
		return nil, err
	}
	createPipeline := &cobra.Command{
		Use:   "create-pipeline -f pipeline.json",
		Short: "Create a new pipeline.",
		Long:  fmt.Sprintf("Create a new pipeline from a spec\n\n%s", pipelineSpec),
		Run: func(cmd *cobra.Command, args []string) {
			client, err := pach.NewFromAddress(address)
			if err != nil {
				pkgcmd.ErrorAndExit("Error connecting to pps: %s", err.Error())
			}
			var buf bytes.Buffer
			var pipelineReader io.Reader
			if pipelinePath == "-" {
				pipelineReader = io.TeeReader(os.Stdin, &buf)
				fmt.Print("Reading from stdin.\n")
			} else {
				rawBytes, err := ioutil.ReadFile(pipelinePath)
				if err != nil {
					pkgcmd.ErrorAndExit("Error reading file %s", pipelinePath)
				}

				pipelineReader = io.TeeReader(strings.NewReader(string(rawBytes)), &buf)
			}
			var request ppsclient.CreatePipelineRequest
			decoder := json.NewDecoder(pipelineReader)
			for {
				s, err := replaceMethodAliases(decoder)
				if err != nil {
					if err == io.EOF {
						break
					}
					err = describeSyntaxError(err, buf)
					pkgcmd.ErrorAndExit("Error parsing pipeline spec: %v", err)
				}
				if err := jsonpb.UnmarshalString(s, &request); err != nil {
					pkgcmd.ErrorAndExit("Error marshalling JSON into protobuf: %v", err)
				}
				if _, err := client.PpsAPIClient.CreatePipeline(
					context.Background(),
					&request,
				); err != nil {
					pkgcmd.ErrorAndExit("Error from CreatePipeline: %v", err)
				}
			}
		},
	}
	createPipeline.Flags().StringVarP(&pipelinePath, "file", "f", "-", "The file containing the pipeline, - reads from stdin.")

	inspectPipeline := &cobra.Command{
		Use:   "inspect-pipeline pipeline-name",
		Short: "Return info about a pipeline.",
		Long:  "Return info about a pipeline.",
		Run: pkgcmd.RunFixedArgs(1, func(args []string) error {
			client, err := pach.NewFromAddress(address)
			if err != nil {
				return err
			}
			pipelineInfo, err := client.InspectPipeline(args[0])
			if err != nil {
				pkgcmd.ErrorAndExit("Error from InspectPipeline: %s", err.Error())
			}
			if pipelineInfo == nil {
				pkgcmd.ErrorAndExit("Pipeline %s not found.", args[0])
			}
			pretty.PrintDetailedPipelineInfo(pipelineInfo)
			return nil
		}),
	}

	listPipeline := &cobra.Command{
		Use:   "list-pipeline",
		Short: "Return info about all pipelines.",
		Long:  "Return info about all pipelines.",
		Run: pkgcmd.RunFixedArgs(0, func(args []string) error {
			client, err := pach.NewFromAddress(address)
			if err != nil {
				return err
			}
			pipelineInfos, err := client.ListPipeline()
			if err != nil {
				pkgcmd.ErrorAndExit("Error from ListPipeline: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			for _, pipelineInfo := range pipelineInfos {
				pretty.PrintPipelineInfo(writer, pipelineInfo)
			}
			return writer.Flush()
		}),
	}

	deletePipeline := &cobra.Command{
		Use:   "delete-pipeline pipeline-name",
		Short: "Delete a pipeline.",
		Long:  "Delete a pipeline.",
		Run: pkgcmd.RunFixedArgs(1, func(args []string) error {
			client, err := pach.NewFromAddress(address)
			if err != nil {
				return err
			}
			if err := client.DeletePipeline(args[0]); err != nil {
				pkgcmd.ErrorAndExit("Error from DeletePipeline: %s", err.Error())
			}
			return nil
		}),
	}

	var specPath string
	runPipeline := &cobra.Command{
		Use:   "run-pipeline pipeline-name [-f job.json]",
		Short: "Run a pipeline once.",
		Long:  fmt.Sprintf("Run a pipeline once, optionally overriding some pipeline options by providing a spec.  The spec looks like this:\n%s", exampleRunPipelineSpec),
		Run: pkgcmd.RunFixedArgs(1, func(args []string) error {
			client, err := pach.NewFromAddress(address)
			if err != nil {
				return err
			}

			request := &ppsclient.CreateJobRequest{
				Pipeline: &ppsclient.Pipeline{
					Name: args[0],
				},
				Force: true,
			}

			var buf bytes.Buffer
			var specReader io.Reader
			if specPath == "-" {
				specReader = io.TeeReader(os.Stdin, &buf)
				fmt.Print("Reading from stdin.\n")
			} else if specPath != "" {
				specFile, err := os.Open(specPath)
				if err != nil {
					pkgcmd.ErrorAndExit("Error opening %s: %v", specPath, err)
				}

				defer func() {
					if err := specFile.Close(); err != nil {
						pkgcmd.ErrorAndExit("Error closing%s: %v", specPath, err)
					}
				}()

				specReader = io.TeeReader(specFile, &buf)
				decoder := json.NewDecoder(specReader)
				s, err := replaceMethodAliases(decoder)
				if err != nil {
					err = describeSyntaxError(err, buf)
					pkgcmd.ErrorAndExit("Error parsing pipeline spec: %v", err)
				}

				if err := jsonpb.UnmarshalString(s, request); err != nil {
					pkgcmd.ErrorAndExit("Error reading from stdin: %v", err)
				}
			}

			job, err := client.PpsAPIClient.CreateJob(
				context.Background(),
				request,
			)
			if err != nil {
				pkgcmd.ErrorAndExit("Error from RunPipeline: %v", err)
			}
			fmt.Println(job.ID)
			return nil
		}),
	}
	runPipeline.Flags().StringVarP(&specPath, "file", "f", "", "The file containing the run-pipeline spec, - reads from stdin.")

	var result []*cobra.Command
	result = append(result, job)
	result = append(result, createJob)
	result = append(result, inspectJob)
	result = append(result, getLogs)
	result = append(result, listJob)
	result = append(result, pipeline)
	result = append(result, createPipeline)
	result = append(result, inspectPipeline)
	result = append(result, listPipeline)
	result = append(result, deletePipeline)
	result = append(result, runPipeline)
	return result, nil
}
Example #9
0
func Cmds(address string) []*cobra.Command {
	var fileNumber int
	var fileModulus int
	var blockNumber int
	var blockModulus int
	shard := func() *pfsclient.Shard {
		return &pfsclient.Shard{
			FileNumber:   uint64(fileNumber),
			FileModulus:  uint64(fileModulus),
			BlockNumber:  uint64(blockNumber),
			BlockModulus: uint64(blockModulus),
		}
	}

	addShardFlags := func(cmd *cobra.Command) {
		cmd.Flags().IntVarP(&fileNumber, "file-shard", "s", 0, "file shard to read")
		cmd.Flags().IntVarP(&fileModulus, "file-modulus", "m", 1, "modulus of file shard")
		cmd.Flags().IntVarP(&blockNumber, "block-shard", "b", 0, "block shard to read")
		cmd.Flags().IntVarP(&blockModulus, "block-modulus", "n", 1, "modulus of block shard")
	}

	repo := &cobra.Command{
		Use:   "repo",
		Short: "Docs for repos.",
		Long: `Repos, short for repository, are the top level data object in Pachyderm.

Repos are created with create-repo.`,
		Run: cmd.RunFixedArgs(0, func(args []string) error {
			return nil
		}),
	}

	createRepo := &cobra.Command{
		Use:   "create-repo repo-name",
		Short: "Create a new repo.",
		Long:  "Create a new repo.",
		Run: cmd.RunFixedArgs(1, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			return client.CreateRepo(args[0])
		}),
	}

	inspectRepo := &cobra.Command{
		Use:   "inspect-repo repo-name",
		Short: "Return info about a repo.",
		Long:  "Return info about a repo.",
		Run: cmd.RunFixedArgs(1, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			repoInfo, err := client.InspectRepo(args[0])
			if err != nil {
				return err
			}
			if repoInfo == nil {
				return fmt.Errorf("repo %s not found", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintRepoHeader(writer)
			pretty.PrintRepoInfo(writer, repoInfo)
			return writer.Flush()
		}),
	}

	var listRepoProvenance cmd.RepeatedStringArg
	listRepo := &cobra.Command{
		Use:   "list-repo",
		Short: "Return all repos.",
		Long:  "Reutrn all repos.",
		Run: cmd.RunFixedArgs(0, func(args []string) error {
			c, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			repoInfos, err := c.ListRepo(listRepoProvenance)
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintRepoHeader(writer)
			for _, repoInfo := range repoInfos {
				pretty.PrintRepoInfo(writer, repoInfo)
			}
			return writer.Flush()
		}),
	}
	listRepo.Flags().VarP(&listRepoProvenance, "provenance", "p", "list only repos with the specified repos provenance")

	deleteRepo := &cobra.Command{
		Use:   "delete-repo repo-name",
		Short: "Delete a repo.",
		Long:  "Delete a repo.",
		Run: cmd.RunFixedArgs(1, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			return client.DeleteRepo(args[0])
		}),
	}

	commit := &cobra.Command{
		Use:   "commit",
		Short: "Docs for commits.",
		Long: `Commits are atomic transactions on the content of a repo.

Creating a commit is a multistep process:
- start a new commit with start-commit
- write files to it through fuse or with put-file
- finish the new commit with finish-commit

Commits that have been started but not finished are NOT durable storage.
Commits become reliable (and immutable) when they are finished.

Commits can be created with another commit as a parent.
This layers the data in the commit over the data in the parent.`,
		Run: cmd.RunFixedArgs(0, func(args []string) error {
			return nil
		}),
	}

	var parentCommitID string
	startCommit := &cobra.Command{
		Use:   "start-commit repo-name [branch]",
		Short: "Start a new commit.",
		Long:  "Start a new commit with parent-commit-id as the parent.",
		Run: cmd.RunBoundedArgs(1, 2, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			branch := ""
			if len(args) == 2 {
				branch = args[1]
			}
			commit, err := client.StartCommit(args[0],
				parentCommitID, branch)
			if err != nil {
				return err
			}
			fmt.Println(commit.ID)
			return nil
		}),
	}
	startCommit.Flags().StringVarP(&parentCommitID, "parent", "p", "", "parent id")

	var cancel bool
	finishCommit := &cobra.Command{
		Use:   "finish-commit repo-name commit-id",
		Short: "Finish a started commit.",
		Long:  "Finish a started commit. Commit-id must be a writeable commit.",
		Run: cmd.RunFixedArgs(2, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			if cancel {
				return client.CancelCommit(args[0], args[1])
			}
			return client.FinishCommit(args[0], args[1])
		}),
	}
	finishCommit.Flags().BoolVarP(&cancel, "cancel", "c", false, "cancel the commit")

	inspectCommit := &cobra.Command{
		Use:   "inspect-commit repo-name commit-id",
		Short: "Return info about a commit.",
		Long:  "Return info about a commit.",
		Run: cmd.RunFixedArgs(2, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			commitInfo, err := client.InspectCommit(args[0], args[1])
			if err != nil {
				return err
			}
			if commitInfo == nil {
				return fmt.Errorf("commit %s not found", args[1])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintCommitInfoHeader(writer)
			pretty.PrintCommitInfo(writer, commitInfo)
			return writer.Flush()
		}),
	}

	var all bool
	var block bool
	var listCommitProvenance cmd.RepeatedStringArg
	listCommit := &cobra.Command{
		Use:   "list-commit repo-name",
		Short: "Return all commits on a set of repos",
		Long: `Return all commits on a set of repos.

Examples:

	# return commits in repo "foo" and repo "bar"
	$ pachctl list-commit foo bar

	# return commits in repo "foo" since commit abc123 and those in repo "bar" since commit def456
	$ pachctl list-commit foo/abc123 bar/def456

	# return commits in repo "foo" that have commits
	# "bar/abc123" and "baz/def456" as provenance
	$ pachctl list-commit foo -p bar/abc123 -p baz/def456

`,
		Run: pkgcobra.Run(func(args []string) error {
			commits, err := cmd.ParseCommits(args)
			if err != nil {
				return err
			}

			var repos []string
			var fromCommits []string
			for _, commit := range commits {
				repos = append(repos, commit.Repo.Name)
				fromCommits = append(fromCommits, commit.ID)
			}

			c, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}

			provenance, err := cmd.ParseCommits(listCommitProvenance)
			if err != nil {
				return err
			}
			commitInfos, err := c.ListCommit(repos, fromCommits, client.CommitTypeNone, block, all, provenance)
			if err != nil {
				return err
			}

			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintCommitInfoHeader(writer)
			for _, commitInfo := range commitInfos {
				pretty.PrintCommitInfo(writer, commitInfo)
			}
			return writer.Flush()
		}),
	}
	listCommit.Flags().BoolVarP(&all, "all", "a", false, "list all commits including cancelled commits")
	listCommit.Flags().BoolVarP(&block, "block", "b", false, "block until there are new commits since the from commits")
	listCommit.Flags().VarP(&listCommitProvenance, "provenance", "p",
		"list only commits with the specified `commit`s provenance, commits are specified as RepoName/CommitID")

	var repos cmd.RepeatedStringArg
	flushCommit := &cobra.Command{
		Use:   "flush-commit commit [commit ...]",
		Short: "Wait for all commits caused by the specified commits to finish and return them.",
		Long: `Wait for all commits caused by the specified commits to finish and return them.

Examples:

	# return commits caused by foo/abc123 and bar/def456
	$ pachctl flush-commit foo/abc123 bar/def456

	# return commits caused by foo/abc123 leading to repos bar and baz
	$ pachctl flush-commit foo/abc123 -r bar -r baz

`,
		Run: pkgcobra.Run(func(args []string) error {
			commits, err := cmd.ParseCommits(args)
			if err != nil {
				return err
			}

			c, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}

			var toRepos []*pfsclient.Repo
			for _, repoName := range repos {
				toRepos = append(toRepos, client.NewRepo(repoName))
			}

			commitInfos, err := c.FlushCommit(commits, toRepos)
			if err != nil {
				return err
			}

			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintCommitInfoHeader(writer)
			for _, commitInfo := range commitInfos {
				pretty.PrintCommitInfo(writer, commitInfo)
			}
			return writer.Flush()
		}),
	}
	flushCommit.Flags().VarP(&repos, "repos", "r", "Wait only for commits leading to a specific set of repos")

	listBranch := &cobra.Command{
		Use:   "list-branch repo-name",
		Short: "Return all branches on a repo.",
		Long:  "Return all branches on a repo.",
		Run: cmd.RunFixedArgs(1, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			commitInfos, err := client.ListBranch(args[0])
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintCommitInfoHeader(writer)
			for _, commitInfo := range commitInfos {
				pretty.PrintCommitInfo(writer, commitInfo)
			}
			return writer.Flush()
		}),
	}

	file := &cobra.Command{
		Use:   "file",
		Short: "Docs for files.",
		Long: `Files are the lowest level data object in Pachyderm.

Files can be written to started (but not finished) commits with put-file.
Files can be read from finished commits with get-file.`,
		Run: cmd.RunFixedArgs(0, func(args []string) error {
			return nil
		}),
	}

	putFile := &cobra.Command{
		Use:   "put-file repo-name commit-id path/to/file",
		Short: "Put a file from stdin",
		Long:  "Put a file from stdin. commit-id must be a writeable commit.",
		Run: cmd.RunFixedArgs(3, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			_, err = client.PutFile(args[0], args[1], args[2], os.Stdin)
			return err
		}),
	}

	var fromCommitID string
	var unsafe bool
	getFile := &cobra.Command{
		Use:   "get-file repo-name commit-id path/to/file",
		Short: "Return the contents of a file.",
		Long:  "Return the contents of a file.",
		Run: cmd.RunFixedArgs(3, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			return client.GetFile(args[0], args[1], args[2], 0, 0, fromCommitID, shard(), os.Stdout)
		}),
	}
	addShardFlags(getFile)
	getFile.Flags().StringVarP(&fromCommitID, "from", "f", "", "only consider data written since this commit")
	getFile.Flags().BoolVar(&unsafe, "unsafe", false, "use this flag if you need to read data written in the current commit; this operation will race with concurrent writes")

	inspectFile := &cobra.Command{
		Use:   "inspect-file repo-name commit-id path/to/file",
		Short: "Return info about a file.",
		Long:  "Return info about a file.",
		Run: cmd.RunFixedArgs(3, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			fileInfo, err := client.InspectFile(args[0], args[1], args[2], "", shard())
			if err != nil {
				return err
			}
			if fileInfo == nil {
				return fmt.Errorf("file %s not found", args[2])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintFileInfoHeader(writer)
			pretty.PrintFileInfo(writer, fileInfo)
			return writer.Flush()
		}),
	}
	addShardFlags(inspectFile)
	inspectFile.Flags().BoolVar(&unsafe, "unsafe", false, "use this flag if you need to inspect files written in the current commit; this operation will race with concurrent writes")

	listFile := &cobra.Command{
		Use:   "list-file repo-name commit-id path/to/dir",
		Short: "Return the files in a directory.",
		Long:  "Return the files in a directory.",
		Run: cmd.RunBoundedArgs(2, 3, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			var path string
			if len(args) == 3 {
				path = args[2]
			}
			fileInfos, err := client.ListFile(args[0], args[1], path, fromCommitID, shard(), true)
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintFileInfoHeader(writer)
			for _, fileInfo := range fileInfos {
				pretty.PrintFileInfo(writer, fileInfo)
			}
			return writer.Flush()
		}),
	}
	addShardFlags(listFile)
	listFile.Flags().StringVarP(&fromCommitID, "from", "f", "", "only list files that are written since this commit")
	listFile.Flags().BoolVar(&unsafe, "unsafe", false, "use this flag if you need to list files written in the current commit; this operation will race with concurrent writes")

	deleteFile := &cobra.Command{
		Use:   "delete-file repo-name commit-id path/to/file",
		Short: "Delete a file.",
		Long:  "Delete a file.",
		Run: cmd.RunFixedArgs(2, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			return client.DeleteFile(args[0], args[1], args[2], false, "")
		}),
	}

	var debug bool
	mount := &cobra.Command{
		Use:   "mount path/to/mount/point",
		Short: "Mount pfs locally.",
		Long:  "Mount pfs locally.",
		Run: cmd.RunFixedArgs(1, func(args []string) error {
			client, err := client.NewFromAddress(address)
			if err != nil {
				return err
			}
			mounter := fuse.NewMounter(address, client.PfsAPIClient)
			mountPoint := args[0]
			err = mounter.Mount(mountPoint, shard(), nil, nil, debug)
			if err != nil {
				return err
			}
			return nil
		}),
	}
	addShardFlags(mount)
	finishCommit.Flags().BoolVarP(&debug, "debug", "d", false, "turn on debug messages")

	var result []*cobra.Command
	result = append(result, repo)
	result = append(result, createRepo)
	result = append(result, inspectRepo)
	result = append(result, listRepo)
	result = append(result, deleteRepo)
	result = append(result, commit)
	result = append(result, startCommit)
	result = append(result, finishCommit)
	result = append(result, inspectCommit)
	result = append(result, listCommit)
	result = append(result, flushCommit)
	result = append(result, listBranch)
	result = append(result, file)
	result = append(result, putFile)
	result = append(result, getFile)
	result = append(result, inspectFile)
	result = append(result, listFile)
	result = append(result, deleteFile)
	result = append(result, mount)
	return result
}