func RunTest(
t *testing.T,
f func(*testing.T, pfs.APIClient, pps.JobAPIClient, pps.PipelineAPIClient),
) {
pfstesting.RunTest(
t,
func(t *testing.T, pfsAPIClient pfs.APIClient, _ pfstesting.Cluster) {
pfsMountDir, err := ioutil.TempDir("", "")
require.NoError(t, err)
mounter := fuse.NewMounter("localhost", pfsAPIClient)
ready := make(chan bool)
go func() {
require.NoError(t, mounter.Mount(pfsMountDir, &pfs.Shard{Number: 0, Modulus: 1}, nil, ready))
}()
<-ready
persistservertesting.RunTestWithRethinkAPIServer(
t,
func(t *testing.T, persistAPIServer persist.APIServer) {
persistAPIClient := persist.NewLocalAPIClient(persistAPIServer)
var pipelineAPIServer pipelineserver.APIServer
prototest.RunT(
t,
testNumServers,
func(servers map[string]*grpc.Server) {
jobAPIServer := jobserver.NewAPIServer(
pfsAPIClient,
persistAPIClient,
nil,
)
jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
pipelineAPIServer = pipelineserver.NewAPIServer(
pfsAPIClient,
jobAPIClient,
persistAPIClient,
)
for _, server := range servers {
pps.RegisterJobAPIServer(server, jobAPIServer)
pps.RegisterPipelineAPIServer(server, pipelineAPIServer)
}
},
func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
pipelineAPIServer.Start()
var clientConn *grpc.ClientConn
for _, c := range clientConns {
clientConn = c
break
}
f(
t,
pfsAPIClient,
pps.NewJobAPIClient(
clientConn,
),
pps.NewPipelineAPIClient(
clientConn,
),
)
},
func() {
_ = mounter.Unmount(pfsMountDir)
},
)
},
)
},
)
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
address := appEnv.PachydermPpsd1Port
if address == "" {
address = appEnv.Address
} else {
address = strings.Replace(address, "tcp://", "", -1)
}
clientConn, err := grpc.Dial(address, grpc.WithInsecure())
if err != nil {
return err
}
jobAPIClient := pps.NewJobAPIClient(clientConn)
pipelineAPIClient := pps.NewPipelineAPIClient(clientConn)
rootCmd := &cobra.Command{
Use: "pps",
Long: `Access the PPS API.
Note that this CLI is experimental and does not even check for common errors.
The environment variable PPS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:651.`,
}
var image string
var outParentCommitID string
createJob := &cobra.Command{
Use: "create-job in-repo-name in-commit-id out-repo-name command [args]",
Short: "Create a new job. Returns the id of the created job.",
Long: `Create a new job. With repo-name/commit-id as input and
out-repo-name as output. A commit will be created for the output.
You can find out the name of the commit with inspect-job.`,
Run: func(cmd *cobra.Command, args []string) {
job, err := jobAPIClient.CreateJob(
context.Background(),
&pps.CreateJobRequest{
Spec: &pps.CreateJobRequest_Transform{
Transform: &pps.Transform{
Image: image,
Cmd: args[3:],
},
},
Input: &pfs.Commit{
Repo: &pfs.Repo{
Name: args[0],
},
Id: args[1],
},
OutputParent: &pfs.Commit{
Repo: &pfs.Repo{
Name: args[2],
},
Id: outParentCommitID,
},
})
if err != nil {
errorAndExit("Error from CreateJob: %s", err.Error())
}
fmt.Println(job.Id)
},
}
createJob.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the job in.")
createJob.Flags().StringVarP(&outParentCommitID, "parent", "p", "", "The parent to use for the output commit.")
inspectJob := &cobra.Command{
Use: "inspect-job job-id",
Short: "Return info about a job.",
Long: "Return info about a job.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
jobInfo, err := jobAPIClient.InspectJob(
context.Background(),
&pps.InspectJobRequest{
Job: &pps.Job{
Id: args[0],
},
},
)
if err != nil {
errorAndExit("Error from InspectJob: %s", err.Error())
}
if jobInfo == nil {
errorAndExit("Job %s not found.", args[0])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintJobHeader(writer)
pretty.PrintJobInfo(writer, jobInfo)
return writer.Flush()
}),
}
var pipelineName string
listJob := &cobra.Command{
Use: "list-job -p pipeline-name",
Short: "Return info about all jobs.",
Long: "Return info about all jobs.",
Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
var pipeline *pps.Pipeline
if pipelineName != "" {
pipeline = &pps.Pipeline{
Name: pipelineName,
}
}
jobInfos, err := jobAPIClient.ListJob(
context.Background(),
&pps.ListJobRequest{
Pipeline: pipeline,
},
)
if err != nil {
errorAndExit("Error from InspectJob: %s", err.Error())
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintJobHeader(writer)
for _, jobInfo := range jobInfos.JobInfo {
pretty.PrintJobInfo(writer, jobInfo)
}
return writer.Flush()
}),
}
listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.")
getJobLogs := &cobra.Command{
Use: "logs job-id",
Short: "Return logs from a job.",
Long: "Return logs from a job.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
logsClient, err := jobAPIClient.GetJobLogs(
context.Background(),
&pps.GetJobLogsRequest{
Job: &pps.Job{
Id: args[0],
},
OutputStream: pps.OutputStream_OUTPUT_STREAM_ALL,
},
)
if err != nil {
errorAndExit("Error from InspectJob: %s", err.Error())
}
if err := protostream.WriteFromStreamingBytesClient(logsClient, os.Stdout); err != nil {
return err
}
return nil
}),
}
createPipeline := &cobra.Command{
Use: "create-pipeline pipeline-name input-repo output-repo command [args]",
Short: "Create a new pipeline.",
Long: "Create a new pipeline.",
Run: func(cmd *cobra.Command, args []string) {
if _, err := pipelineAPIClient.CreatePipeline(
context.Background(),
&pps.CreatePipelineRequest{
Pipeline: &pps.Pipeline{
Name: args[0],
},
Transform: &pps.Transform{
Image: image,
Cmd: args[3:],
},
Input: &pfs.Repo{
Name: args[1],
},
Output: &pfs.Repo{
Name: args[2],
},
},
); err != nil {
errorAndExit("Error from CreatePipeline: %s", err.Error())
}
},
}
createPipeline.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the pipeline's jobs in.")
inspectPipeline := &cobra.Command{
Use: "inspect-pipeline pipeline-name",
Short: "Return info about a pipeline.",
Long: "Return info about a pipeline.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
pipelineInfo, err := pipelineAPIClient.InspectPipeline(
context.Background(),
&pps.InspectPipelineRequest{
Pipeline: &pps.Pipeline{
Name: args[0],
},
},
)
if err != nil {
errorAndExit("Error from InspectPipeline: %s", err.Error())
}
if pipelineInfo == nil {
errorAndExit("Pipeline %s not found.", args[0])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintPipelineHeader(writer)
pretty.PrintPipelineInfo(writer, pipelineInfo)
return writer.Flush()
}),
}
listPipeline := &cobra.Command{
Use: "list-pipeline",
Short: "Return info about all pipelines.",
Long: "Return info about all pipelines.",
Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
pipelineInfos, err := pipelineAPIClient.ListPipeline(
context.Background(),
&pps.ListPipelineRequest{},
)
if err != nil {
errorAndExit("Error from ListPipeline: %s", err.Error())
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintPipelineHeader(writer)
for _, pipelineInfo := range pipelineInfos.PipelineInfo {
pretty.PrintPipelineInfo(writer, pipelineInfo)
}
return writer.Flush()
}),
}
deletePipeline := &cobra.Command{
Use: "delete-pipeline pipeline-name",
Short: "Delete a pipeline.",
Long: "Delete a pipeline.",
Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
if _, err := pipelineAPIClient.DeletePipeline(
context.Background(),
&pps.DeletePipelineRequest{
Pipeline: &pps.Pipeline{
Name: args[0],
},
},
); err != nil {
errorAndExit("Error from DeletePipeline: %s", err.Error())
}
return nil
}),
}
rootCmd.AddCommand(createJob)
rootCmd.AddCommand(inspectJob)
rootCmd.AddCommand(listJob)
rootCmd.AddCommand(getJobLogs)
rootCmd.AddCommand(createPipeline)
rootCmd.AddCommand(inspectPipeline)
rootCmd.AddCommand(listPipeline)
rootCmd.AddCommand(deletePipeline)
return rootCmd.Execute()
}