예제 #1
0
파일: testing.go 프로젝트: klucar/pachyderm
func RunTest(
	t *testing.T,
	f func(*testing.T, pfs.APIClient, pps.JobAPIClient, pps.PipelineAPIClient),
) {
	pfstesting.RunTest(
		t,
		func(t *testing.T, pfsAPIClient pfs.APIClient, _ pfstesting.Cluster) {
			pfsMountDir, err := ioutil.TempDir("", "")
			require.NoError(t, err)
			mounter := fuse.NewMounter("localhost", pfsAPIClient)
			ready := make(chan bool)
			go func() {
				require.NoError(t, mounter.Mount(pfsMountDir, &pfs.Shard{Number: 0, Modulus: 1}, nil, ready))
			}()
			<-ready
			persistservertesting.RunTestWithRethinkAPIServer(
				t,
				func(t *testing.T, persistAPIServer persist.APIServer) {
					persistAPIClient := persist.NewLocalAPIClient(persistAPIServer)
					var pipelineAPIServer pipelineserver.APIServer
					prototest.RunT(
						t,
						testNumServers,
						func(servers map[string]*grpc.Server) {
							jobAPIServer := jobserver.NewAPIServer(
								pfsAPIClient,
								persistAPIClient,
								nil,
							)
							jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
							pipelineAPIServer = pipelineserver.NewAPIServer(
								pfsAPIClient,
								jobAPIClient,
								persistAPIClient,
							)
							for _, server := range servers {
								pps.RegisterJobAPIServer(server, jobAPIServer)
								pps.RegisterPipelineAPIServer(server, pipelineAPIServer)
							}
						},
						func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
							pipelineAPIServer.Start()
							var clientConn *grpc.ClientConn
							for _, c := range clientConns {
								clientConn = c
								break
							}
							f(
								t,
								pfsAPIClient,
								pps.NewJobAPIClient(
									clientConn,
								),
								pps.NewPipelineAPIClient(
									clientConn,
								),
							)
						},
						func() {
							_ = mounter.Unmount(pfsMountDir)
						},
					)
				},
			)
		},
	)
}
예제 #2
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	address := appEnv.PachydermPpsd1Port
	if address == "" {
		address = appEnv.Address
	} else {
		address = strings.Replace(address, "tcp://", "", -1)
	}
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return err
	}
	jobAPIClient := pps.NewJobAPIClient(clientConn)
	pipelineAPIClient := pps.NewPipelineAPIClient(clientConn)
	rootCmd := &cobra.Command{
		Use: "pps",
		Long: `Access the PPS API.

Note that this CLI is experimental and does not even check for common errors.
The environment variable PPS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:651.`,
	}

	var image string
	var outParentCommitID string
	createJob := &cobra.Command{
		Use:   "create-job in-repo-name in-commit-id out-repo-name command [args]",
		Short: "Create a new job. Returns the id of the created job.",
		Long: `Create a new job. With repo-name/commit-id as input and
out-repo-name as output. A commit will be created for the output.
You can find out the name of the commit with inspect-job.`,
		Run: func(cmd *cobra.Command, args []string) {
			job, err := jobAPIClient.CreateJob(
				context.Background(),
				&pps.CreateJobRequest{
					Spec: &pps.CreateJobRequest_Transform{
						Transform: &pps.Transform{
							Image: image,
							Cmd:   args[3:],
						},
					},
					Input: &pfs.Commit{
						Repo: &pfs.Repo{
							Name: args[0],
						},
						Id: args[1],
					},
					OutputParent: &pfs.Commit{
						Repo: &pfs.Repo{
							Name: args[2],
						},
						Id: outParentCommitID,
					},
				})
			if err != nil {
				errorAndExit("Error from CreateJob: %s", err.Error())
			}
			fmt.Println(job.Id)
		},
	}
	createJob.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the job in.")
	createJob.Flags().StringVarP(&outParentCommitID, "parent", "p", "", "The parent to use for the output commit.")

	inspectJob := &cobra.Command{
		Use:   "inspect-job job-id",
		Short: "Return info about a job.",
		Long:  "Return info about a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			jobInfo, err := jobAPIClient.InspectJob(
				context.Background(),
				&pps.InspectJobRequest{
					Job: &pps.Job{
						Id: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			if jobInfo == nil {
				errorAndExit("Job %s not found.", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			pretty.PrintJobInfo(writer, jobInfo)
			return writer.Flush()
		}),
	}

	var pipelineName string
	listJob := &cobra.Command{
		Use:   "list-job -p pipeline-name",
		Short: "Return info about all jobs.",
		Long:  "Return info about all jobs.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			var pipeline *pps.Pipeline
			if pipelineName != "" {
				pipeline = &pps.Pipeline{
					Name: pipelineName,
				}
			}
			jobInfos, err := jobAPIClient.ListJob(
				context.Background(),
				&pps.ListJobRequest{
					Pipeline: pipeline,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			for _, jobInfo := range jobInfos.JobInfo {
				pretty.PrintJobInfo(writer, jobInfo)
			}
			return writer.Flush()
		}),
	}
	listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.")

	getJobLogs := &cobra.Command{
		Use:   "logs job-id",
		Short: "Return logs from a job.",
		Long:  "Return logs from a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			logsClient, err := jobAPIClient.GetJobLogs(
				context.Background(),
				&pps.GetJobLogsRequest{
					Job: &pps.Job{
						Id: args[0],
					},
					OutputStream: pps.OutputStream_OUTPUT_STREAM_ALL,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			if err := protostream.WriteFromStreamingBytesClient(logsClient, os.Stdout); err != nil {
				return err
			}
			return nil
		}),
	}

	createPipeline := &cobra.Command{
		Use:   "create-pipeline pipeline-name input-repo output-repo command [args]",
		Short: "Create a new pipeline.",
		Long:  "Create a new pipeline.",
		Run: func(cmd *cobra.Command, args []string) {
			if _, err := pipelineAPIClient.CreatePipeline(
				context.Background(),
				&pps.CreatePipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
					Transform: &pps.Transform{
						Image: image,
						Cmd:   args[3:],
					},
					Input: &pfs.Repo{
						Name: args[1],
					},
					Output: &pfs.Repo{
						Name: args[2],
					},
				},
			); err != nil {
				errorAndExit("Error from CreatePipeline: %s", err.Error())
			}
		},
	}
	createPipeline.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the pipeline's jobs in.")

	inspectPipeline := &cobra.Command{
		Use:   "inspect-pipeline pipeline-name",
		Short: "Return info about a pipeline.",
		Long:  "Return info about a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			pipelineInfo, err := pipelineAPIClient.InspectPipeline(
				context.Background(),
				&pps.InspectPipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectPipeline: %s", err.Error())
			}
			if pipelineInfo == nil {
				errorAndExit("Pipeline %s not found.", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			pretty.PrintPipelineInfo(writer, pipelineInfo)
			return writer.Flush()
		}),
	}

	listPipeline := &cobra.Command{
		Use:   "list-pipeline",
		Short: "Return info about all pipelines.",
		Long:  "Return info about all pipelines.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			pipelineInfos, err := pipelineAPIClient.ListPipeline(
				context.Background(),
				&pps.ListPipelineRequest{},
			)
			if err != nil {
				errorAndExit("Error from ListPipeline: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			for _, pipelineInfo := range pipelineInfos.PipelineInfo {
				pretty.PrintPipelineInfo(writer, pipelineInfo)
			}
			return writer.Flush()
		}),
	}

	deletePipeline := &cobra.Command{
		Use:   "delete-pipeline pipeline-name",
		Short: "Delete a pipeline.",
		Long:  "Delete a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			if _, err := pipelineAPIClient.DeletePipeline(
				context.Background(),
				&pps.DeletePipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
				},
			); err != nil {
				errorAndExit("Error from DeletePipeline: %s", err.Error())
			}
			return nil
		}),
	}

	rootCmd.AddCommand(createJob)
	rootCmd.AddCommand(inspectJob)
	rootCmd.AddCommand(listJob)
	rootCmd.AddCommand(getJobLogs)
	rootCmd.AddCommand(createPipeline)
	rootCmd.AddCommand(inspectPipeline)
	rootCmd.AddCommand(listPipeline)
	rootCmd.AddCommand(deletePipeline)
	return rootCmd.Execute()
}