Example #1
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	containerClient, err := getContainerClient()
	if err != nil {
		return err
	}
	rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsdAddress, err := getPfsdAddress()
	if err != nil {
		return err
	}
	clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewAPIClient(clientConn)
	jobAPIServer := jobserver.NewAPIServer(
		pfsAPIClient,
		rethinkAPIClient,
		containerClient,
		appEnv.PfsMountDir,
		jobserverrun.JobRunnerOptions{
			RemoveContainers: appEnv.RemoveContainers,
		},
	)
	jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
	pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient)
	errC := make(chan error)
	go func() {
		errC <- protoserver.Serve(
			uint16(appEnv.Port),
			func(s *grpc.Server) {
				pps.RegisterJobAPIServer(s, jobAPIServer)
				pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
			},
			protoserver.ServeOptions{
				DebugPort: uint16(appEnv.DebugPort),
				Version:   pachyderm.Version,
			},
		)
	}()
	// TODO: pretty sure without this there is a problem, this is bad, we would
	// prefer a callback for when the server is ready to accept requests
	time.Sleep(1 * time.Second)
	if err := pipelineAPIServer.Start(); err != nil {
		return err
	}
	return <-errC
}
Example #2
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsdAddress, err := getPfsdAddress()
	if err != nil {
		return err
	}
	clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewAPIClient(clientConn)
	kubeAddr, err := getKubeAddress()
	if err != nil {
		return err
	}
	config := &kube.Config{
		Host:     kubeAddr,
		Insecure: true,
	}
	kubeClient, err := kube.New(config)
	if err != nil {
		protolog.Printf("Error creating kubernetes client: %s", err.Error())
	}
	jobAPIServer := jobserver.NewAPIServer(
		pfsAPIClient,
		rethinkAPIClient,
		kubeClient,
	)
	jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
	pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient)
	if err := pipelineAPIServer.Start(); err != nil {
		return err
	}
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pps.RegisterJobAPIServer(s, jobAPIServer)
			pps.RegisterInternalJobAPIServer(s, jobAPIServer)
			pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
		},
		protoserver.ServeOptions{
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
		},
	)
}
Example #3
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	rethinkAPIServer, err := getRethinkAPIServer(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsdAddress, err := getPfsdAddress()
	if err != nil {
		return err
	}
	clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewAPIClient(clientConn)
	kubeClient, err := getKubeClient()
	if err != nil {
		return err
	}
	jobAPIServer := jobserver.NewAPIServer(
		pfsAPIClient,
		rethinkAPIServer,
		kubeClient,
	)
	jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
	pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIServer)
	if err := pipelineAPIServer.Start(); err != nil {
		return err
	}
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pps.RegisterJobAPIServer(s, jobAPIServer)
			pps.RegisterInternalJobAPIServer(s, jobAPIServer)
			pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
		},
		protoserver.ServeOptions{
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
		},
	)
}
Example #4
0
func RunTest(
	t *testing.T,
	f func(*testing.T, pfs.APIClient, pps.JobAPIClient, pps.PipelineAPIClient),
) {
	pfstesting.RunTest(
		t,
		func(t *testing.T, pfsAPIClient pfs.APIClient, _ pfstesting.Cluster) {
			pfsMountDir, err := ioutil.TempDir("", "")
			require.NoError(t, err)
			mounter := fuse.NewMounter("localhost", pfsAPIClient)
			ready := make(chan bool)
			go func() {
				require.NoError(t, mounter.Mount(pfsMountDir, &pfs.Shard{Number: 0, Modulus: 1}, nil, ready))
			}()
			<-ready
			persistservertesting.RunTestWithRethinkAPIServer(
				t,
				func(t *testing.T, persistAPIServer persist.APIServer) {
					persistAPIClient := persist.NewLocalAPIClient(persistAPIServer)
					var pipelineAPIServer pipelineserver.APIServer
					prototest.RunT(
						t,
						testNumServers,
						func(servers map[string]*grpc.Server) {
							jobAPIServer := jobserver.NewAPIServer(
								pfsAPIClient,
								persistAPIClient,
								nil,
							)
							jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
							pipelineAPIServer = pipelineserver.NewAPIServer(
								pfsAPIClient,
								jobAPIClient,
								persistAPIClient,
							)
							for _, server := range servers {
								pps.RegisterJobAPIServer(server, jobAPIServer)
								pps.RegisterPipelineAPIServer(server, pipelineAPIServer)
							}
						},
						func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
							pipelineAPIServer.Start()
							var clientConn *grpc.ClientConn
							for _, c := range clientConns {
								clientConn = c
								break
							}
							f(
								t,
								pfsAPIClient,
								pps.NewJobAPIClient(
									clientConn,
								),
								pps.NewPipelineAPIClient(
									clientConn,
								),
							)
						},
						func() {
							_ = mounter.Unmount(pfsMountDir)
						},
					)
				},
			)
		},
	)
}