Esempio n. 1
0
func getAPIClient(address string) (pfs.APIClient, error) {
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	return pfs.NewAPIClient(clientConn), nil
}
Esempio n. 2
0
func RunBench(
	b *testing.B,
	f func(*testing.B, pfs.APIClient),
) {
	discoveryClient, err := getEtcdClient()
	require.NoError(b, err)
	var cluster *cluster
	prototest.RunB(
		b,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			cluster = registerFunc(b, discoveryClient, servers)
		},
		func(b *testing.B, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			go func() {
				require.Equal(b, cluster.realSharder.AssignRoles(cluster.cancel), shard.ErrCancelled)
			}()
			cluster.WaitForAvailability()
			f(
				b,
				pfs.NewAPIClient(
					clientConn,
				),
			)
		},
	)
	cluster.Shutdown()
}
Esempio n. 3
0
func getPfsClient(tb testing.TB) pfs.APIClient {
	pfsdAddr := os.Getenv("PFSD_PORT_650_TCP_ADDR")
	if pfsdAddr == "" {
		tb.Error("PFSD_PORT_650_TCP_ADDR not set")
	}
	clientConn, err := grpc.Dial(fmt.Sprintf("%s:650", pfsdAddr), grpc.WithInsecure())
	require.NoError(tb, err)
	return pfs.NewAPIClient(clientConn)
}
Esempio n. 4
0
func (m *mounterProvider) getOnce() (Mounter, error) {
	clientConn, err := grpc.Dial(m.pfsAddress, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	return NewMounter(
		pfs.NewAPIClient(
			clientConn,
		),
	), nil
}
Esempio n. 5
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	containerClient, err := getContainerClient()
	if err != nil {
		return err
	}
	rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsdAddress, err := getPfsdAddress()
	if err != nil {
		return err
	}
	clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewAPIClient(clientConn)
	jobAPIServer := jobserver.NewAPIServer(
		pfsAPIClient,
		rethinkAPIClient,
		containerClient,
		appEnv.PfsMountDir,
		jobserverrun.JobRunnerOptions{
			RemoveContainers: appEnv.RemoveContainers,
		},
	)
	jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
	pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient)
	errC := make(chan error)
	go func() {
		errC <- protoserver.Serve(
			uint16(appEnv.Port),
			func(s *grpc.Server) {
				pps.RegisterJobAPIServer(s, jobAPIServer)
				pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
			},
			protoserver.ServeOptions{
				DebugPort: uint16(appEnv.DebugPort),
				Version:   pachyderm.Version,
			},
		)
	}()
	// TODO: pretty sure without this there is a problem, this is bad, we would
	// prefer a callback for when the server is ready to accept requests
	time.Sleep(1 * time.Second)
	if err := pipelineAPIServer.Start(); err != nil {
		return err
	}
	return <-errC
}
Esempio n. 6
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsdAddress, err := getPfsdAddress()
	if err != nil {
		return err
	}
	clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewAPIClient(clientConn)
	kubeAddr, err := getKubeAddress()
	if err != nil {
		return err
	}
	config := &kube.Config{
		Host:     kubeAddr,
		Insecure: true,
	}
	kubeClient, err := kube.New(config)
	if err != nil {
		protolog.Printf("Error creating kubernetes client: %s", err.Error())
	}
	jobAPIServer := jobserver.NewAPIServer(
		pfsAPIClient,
		rethinkAPIClient,
		kubeClient,
	)
	jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
	pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient)
	if err := pipelineAPIServer.Start(); err != nil {
		return err
	}
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pps.RegisterJobAPIServer(s, jobAPIServer)
			pps.RegisterInternalJobAPIServer(s, jobAPIServer)
			pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
		},
		protoserver.ServeOptions{
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
		},
	)
}
Esempio n. 7
0
func (a *apiServer) DeleteCommit(ctx context.Context, request *pfs.DeleteCommitRequest) (*google_protobuf.Empty, error) {
	a.versionLock.RLock()
	defer a.versionLock.RUnlock()
	ctx = versionToContext(a.version, ctx)
	clientConns, err := a.router.GetAllClientConns(a.version)
	if err != nil {
		return nil, err
	}
	for _, clientConn := range clientConns {
		if _, err := pfs.NewAPIClient(clientConn).DeleteCommit(ctx, request); err != nil {
			return nil, err
		}
	}
	return google_protobuf.EmptyInstance, nil
}
Esempio n. 8
0
func (a *apiServer) getPfsClient() (pfs.APIClient, error) {
	if a.pfsAPIClient == nil {
		var onceErr error
		a.pfsClientOnce.Do(func() {
			clientConn, err := grpc.Dial(a.pfsAddress, grpc.WithInsecure())
			if err != nil {
				onceErr = err
			}
			a.pfsAPIClient = pfs.NewAPIClient(clientConn)
		})
		if onceErr != nil {
			return nil, onceErr
		}
	}
	return a.pfsAPIClient, nil
}
Esempio n. 9
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	rethinkAPIServer, err := getRethinkAPIServer(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsdAddress, err := getPfsdAddress()
	if err != nil {
		return err
	}
	clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewAPIClient(clientConn)
	kubeClient, err := getKubeClient()
	if err != nil {
		return err
	}
	jobAPIServer := jobserver.NewAPIServer(
		pfsAPIClient,
		rethinkAPIServer,
		kubeClient,
	)
	jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
	pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIServer)
	if err := pipelineAPIServer.Start(); err != nil {
		return err
	}
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pps.RegisterJobAPIServer(s, jobAPIServer)
			pps.RegisterInternalJobAPIServer(s, jobAPIServer)
			pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
		},
		protoserver.ServeOptions{
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
		},
	)
}
Esempio n. 10
0
func TestWriteAndRead(t *testing.T) {
	t.Parallel()

	// don't leave goroutines running
	var wg sync.WaitGroup
	defer wg.Wait()

	tmp, err := ioutil.TempDir("", "pachyderm-test-")
	if err != nil {
		t.Fatalf("tempdir: %v", err)
	}
	defer func() {
		if err := os.RemoveAll(tmp); err != nil {
			t.Errorf("cannot remove tempdir: %v", err)
		}
	}()

	// closed on successful termination
	quit := make(chan struct{})
	defer close(quit)
	listener, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("cannot listen: %v", err)
	}
	defer func() {
		_ = listener.Close()
	}()

	// TODO try to share more of this setup code with various main
	// functions
	localAddress := listener.Addr().String()
	srv := grpc.NewServer()
	const (
		numShards = 1
	)
	sharder := shard.NewLocalSharder(localAddress, numShards)
	hasher := pfs.NewHasher(numShards, 1)
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		localAddress,
	)

	blockDir := filepath.Join(tmp, "blocks")
	blockServer, err := server.NewLocalBlockAPIServer(blockDir)
	if err != nil {
		t.Fatalf("NewLocalBlockAPIServer: %v", err)
	}
	pfs.RegisterBlockAPIServer(srv, blockServer)

	driver, err := drive.NewDriver(localAddress)
	if err != nil {
		t.Fatalf("NewDriver: %v", err)
	}

	apiServer := server.NewAPIServer(
		hasher,
		router,
	)
	pfs.RegisterAPIServer(srv, apiServer)

	internalAPIServer := server.NewInternalAPIServer(
		hasher,
		router,
		driver,
	)
	pfs.RegisterInternalAPIServer(srv, internalAPIServer)

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := srv.Serve(listener); err != nil {
			select {
			case <-quit:
				// orderly shutdown
				return
			default:
				t.Errorf("grpc serve: %v", err)
			}
		}
	}()

	clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
	if err != nil {
		t.Fatalf("grpc dial: %v", err)
	}
	apiClient := pfs.NewAPIClient(clientConn)
	mounter := fuse.NewMounter(localAddress, apiClient)

	mountpoint := filepath.Join(tmp, "mnt")
	if err := os.Mkdir(mountpoint, 0700); err != nil {
		t.Fatalf("mkdir mountpoint: %v", err)
	}

	ready := make(chan bool)
	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
			t.Errorf("mount and serve: %v", err)
		}
	}()

	<-ready
	defer func() {
		if err := mounter.Unmount(mountpoint); err != nil {
			t.Errorf("unmount: %v", err)
		}
	}()

	const (
		repoName = "foo"
	)
	if err := pfsutil.CreateRepo(apiClient, repoName); err != nil {
		t.Fatalf("CreateRepo: %v", err)
	}
	commit, err := pfsutil.StartCommit(apiClient, repoName, "")
	if err != nil {
		t.Fatalf("StartCommit: %v", err)
	}

	const (
		greeting = "Hello, world\n"
	)
	filePath := filepath.Join(mountpoint, repoName, commit.Id, "greeting")

	if err := ioutil.WriteFile(filePath, []byte(greeting), 0644); err != nil {
		t.Fatalf("WriteFile: %v", err)
	}

	buf, err := ioutil.ReadFile(filePath)
	if err != nil {
		t.Fatalf("ReadFile: %v", err)
	}
	if g, e := string(buf), greeting; g != e {
		t.Errorf("wrong content: %q != %q", g, e)
	}
}
Esempio n. 11
0
func TestCommitFinishedReadDir(t *testing.T) {
	t.Parallel()

	// don't leave goroutines running
	var wg sync.WaitGroup
	defer wg.Wait()

	tmp, err := ioutil.TempDir("", "pachyderm-test-")
	if err != nil {
		t.Fatalf("tempdir: %v", err)
	}
	defer func() {
		if err := os.RemoveAll(tmp); err != nil {
			t.Errorf("cannot remove tempdir: %v", err)
		}
	}()

	// closed on successful termination
	quit := make(chan struct{})
	defer close(quit)
	listener, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("cannot listen: %v", err)
	}
	defer func() {
		_ = listener.Close()
	}()

	// TODO try to share more of this setup code with various main
	// functions
	localAddress := listener.Addr().String()
	srv := grpc.NewServer()
	const (
		numShards = 1
	)
	sharder := shard.NewLocalSharder(localAddress, numShards)
	hasher := pfs.NewHasher(numShards, 1)
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		localAddress,
	)

	blockDir := filepath.Join(tmp, "blocks")
	blockServer, err := server.NewLocalBlockAPIServer(blockDir)
	if err != nil {
		t.Fatalf("NewLocalBlockAPIServer: %v", err)
	}
	pfs.RegisterBlockAPIServer(srv, blockServer)

	driver, err := drive.NewDriver(localAddress)
	if err != nil {
		t.Fatalf("NewDriver: %v", err)
	}

	apiServer := server.NewAPIServer(
		hasher,
		router,
	)
	pfs.RegisterAPIServer(srv, apiServer)

	internalAPIServer := server.NewInternalAPIServer(
		hasher,
		router,
		driver,
	)
	pfs.RegisterInternalAPIServer(srv, internalAPIServer)

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := srv.Serve(listener); err != nil {
			select {
			case <-quit:
				// orderly shutdown
				return
			default:
				t.Errorf("grpc serve: %v", err)
			}
		}
	}()

	clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
	if err != nil {
		t.Fatalf("grpc dial: %v", err)
	}
	apiClient := pfs.NewAPIClient(clientConn)
	mounter := fuse.NewMounter(localAddress, apiClient)

	mountpoint := filepath.Join(tmp, "mnt")
	if err := os.Mkdir(mountpoint, 0700); err != nil {
		t.Fatalf("mkdir mountpoint: %v", err)
	}

	ready := make(chan bool)
	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
			t.Errorf("mount and serve: %v", err)
		}
	}()

	<-ready
	defer func() {
		if err := mounter.Unmount(mountpoint); err != nil {
			t.Errorf("unmount: %v", err)
		}
	}()

	const (
		repoName = "foo"
	)
	if err := pfsutil.CreateRepo(apiClient, repoName); err != nil {
		t.Fatalf("CreateRepo: %v", err)
	}
	commit, err := pfsutil.StartCommit(apiClient, repoName, "")
	if err != nil {
		t.Fatalf("StartCommit: %v", err)
	}
	t.Logf("open commit %v", commit.Id)

	const (
		greetingName = "greeting"
		greeting     = "Hello, world\n"
		greetingPerm = 0644
	)
	if err := ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.Id, greetingName), []byte(greeting), greetingPerm); err != nil {
		t.Fatalf("WriteFile: %v", err)
	}
	const (
		scriptName = "script"
		script     = "#!/bin/sh\necho foo\n"
		scriptPerm = 0750
	)
	if err := ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.Id, scriptName), []byte(script), scriptPerm); err != nil {
		t.Fatalf("WriteFile: %v", err)
	}

	if err := pfsutil.FinishCommit(apiClient, repoName, commit.Id); err != nil {
		t.Fatalf("FinishCommit: %v", err)
	}

	if err := fstestutil.CheckDir(filepath.Join(mountpoint, repoName, commit.Id), map[string]fstestutil.FileInfoCheck{
		greetingName: func(fi os.FileInfo) error {
			// TODO respect greetingPerm
			if g, e := fi.Mode(), os.FileMode(0666); g != e {
				return fmt.Errorf("wrong mode: %v != %v", g, e)
			}
			if g, e := fi.Size(), int64(len(greeting)); g != e {
				t.Errorf("wrong size: %v != %v", g, e)
			}
			// TODO show fileModTime as mtime
			// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
			// 	t.Errorf("wrong mtime: %v != %v", g, e)
			// }
			return nil
		},
		scriptName: func(fi os.FileInfo) error {
			// TODO respect scriptPerm
			if g, e := fi.Mode(), os.FileMode(0666); g != e {
				return fmt.Errorf("wrong mode: %v != %v", g, e)
			}
			if g, e := fi.Size(), int64(len(script)); g != e {
				t.Errorf("wrong size: %v != %v", g, e)
			}
			// TODO show fileModTime as mtime
			// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
			// 	t.Errorf("wrong mtime: %v != %v", g, e)
			// }
			return nil
		},
	}); err != nil {
		t.Errorf("wrong directory content: %v", err)
	}
}
Esempio n. 12
0
func TestRootReadDir(t *testing.T) {
	t.Parallel()

	// don't leave goroutines running
	var wg sync.WaitGroup
	defer wg.Wait()

	tmp, err := ioutil.TempDir("", "pachyderm-test-")
	if err != nil {
		t.Fatalf("tempdir: %v", err)
	}
	defer func() {
		if err := os.RemoveAll(tmp); err != nil {
			t.Errorf("cannot remove tempdir: %v", err)
		}
	}()

	// closed on successful termination
	quit := make(chan struct{})
	defer close(quit)
	listener, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("cannot listen: %v", err)
	}
	defer func() {
		_ = listener.Close()
	}()

	// TODO try to share more of this setup code with various main
	// functions
	localAddress := listener.Addr().String()
	srv := grpc.NewServer()
	const (
		numShards = 1
	)
	sharder := shard.NewLocalSharder(localAddress, numShards)
	hasher := pfs.NewHasher(numShards, 1)
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		localAddress,
	)

	blockDir := filepath.Join(tmp, "blocks")
	blockServer, err := server.NewLocalBlockAPIServer(blockDir)
	if err != nil {
		t.Fatalf("NewLocalBlockAPIServer: %v", err)
	}
	pfs.RegisterBlockAPIServer(srv, blockServer)

	driver, err := drive.NewDriver(localAddress)
	if err != nil {
		t.Fatalf("NewDriver: %v", err)
	}

	apiServer := server.NewAPIServer(
		hasher,
		router,
	)
	pfs.RegisterAPIServer(srv, apiServer)

	internalAPIServer := server.NewInternalAPIServer(
		hasher,
		router,
		driver,
	)
	pfs.RegisterInternalAPIServer(srv, internalAPIServer)

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := srv.Serve(listener); err != nil {
			select {
			case <-quit:
				// orderly shutdown
				return
			default:
				t.Errorf("grpc serve: %v", err)
			}
		}
	}()

	clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
	if err != nil {
		t.Fatalf("grpc dial: %v", err)
	}
	apiClient := pfs.NewAPIClient(clientConn)
	mounter := fuse.NewMounter(localAddress, apiClient)

	mountpoint := filepath.Join(tmp, "mnt")
	if err := os.Mkdir(mountpoint, 0700); err != nil {
		t.Fatalf("mkdir mountpoint: %v", err)
	}

	ready := make(chan bool)
	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
			t.Errorf("mount and serve: %v", err)
		}
	}()

	<-ready
	defer func() {
		if err := mounter.Unmount(mountpoint); err != nil {
			t.Errorf("unmount: %v", err)
		}
	}()

	if err := pfsutil.CreateRepo(apiClient, "one"); err != nil {
		t.Fatalf("CreateRepo: %v", err)
	}
	if err := pfsutil.CreateRepo(apiClient, "two"); err != nil {
		t.Fatalf("CreateRepo: %v", err)
	}

	if err := fstestutil.CheckDir(mountpoint, map[string]fstestutil.FileInfoCheck{
		"one": func(fi os.FileInfo) error {
			if g, e := fi.Mode(), os.ModeDir|0555; g != e {
				return fmt.Errorf("wrong mode: %v != %v", g, e)
			}
			// TODO show repoSize in repo stat?
			if g, e := fi.Size(), int64(0); g != e {
				t.Errorf("wrong size: %v != %v", g, e)
			}
			// TODO show RepoInfo.Created as time
			// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
			// 	t.Errorf("wrong mtime: %v != %v", g, e)
			// }
			return nil
		},
		"two": func(fi os.FileInfo) error {
			if g, e := fi.Mode(), os.ModeDir|0555; g != e {
				return fmt.Errorf("wrong mode: %v != %v", g, e)
			}
			// TODO show repoSize in repo stat?
			if g, e := fi.Size(), int64(0); g != e {
				t.Errorf("wrong size: %v != %v", g, e)
			}
			// TODO show RepoInfo.Created as time
			// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
			// 	t.Errorf("wrong mtime: %v != %v", g, e)
			// }
			return nil
		},
	}); err != nil {
		t.Errorf("wrong directory content: %v", err)
	}
}
Esempio n. 13
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	address := appEnv.PachydermPfsd1Port
	if address == "" {
		address = appEnv.Address
	} else {
		address = strings.Replace(address, "tcp://", "", -1)
	}
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return err
	}
	apiClient := pfs.NewAPIClient(clientConn)

	var shard int
	var modulus int

	createRepo := &cobra.Command{
		Use:   "create-repo repo-name",
		Short: "Create a new repo.",
		Long:  "Create a new repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			return pfsutil.CreateRepo(apiClient, args[0])
		}),
	}

	inspectRepo := &cobra.Command{
		Use:   "inspect-repo repo-name",
		Short: "Return info about a repo.",
		Long:  "Return info about a repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			repoInfo, err := pfsutil.InspectRepo(apiClient, args[0])
			if err != nil {
				return err
			}
			if repoInfo == nil {
				return fmt.Errorf("repo %s not found", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintRepoHeader(writer)
			pretty.PrintRepoInfo(writer, repoInfo)
			return writer.Flush()
		}),
	}

	listRepo := &cobra.Command{
		Use:   "list-repo",
		Short: "Return all repos.",
		Long:  "Reutrn all repos.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			repoInfos, err := pfsutil.ListRepo(apiClient)
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintRepoHeader(writer)
			for _, repoInfo := range repoInfos {
				pretty.PrintRepoInfo(writer, repoInfo)
			}
			return writer.Flush()
		}),
	}

	deleteRepo := &cobra.Command{
		Use:   "delete-repo repo-name",
		Short: "Delete a repo.",
		Long:  "Delete a repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			return pfsutil.DeleteRepo(apiClient, args[0])
		}),
	}

	startCommit := &cobra.Command{
		Use:   "start-commit repo-name parent-commit-id",
		Short: "Start a new commit.",
		Long:  "Start a new commit with parent-commit-id as the parent.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			commit, err := pfsutil.StartCommit(apiClient, args[0], args[1])
			if err != nil {
				return err
			}
			fmt.Println(commit.Id)
			return nil
		}),
	}

	finishCommit := &cobra.Command{
		Use:   "finish-commit repo-name commit-id",
		Short: "Finish a started commit.",
		Long:  "Finish a started commit. Commit-id must be a writeable commit.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			return pfsutil.FinishCommit(apiClient, args[0], args[1])
		}),
	}

	inspectCommit := &cobra.Command{
		Use:   "inspect-commit repo-name commit-id",
		Short: "Return info about a commit.",
		Long:  "Return info about a commit.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			commitInfo, err := pfsutil.InspectCommit(apiClient, args[0], args[1])
			if err != nil {
				return err
			}
			if commitInfo == nil {
				return fmt.Errorf("commit %s not found", args[1])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintCommitInfoHeader(writer)
			pretty.PrintCommitInfo(writer, commitInfo)
			return writer.Flush()
		}),
	}

	listCommit := &cobra.Command{
		Use:   "list-commit repo-name",
		Short: "Return all commits on a repo.",
		Long:  "Return all commits on a repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			commitInfos, err := pfsutil.ListCommit(apiClient, args[0])
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintCommitInfoHeader(writer)
			for _, commitInfo := range commitInfos {
				pretty.PrintCommitInfo(writer, commitInfo)
			}
			return writer.Flush()
		}),
	}

	deleteCommit := &cobra.Command{
		Use:   "delete-commit repo-name commit-id",
		Short: "Delete a commit.",
		Long:  "Delete a commit.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			return pfsutil.DeleteCommit(apiClient, args[0], args[1])
		}),
	}

	putBlock := &cobra.Command{
		Use:   "put-block repo-name commit-id path/to/file",
		Short: "Put a block from stdin",
		Long:  "Put a block from stdin. Directories must exist. commit-id must be a writeable commit.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			block, err := pfsutil.PutBlock(apiClient, args[0], args[1], args[2], os.Stdin)
			if err != nil {
				return err
			}
			fmt.Println(block.Hash)
			return nil
		}),
	}

	getBlock := &cobra.Command{
		Use:   "get-block hash",
		Short: "Return the contents of a block.",
		Long:  "Return the contents of a block.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			return pfsutil.GetBlock(apiClient, args[0], os.Stdout)
		}),
	}

	inspectBlock := &cobra.Command{
		Use:   "inspect-block hash",
		Short: "Return info about a block.",
		Long:  "Return info about a block.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			blockInfo, err := pfsutil.InspectBlock(apiClient, args[0])
			if err != nil {
				return err
			}
			if blockInfo == nil {
				return fmt.Errorf("block %s not found", args[2])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintBlockInfoHeader(writer)
			pretty.PrintBlockInfo(writer, blockInfo)
			return writer.Flush()
		}),
	}

	listBlock := &cobra.Command{
		Use:   "list-block",
		Short: "Return the blocks in a directory.",
		Long:  "Return the blocks in a directory.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			blockInfos, err := pfsutil.ListBlock(apiClient, uint64(shard), uint64(modulus))
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintBlockInfoHeader(writer)
			for _, blockInfo := range blockInfos {
				pretty.PrintBlockInfo(writer, blockInfo)
			}
			return writer.Flush()
		}),
	}
	listBlock.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
	listBlock.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")

	mkdir := &cobra.Command{
		Use:   "mkdir repo-name commit-id path/to/dir",
		Short: "Make a directory.",
		Long:  "Make a directory. Parent directories need not exist.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			return pfsutil.MakeDirectory(apiClient, args[0], args[1], args[2])
		}),
	}

	putFile := &cobra.Command{
		Use:   "put-file repo-name commit-id path/to/file",
		Short: "Put a file from stdin",
		Long:  "Put a file from stdin. Directories must exist. commit-id must be a writeable commit.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			_, err := pfsutil.PutFile(apiClient, args[0], args[1], args[2], 0, os.Stdin)
			return err
		}),
	}

	getFile := &cobra.Command{
		Use:   "get-file repo-name commit-id path/to/file",
		Short: "Return the contents of a file.",
		Long:  "Return the contents of a file.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			return pfsutil.GetFile(apiClient, args[0], args[1], args[2], 0, math.MaxInt64, os.Stdout)
		}),
	}

	inspectFile := &cobra.Command{
		Use:   "inspect-file repo-name commit-id path/to/file",
		Short: "Return info about a file.",
		Long:  "Return info about a file.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			fileInfo, err := pfsutil.InspectFile(apiClient, args[0], args[1], args[2])
			if err != nil {
				return err
			}
			if fileInfo == nil {
				return fmt.Errorf("file %s not found", args[2])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintFileInfoHeader(writer)
			pretty.PrintFileInfo(writer, fileInfo)
			return writer.Flush()
		}),
	}

	listFile := &cobra.Command{
		Use:   "list-file repo-name commit-id path/to/dir",
		Short: "Return the files in a directory.",
		Long:  "Return the files in a directory.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			fileInfos, err := pfsutil.ListFile(apiClient, args[0], args[1], args[2], uint64(shard), uint64(modulus))
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintFileInfoHeader(writer)
			for _, fileInfo := range fileInfos {
				pretty.PrintFileInfo(writer, fileInfo)
			}
			return writer.Flush()
		}),
	}
	listFile.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
	listFile.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")

	deleteFile := &cobra.Command{
		Use:   "delete-file repo-name commit-id path/to/file",
		Short: "Delete a file.",
		Long:  "Delete a file.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			return pfsutil.DeleteFile(apiClient, args[0], args[1], args[2])
		}),
	}

	listChange := &cobra.Command{
		Use:   "list-change repo-name commit-id path/to/dir",
		Short: "Return the changes in a directory.",
		Long:  "Return the changes in a directory.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			changeInfos, err := pfsutil.ListChange(apiClient, args[0], args[1], args[2], uint64(shard), uint64(modulus))
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintChangeHeader(writer)
			for _, changeInfo := range changeInfos {
				pretty.PrintChange(writer, changeInfo)
			}
			return writer.Flush()
		}),
	}
	listChange.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
	listChange.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")

	inspectServer := &cobra.Command{
		Use:   "inspect-server server-id",
		Short: "Inspect a server.",
		Long:  "Inspect a server.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			serverInfo, err := pfsutil.InspectServer(apiClient, args[0])
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintServerInfoHeader(writer)
			pretty.PrintServerInfo(writer, serverInfo)
			return writer.Flush()
		}),
	}

	listServer := &cobra.Command{
		Use:   "list-server",
		Short: "Return all servers in the cluster.",
		Long:  "Return all servers in the cluster.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			serverInfos, err := pfsutil.ListServer(apiClient)
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintServerInfoHeader(writer)
			for _, serverInfo := range serverInfos {
				pretty.PrintServerInfo(writer, serverInfo)
			}
			return writer.Flush()
		}),
	}

	mount := &cobra.Command{
		Use:   "mount [mountpoint]",
		Short: "Mount pfs locally.",
		Long:  "Mount pfs locally.",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 0, Max: 1}, func(args []string) error {
			mountPoint := "/pfs"
			if len(args) > 0 {
				mountPoint = args[0]
			}
			mounter := fuse.NewMounter(apiClient)
			return mounter.Mount(address, mountPoint, uint64(shard), uint64(modulus))
		}),
	}
	mount.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
	mount.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")

	rootCmd := &cobra.Command{
		Use: "pfs",
		Long: `Access the PFS API.

Note that this CLI is experimental and does not even check for common errors.
The environment variable PFS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:650.`,
	}

	rootCmd.AddCommand(protoclient.NewVersionCommand(clientConn, pachyderm.Version, nil))
	rootCmd.AddCommand(createRepo)
	rootCmd.AddCommand(inspectRepo)
	rootCmd.AddCommand(listRepo)
	rootCmd.AddCommand(deleteRepo)
	rootCmd.AddCommand(startCommit)
	rootCmd.AddCommand(finishCommit)
	rootCmd.AddCommand(inspectCommit)
	rootCmd.AddCommand(listCommit)
	rootCmd.AddCommand(deleteCommit)
	rootCmd.AddCommand(mkdir)
	rootCmd.AddCommand(putBlock)
	rootCmd.AddCommand(getBlock)
	rootCmd.AddCommand(inspectBlock)
	rootCmd.AddCommand(listBlock)
	rootCmd.AddCommand(putFile)
	rootCmd.AddCommand(getFile)
	rootCmd.AddCommand(inspectFile)
	rootCmd.AddCommand(listFile)
	rootCmd.AddCommand(deleteFile)
	rootCmd.AddCommand(listChange)
	rootCmd.AddCommand(inspectServer)
	rootCmd.AddCommand(listServer)
	rootCmd.AddCommand(mount)
	return rootCmd.Execute()
}
Esempio n. 14
0
func NewAPIClient(cc *grpc.ClientConn) *APIClient {
	return &APIClient{
		pfs.NewAPIClient(cc),
		pps.NewAPIClient(cc),
	}
}