Beispiel #1
0
func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) Cluster {
	cluster := cluster{
		rolers:  make(map[string]role.Roler),
		servers: make(map[string]server.CombinedAPIServer),
		addresser: route.NewDiscoveryAddresser(
			discoveryClient,
			testNamespace(),
		),
		sharder: route.NewSharder(
			testShardsPerServer * testNumServers,
		),
		tb: tb,
	}
	for address, s := range servers {
		combinedAPIServer := server.NewCombinedAPIServer(
			cluster.sharder,
			route.NewRouter(
				cluster.addresser,
				grpcutil.NewDialer(
					grpc.WithInsecure(),
				),
				address,
			),
			getDriver(tb, address),
		)
		pfs.RegisterApiServer(s, combinedAPIServer)
		pfs.RegisterInternalApiServer(s, combinedAPIServer)
		roler := role.NewRoler(cluster.addresser, cluster.sharder, combinedAPIServer, address, testNumReplicas)
		go func() { require.Equal(tb, roler.Run(), discovery.ErrCancelled) }()
		cluster.addresses = append(cluster.addresses, address)
		cluster.rolers[address] = roler
		cluster.servers[address] = combinedAPIServer
	}
	return &cluster
}
Beispiel #2
0
func registerFunc(driver drive.Driver, discoveryClient discovery.Client, servers map[string]*grpc.Server) {
	addresser := route.NewDiscoveryAddresser(
		discoveryClient,
		testNamespace(),
	)
	i := 0
	for address := range servers {
		for j := 0; j < testShardsPerServer; j++ {
			// TODO(pedge): error
			_ = addresser.SetMasterAddress((i*testShardsPerServer)+j, address, 0)
		}
		i++
	}
	for address, s := range servers {
		combinedAPIServer := server.NewCombinedAPIServer(
			route.NewSharder(
				testShardsPerServer*testNumServers,
			),
			route.NewRouter(
				addresser,
				grpcutil.NewDialer(),
				address,
			),
			driver,
		)
		pfs.RegisterApiServer(s, combinedAPIServer)
		pfs.RegisterInternalApiServer(s, combinedAPIServer)
	}
}
Beispiel #3
0
func runTest(
	t *testing.T,
	driver drive.Driver,
	f func(t *testing.T, apiClient pfs.ApiClient),
) {
	grpctest.Run(
		t,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			discoveryClient := discovery.NewMockClient()
			i := 0
			addresses := make([]string, testNumServers)
			for address := range servers {
				shards := make([]string, testShardsPerServer)
				for j := 0; j < testShardsPerServer; j++ {
					shards[j] = fmt.Sprintf("%d", (i*testShardsPerServer)+j)
				}
				_ = discoveryClient.Set(address+"-master", strings.Join(shards, ","))
				addresses[i] = address
				i++
			}
			_ = discoveryClient.Set("all-addresses", strings.Join(addresses, ","))
			for address, server := range servers {
				combinedAPIServer := NewCombinedAPIServer(
					route.NewSharder(
						testShardsPerServer*testNumServers,
					),
					route.NewRouter(
						route.NewDiscoveryAddresser(
							discoveryClient,
						),
						route.NewDialer(),
						address,
					),
					driver,
				)
				pfs.RegisterApiServer(server, combinedAPIServer)
				pfs.RegisterInternalApiServer(server, combinedAPIServer)
			}
		},
		func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			for _, c := range clientConns {
				if c != clientConn {
					_ = c.Close()
				}
			}
			f(
				t,
				pfs.NewApiClient(
					clientConn,
				),
			)
		},
	)
}
Beispiel #4
0
func NewServerGroup(t *testing.T, addresser route.Addresser, numServers int, offset int, numReplicas int) *serverGroup {
	sharder := route.NewSharder(testNumShards)
	serverGroup := serverGroup{offset: offset}
	for i := 0; i < numServers; i++ {
		serverGroup.servers = append(serverGroup.servers, newServer(t))
		serverGroup.rolers = append(serverGroup.rolers, NewRoler(addresser, sharder, serverGroup.servers[i], fmt.Sprintf("server-%d", i+offset), numReplicas))
	}
	return &serverGroup
}
Beispiel #5
0
func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) *cluster {
	realSharder := shard.NewTestSharder(
		discoveryClient,
		testShardsPerServer*testNumServers,
		testNumReplicas,
		testNamespace(),
	)
	sharder := route.NewSharder(
		testShardsPerServer*testNumServers,
		testNumReplicas,
	)
	cluster := cluster{
		servers:         make(map[string]server.APIServer),
		internalServers: make(map[string]server.InternalAPIServer),
		cancels:         make(map[string]chan bool),
		internalCancels: make(map[string]chan bool),
		cancel:          make(chan bool),
		realSharder:     realSharder,
		sharder:         sharder,
		tb:              tb,
	}
	for address, s := range servers {
		cluster.addresses = append(cluster.addresses, address)
		router := route.NewRouter(
			cluster.realSharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		)
		apiServer := server.NewAPIServer(
			cluster.sharder,
			router,
		)
		cluster.servers[address] = apiServer
		cluster.cancels[address] = make(chan bool)
		go func(address string) {
			require.Equal(tb, cluster.realSharder.RegisterFrontend(cluster.cancels[address], address, cluster.servers[address]), shard.ErrCancelled)
		}(address)
		pfs.RegisterAPIServer(s, apiServer)
		internalAPIServer := server.NewInternalAPIServer(
			cluster.sharder,
			router,
			getDriver(tb, address),
		)
		pfs.RegisterInternalAPIServer(s, internalAPIServer)
		cluster.internalServers[address] = internalAPIServer
		cluster.internalCancels[address] = make(chan bool)
		go func(address string) {
			require.Equal(tb, cluster.realSharder.Register(cluster.internalCancels[address], address, cluster.internalServers[address]), shard.ErrCancelled)
		}(address)
	}
	return &cluster
}
func (a *internalAPIServer) ListFile(ctx context.Context, request *pfs.ListFileRequest) (response *pfs.FileInfos, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	version, err := a.getVersion(ctx)
	if err != nil {
		return nil, err
	}
	shards, err := a.router.GetMasterShards(version)
	if err != nil {
		return nil, err
	}
	if request.Shard == nil {
		request.Shard = &pfs.Shard{Number: 0, Modulus: 1}
	}
	sharder := route.NewSharder(request.Shard.Modulus, 0)
	var wg sync.WaitGroup
	var lock sync.Mutex
	var fileInfos []*pfs.FileInfo
	seenDirectories := make(map[string]bool)
	var loopErr error
	for shard := range shards {
		wg.Add(1)
		go func(shard uint64) {
			defer wg.Done()
			subFileInfos, err := a.driver.ListFile(request.File, shard)
			lock.Lock()
			defer lock.Unlock()
			if err != nil {
				if loopErr == nil {
					loopErr = err
				}
				return
			}
			for _, fileInfo := range subFileInfos {
				if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR {
					if seenDirectories[fileInfo.File.Path] {
						continue
					}
					seenDirectories[fileInfo.File.Path] = true
				}
				if sharder.GetShard(fileInfo.File) == request.Shard.Number {
					fileInfos = append(fileInfos, fileInfo)
				}
			}
		}(shard)
	}
	wg.Wait()
	if loopErr != nil {
		return nil, loopErr
	}
	return &pfs.FileInfos{
		FileInfo: fileInfos,
	}, nil
}
Beispiel #7
0
func do() error {
	runtime.GOMAXPROCS(runtime.NumCPU())

	appEnv := &appEnv{}
	if err := env.Populate(appEnv, env.PopulateOptions{}); err != nil {
		return err
	}
	if appEnv.NumShards == 0 {
		appEnv.NumShards = defaultNumShards
	}
	var btrfsAPI btrfs.API
	switch appEnv.BtrfsDriverType {
	case "exec":
		btrfsAPI = btrfs.NewExecAPI()
	case "ffi":
		fallthrough
	default:
		btrfsAPI = btrfs.NewFFIAPI()
	}
	address := fmt.Sprintf("0.0.0.0:%d", appEnv.APIPort)
	combinedAPIServer := server.NewCombinedAPIServer(
		route.NewSharder(
			appEnv.NumShards,
		),
		route.NewRouter(
			route.NewSingleAddresser(
				address,
				appEnv.NumShards,
			),
			route.NewDialer(),
			address,
		),
		drive.NewBtrfsDriver(
			appEnv.BtrfsRoot,
			btrfsAPI,
		),
	)
	server := grpc.NewServer(grpc.MaxConcurrentStreams(math.MaxUint32))
	pfs.RegisterApiServer(server, combinedAPIServer)
	pfs.RegisterInternalApiServer(server, combinedAPIServer)
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", appEnv.APIPort))
	if err != nil {
		return err
	}

	errC := make(chan error)
	go func() { errC <- server.Serve(listener) }()
	go func() { errC <- http.ListenAndServe(":8080", nil) }()
	if appEnv.TracePort != 0 {
		go func() { errC <- http.ListenAndServe(fmt.Sprintf(":%d", appEnv.TracePort), nil) }()
	}
	return <-errC
}
func (a *internalAPIServer) ListFile(ctx context.Context, request *pfs.ListFileRequest) (response *pfs.FileInfos, retErr error) {
	defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
	version, err := a.getVersion(ctx)
	if err != nil {
		return nil, err
	}
	shards, err := a.router.GetMasterShards(version)
	if err != nil {
		return nil, err
	}
	if request.Shard == nil {
		request.Shard = &pfs.Shard{Number: 0, Modulus: 1}
	}
	sharder := route.NewSharder(request.Shard.Modulus, 0)
	var wg sync.WaitGroup
	var lock sync.Mutex
	var fileInfos []*pfs.FileInfo
	var loopErr error
	for shard := range shards {
		shard := shard
		wg.Add(1)
		go func() {
			defer wg.Done()
			subFileInfos, err := a.driver.ListFile(request.File, shard)
			lock.Lock()
			defer lock.Unlock()
			if err != nil && err != pfs.ErrFileNotFound {
				if loopErr == nil {
					loopErr = err
				}
				return
			}
			for _, fileInfo := range subFileInfos {
				if sharder.GetShard(fileInfo.File) == request.Shard.Number ||
					fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR {
					fileInfos = append(fileInfos, fileInfo)
				}
			}
		}()
	}
	wg.Wait()
	if loopErr != nil {
		return nil, loopErr
	}
	return &pfs.FileInfos{
		FileInfo: pfs.ReduceFileInfos(fileInfos),
	}, nil
}
Beispiel #9
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	discoveryClient, err := getEtcdClient()
	if err != nil {
		return err
	}
	address := fmt.Sprintf("0.0.0.0:%d", appEnv.APIPort)
	addresser := route.NewDiscoveryAddresser(
		discoveryClient,
		"namespace",
	)
	for i := 0; i < appEnv.NumShards; i++ {
		if err := addresser.SetMasterAddress(i, address, 0); err != nil {
			return err
		}
	}
	var driver drive.Driver
	switch appEnv.DriverType {
	case "btrfs":
		driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
	}
	combinedAPIServer := server.NewCombinedAPIServer(
		route.NewSharder(
			appEnv.NumShards,
		),
		route.NewRouter(
			addresser,
			grpcutil.NewDialer(),
			address,
		),
		driver,
	)
	return grpcutil.GrpcDo(
		appEnv.APIPort,
		appEnv.TracePort,
		pachyderm.Version,
		func(s *grpc.Server) {
			pfs.RegisterApiServer(s, combinedAPIServer)
			pfs.RegisterInternalApiServer(s, combinedAPIServer)
		},
	)
}
Beispiel #10
0
func (a *internalAPIServer) ListChange(ctx context.Context, request *pfs.ListChangeRequest) (response *pfs.Changes, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	version, err := a.getVersion(ctx)
	if err != nil {
		return nil, err
	}
	shards, err := a.router.GetMasterShards(version)
	if err != nil {
		return nil, err
	}
	if request.Shard == nil {
		request.Shard = &pfs.Shard{Number: 0, Modulus: 1}
	}
	sharder := route.NewSharder(request.Shard.Modulus, 0)
	var wg sync.WaitGroup
	var lock sync.Mutex
	var changes []*pfs.Change
	var loopErr error
	for shard := range shards {
		wg.Add(1)
		go func(shard uint64) {
			defer wg.Done()
			subChanges, err := a.driver.ListChange(request.File, request.From, shard)
			lock.Lock()
			defer lock.Unlock()
			if err != nil {
				if loopErr == nil {
					loopErr = err
				}
				return
			}
			for _, change := range subChanges {
				if sharder.GetShard(change.File) == request.Shard.Number {
					changes = append(changes, change)
				}
			}
		}(shard)
	}
	wg.Wait()
	if loopErr != nil {
		return nil, loopErr
	}
	return &pfs.Changes{
		Change: changes,
	}, nil
}
Beispiel #11
0
func (a *internalAPIServer) ListBlock(ctx context.Context, request *pfs.ListBlockRequest) (response *pfs.BlockInfos, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	version, err := a.getVersion(ctx)
	if err != nil {
		return nil, err
	}
	shards, err := a.router.GetMasterShards(version)
	if err != nil {
		return nil, err
	}
	if request.Shard == nil {
		request.Shard = &pfs.Shard{Number: 0, Modulus: 1}
	}
	sharder := route.NewSharder(request.Shard.Modulus, 0)
	var wg sync.WaitGroup
	var lock sync.Mutex
	var blockInfos []*pfs.BlockInfo
	var loopErr error
	for shard := range shards {
		wg.Add(1)
		go func(shard uint64) {
			defer wg.Done()
			subBlockInfos, err := a.driver.ListBlock(shard)
			lock.Lock()
			defer lock.Unlock()
			if err != nil {
				if loopErr == nil {
					loopErr = err
				}
				return
			}
			for _, blockInfo := range subBlockInfos {
				if request.Shard == nil || sharder.GetBlockShard(blockInfo.Block) == request.Shard.Number {
					blockInfos = append(blockInfos, blockInfo)
				}
			}
		}(shard)
	}
	wg.Wait()
	if loopErr != nil {
		return nil, loopErr
	}
	return &pfs.BlockInfos{
		BlockInfo: blockInfos,
	}, nil
}
Beispiel #12
0
func checkBlockWrites(tb testing.TB, apiClient pfs.APIClient, repoName string, commitID string) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			buffer := bytes.NewBuffer(nil)
			sharder := route.NewSharder(testShardsPerServer*testNumServers, testNumReplicas)
			block := sharder.GetBlock([]byte(fmt.Sprintf("hello%d", i)))
			iErr := pfsutil.GetBlock(apiClient, block.Hash, &pfs.Shard{Number: 0, Modulus: 1}, buffer)
			require.NoError(tb, iErr)

			// buffer = bytes.NewBuffer(nil)
			// require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
			// iErr = pfsutil.GetFile(apiClient, repoName, commitID,
			// 	fmt.Sprintf("a/d/file%d", i), 0, math.MaxInt64, buffer)
			// require.NoError(tb, iErr)
			// require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
		}()
	}
}
Beispiel #13
0
func registerFunc(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) error {
	addresser := route.NewDiscoveryAddresser(
		discoveryClient,
		testNamespace(),
	)
	i := 0
	for address := range servers {
		for j := 0; j < testShardsPerServer; j++ {
			if err := addresser.SetMasterAddress((i*testShardsPerServer)+j, address, 0); err != nil {
				return err
			}
			if err := addresser.SetReplicaAddress((((i+1)%len(servers))*testShardsPerServer)+j, address, 0); err != nil {
				return err
			}
			if err := addresser.SetReplicaAddress((((i+2)%len(servers))*testShardsPerServer)+j, address, 0); err != nil {
				return err
			}
		}
		i++
	}
	for address, s := range servers {
		combinedAPIServer := server.NewCombinedAPIServer(
			route.NewSharder(
				testShardsPerServer*testNumServers,
			),
			route.NewRouter(
				addresser,
				grpcutil.NewDialer(),
				address,
			),
			getDriver(tb, address),
		)
		pfs.RegisterApiServer(s, combinedAPIServer)
		pfs.RegisterInternalApiServer(s, combinedAPIServer)
	}
	return nil
}
Beispiel #14
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	discoveryClient, err := getEtcdClient()
	if err != nil {
		return err
	}
	address := appEnv.Address
	if address == "" {
		address, err = netutil.ExternalIP()
		if err != nil {
			return err
		}
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	addresser := route.NewDiscoveryAddresser(
		discoveryClient,
		"namespace",
	)
	for i := 0; i < appEnv.NumShards; i++ {
		if _, err := addresser.SetMasterAddress(i, route.Address{address, false}); err != nil {
			return err
		}
	}
	var driver drive.Driver
	switch appEnv.DriverType {
	case "btrfs":
		driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
	}
	combinedAPIServer := server.NewCombinedAPIServer(
		route.NewSharder(
			appEnv.NumShards,
		),
		route.NewRouter(
			addresser,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		driver,
	)
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pfs.RegisterApiServer(s, combinedAPIServer)
			pfs.RegisterInternalApiServer(s, combinedAPIServer)
		},
		protoserver.ServeOptions{
			HTTPPort:  uint16(appEnv.HTTPPort),
			TracePort: uint16(appEnv.TracePort),
			Version:   pachyderm.Version,
			HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
				return pfs.RegisterApiHandler(ctx, mux, clientConn)
			},
		},
	)
}
Beispiel #15
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	discoveryClient, err := getEtcdClient(appEnv)
	if err != nil {
		return err
	}
	address := appEnv.Address
	if address == "" {
		address, err = netutil.ExternalIP()
		if err != nil {
			return err
		}
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	sharder := shard.NewSharder(
		discoveryClient,
		appEnv.NumShards,
		0,
		"namespace",
	)
	objdAddress, err := getObjdAddress(appEnv)
	if err != nil {
		return err
	}
	clientConn, err := grpc.Dial(objdAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	objAPIClient := pfs.NewBlockAPIClient(clientConn)
	driver, err := drive.NewDriver(objAPIClient)
	if err != nil {
		return err
	}
	apiServer := server.NewAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			1,
		),
		route.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
	)
	go func() {
		if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil {
			protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
		}
	}()
	internalAPIServer := server.NewInternalAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			1,
		),
		route.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		driver,
	)
	go func() {
		if err := sharder.Register(nil, address, internalAPIServer); err != nil {
			protolion.Printf("Error from sharder.Register %s", err.Error())
		}
	}()
	return protoserver.ServeWithHTTP(
		func(s *grpc.Server) {
			pfs.RegisterAPIServer(s, apiServer)
			pfs.RegisterInternalAPIServer(s, internalAPIServer)
		},
		func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
			return pfs.RegisterAPIHandler(ctx, mux, clientConn)
		},
		protoserver.ServeWithHTTPOptions{
			ServeOptions: protoserver.ServeOptions{
				Version: pachyderm.Version,
			},
		},
		protoserver.ServeEnv{
			GRPCPort: appEnv.Port,
		},
		pkghttp.HandlerEnv{
			Port: appEnv.HTTPPort,
		},
	)
}
Beispiel #16
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	discoveryClient, err := getEtcdClient()
	if err != nil {
		return err
	}
	address := appEnv.Address
	if address == "" {
		address, err = netutil.ExternalIP()
		if err != nil {
			return err
		}
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	sharder := shard.NewSharder(
		discoveryClient,
		appEnv.NumShards,
		appEnv.NumReplicas,
		"namespace",
	)
	var driver drive.Driver
	switch appEnv.DriverType {
	case "btrfs":
		driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
	}
	apiServer := server.NewAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			0,
		),
		route.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
	)
	go func() {
		if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil {
			protolog.Printf("Error from sharder.RegisterFrontend %s", err.Error())
		}
	}()
	internalAPIServer := server.NewInternalAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			0,
		),
		route.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		driver,
	)
	go func() {
		if err := sharder.Register(nil, address, internalAPIServer); err != nil {
			protolog.Printf("Error from sharder.Register %s", err.Error())
		}
	}()
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pfs.RegisterAPIServer(s, apiServer)
			pfs.RegisterInternalAPIServer(s, internalAPIServer)
		},
		protoserver.ServeOptions{
			HTTPPort:  uint16(appEnv.HTTPPort),
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
			HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
				return pfs.RegisterAPIHandler(ctx, mux, clientConn)
			},
		},
	)
}
Beispiel #17
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	logrus.Register()
	discoveryClient, err := getEtcdClient()
	if err != nil {
		return err
	}
	address := appEnv.Address
	if address == "" {
		address, err = netutil.ExternalIP()
		if err != nil {
			return err
		}
	}
	sharder := route.NewSharder(appEnv.NumShards, 0)
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	addresser := route.NewDiscoveryAddresser(
		discoveryClient,
		sharder,
		"namespace",
	)
	var driver drive.Driver
	switch appEnv.DriverType {
	case "btrfs":
		driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
	}
	apiServer := server.NewAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			0,
		),
		route.NewRouter(
			addresser,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
	)
	internalAPIServer := server.NewInternalAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			0,
		),
		route.NewRouter(
			addresser,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		driver,
	)
	go func() {
		if err := addresser.Register(nil, "id", address, internalAPIServer); err != nil {
			log.Print(err)
		}
	}()
	go func() {
		if err := addresser.AssignRoles(nil); err != nil {
			log.Print(err)
		}
	}()
	// TODO(pedge): no!
	trace.AuthRequest = func(_ *http.Request) (bool, bool) {
		return true, true
	}
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pfs.RegisterApiServer(s, apiServer)
			pfs.RegisterInternalApiServer(s, internalAPIServer)
		},
		protoserver.ServeOptions{
			HTTPPort:  uint16(appEnv.HTTPPort),
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
			HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
				return pfs.RegisterApiHandler(ctx, mux, clientConn)
			},
		},
	)
}