func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) Cluster {
cluster := cluster{
rolers: make(map[string]role.Roler),
servers: make(map[string]server.CombinedAPIServer),
addresser: route.NewDiscoveryAddresser(
discoveryClient,
testNamespace(),
),
sharder: route.NewSharder(
testShardsPerServer * testNumServers,
),
tb: tb,
}
for address, s := range servers {
combinedAPIServer := server.NewCombinedAPIServer(
cluster.sharder,
route.NewRouter(
cluster.addresser,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
getDriver(tb, address),
)
pfs.RegisterApiServer(s, combinedAPIServer)
pfs.RegisterInternalApiServer(s, combinedAPIServer)
roler := role.NewRoler(cluster.addresser, cluster.sharder, combinedAPIServer, address, testNumReplicas)
go func() { require.Equal(tb, roler.Run(), discovery.ErrCancelled) }()
cluster.addresses = append(cluster.addresses, address)
cluster.rolers[address] = roler
cluster.servers[address] = combinedAPIServer
}
return &cluster
}
func registerFunc(driver drive.Driver, discoveryClient discovery.Client, servers map[string]*grpc.Server) {
addresser := route.NewDiscoveryAddresser(
discoveryClient,
testNamespace(),
)
i := 0
for address := range servers {
for j := 0; j < testShardsPerServer; j++ {
// TODO(pedge): error
_ = addresser.SetMasterAddress((i*testShardsPerServer)+j, address, 0)
}
i++
}
for address, s := range servers {
combinedAPIServer := server.NewCombinedAPIServer(
route.NewSharder(
testShardsPerServer*testNumServers,
),
route.NewRouter(
addresser,
grpcutil.NewDialer(),
address,
),
driver,
)
pfs.RegisterApiServer(s, combinedAPIServer)
pfs.RegisterInternalApiServer(s, combinedAPIServer)
}
}
func do() error {
runtime.GOMAXPROCS(runtime.NumCPU())
appEnv := &appEnv{}
if err := env.Populate(appEnv, env.PopulateOptions{}); err != nil {
return err
}
if appEnv.NumShards == 0 {
appEnv.NumShards = defaultNumShards
}
var btrfsAPI btrfs.API
switch appEnv.BtrfsDriverType {
case "exec":
btrfsAPI = btrfs.NewExecAPI()
case "ffi":
fallthrough
default:
btrfsAPI = btrfs.NewFFIAPI()
}
address := fmt.Sprintf("0.0.0.0:%d", appEnv.APIPort)
combinedAPIServer := server.NewCombinedAPIServer(
route.NewSharder(
appEnv.NumShards,
),
route.NewRouter(
route.NewSingleAddresser(
address,
appEnv.NumShards,
),
route.NewDialer(),
address,
),
drive.NewBtrfsDriver(
appEnv.BtrfsRoot,
btrfsAPI,
),
)
server := grpc.NewServer(grpc.MaxConcurrentStreams(math.MaxUint32))
pfs.RegisterApiServer(server, combinedAPIServer)
pfs.RegisterInternalApiServer(server, combinedAPIServer)
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", appEnv.APIPort))
if err != nil {
return err
}
errC := make(chan error)
go func() { errC <- server.Serve(listener) }()
go func() { errC <- http.ListenAndServe(":8080", nil) }()
if appEnv.TracePort != 0 {
go func() { errC <- http.ListenAndServe(fmt.Sprintf(":%d", appEnv.TracePort), nil) }()
}
return <-errC
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
discoveryClient, err := getEtcdClient()
if err != nil {
return err
}
address := fmt.Sprintf("0.0.0.0:%d", appEnv.APIPort)
addresser := route.NewDiscoveryAddresser(
discoveryClient,
"namespace",
)
for i := 0; i < appEnv.NumShards; i++ {
if err := addresser.SetMasterAddress(i, address, 0); err != nil {
return err
}
}
var driver drive.Driver
switch appEnv.DriverType {
case "btrfs":
driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
if err != nil {
return err
}
default:
return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
}
combinedAPIServer := server.NewCombinedAPIServer(
route.NewSharder(
appEnv.NumShards,
),
route.NewRouter(
addresser,
grpcutil.NewDialer(),
address,
),
driver,
)
return grpcutil.GrpcDo(
appEnv.APIPort,
appEnv.TracePort,
pachyderm.Version,
func(s *grpc.Server) {
pfs.RegisterApiServer(s, combinedAPIServer)
pfs.RegisterInternalApiServer(s, combinedAPIServer)
},
)
}
func registerFunc(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) error {
addresser := route.NewDiscoveryAddresser(
discoveryClient,
testNamespace(),
)
i := 0
for address := range servers {
for j := 0; j < testShardsPerServer; j++ {
if err := addresser.SetMasterAddress((i*testShardsPerServer)+j, address, 0); err != nil {
return err
}
if err := addresser.SetReplicaAddress((((i+1)%len(servers))*testShardsPerServer)+j, address, 0); err != nil {
return err
}
if err := addresser.SetReplicaAddress((((i+2)%len(servers))*testShardsPerServer)+j, address, 0); err != nil {
return err
}
}
i++
}
for address, s := range servers {
combinedAPIServer := server.NewCombinedAPIServer(
route.NewSharder(
testShardsPerServer*testNumServers,
),
route.NewRouter(
addresser,
grpcutil.NewDialer(),
address,
),
getDriver(tb, address),
)
pfs.RegisterApiServer(s, combinedAPIServer)
pfs.RegisterInternalApiServer(s, combinedAPIServer)
}
return nil
}