func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
discoveryClient, err := getEtcdClient()
if err != nil {
return err
}
sharder := shard.NewSharder(
discoveryClient,
appEnv.NumShards,
appEnv.NumReplicas,
"namespace",
)
return sharder.AssignRoles(nil)
}
func runMasterReplicaTest(t *testing.T, client discovery.Client) {
sharder := shard.NewSharder(client, testNumShards, testNumReplicas, "TestMasterReplica")
cancel := make(chan bool)
go func() {
require.Equal(t, shard.ErrCancelled, sharder.AssignRoles(cancel))
}()
defer func() {
close(cancel)
}()
serverGroup1 := NewServerGroup(t, sharder, testNumServers/2, 0)
go serverGroup1.run(t)
start := time.Now()
for !serverGroup1.satisfied((testNumShards * (testNumReplicas + 1)) / (testNumServers / 2)) {
time.Sleep(500 * time.Millisecond)
if time.Since(start) > time.Second*time.Duration(30) {
t.Fatal("test timed out")
}
}
serverGroup2 := NewServerGroup(t, sharder, testNumServers/2, testNumServers/2)
go serverGroup2.run(t)
start = time.Now()
for !serverGroup1.satisfied((testNumShards*(testNumReplicas+1))/testNumServers) ||
!serverGroup2.satisfied((testNumShards*(testNumReplicas+1))/testNumServers) {
time.Sleep(time.Second)
if time.Since(start) > time.Second*time.Duration(60) {
t.Fatal("test timed out")
}
}
close(serverGroup1.cancel)
for !serverGroup2.satisfied((testNumShards * (testNumReplicas + 1)) / (testNumServers / 2)) {
time.Sleep(500 * time.Millisecond)
if time.Since(start) > time.Second*time.Duration(60) {
t.Fatal("test timed out")
}
}
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
etcdClient := getEtcdClient(appEnv)
rethinkAPIServer, err := getRethinkAPIServer(appEnv)
if err != nil {
return err
}
kubeClient, err := getKubeClient(appEnv)
if err != nil {
return err
}
address, err := netutil.ExternalIP()
if err != nil {
return err
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
sharder := shard.NewSharder(
etcdClient,
appEnv.NumShards,
appEnv.Namespace,
)
go func() {
if err := sharder.AssignRoles(address, nil); err != nil {
protolion.Printf("Error from sharder.AssignRoles: %s", err.Error())
}
}()
driver, err := drive.NewDriver(address)
if err != nil {
return err
}
apiServer := pfs_server.NewAPIServer(
pfs.NewHasher(
appEnv.NumShards,
1,
),
shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
go func() {
if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{apiServer}); err != nil {
protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
}
}()
internalAPIServer := pfs_server.NewInternalAPIServer(
pfs.NewHasher(
appEnv.NumShards,
1,
),
shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
go func() {
if err := sharder.Register(nil, address, []shard.Server{internalAPIServer}); err != nil {
protolion.Printf("Error from sharder.Register %s", err.Error())
}
}()
ppsAPIServer := pps_server.NewAPIServer(
pps.NewHasher(appEnv.NumShards, appEnv.NumShards),
shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
address,
rethinkAPIServer,
kubeClient,
)
var blockAPIServer pfs.BlockAPIServer
if err := func() error {
bucket, err := ioutil.ReadFile("/amazon-secret/bucket")
if err != nil {
return err
}
id, err := ioutil.ReadFile("/amazon-secret/id")
if err != nil {
return err
}
secret, err := ioutil.ReadFile("/amazon-secret/secret")
if err != nil {
return err
}
token, err := ioutil.ReadFile("/amazon-secret/token")
if err != nil {
return err
}
region, err := ioutil.ReadFile("/amazon-secret/region")
if err != nil {
return err
}
objClient, err := obj.NewAmazonClient(string(bucket), string(id), string(secret), string(token), string(region))
if err != nil {
return err
}
blockAPIServer, err = pfs_server.NewObjBlockAPIServer(appEnv.StorageRoot, objClient)
if err != nil {
return err
}
return nil
}(); err != nil {
protolion.Errorf("failed to create obj backend, falling back to local")
blockAPIServer, err = pfs_server.NewLocalBlockAPIServer(appEnv.StorageRoot)
if err != nil {
return err
}
}
return protoserver.ServeWithHTTP(
func(s *grpc.Server) {
pfs.RegisterAPIServer(s, apiServer)
pfs.RegisterInternalAPIServer(s, internalAPIServer)
pfs.RegisterBlockAPIServer(s, blockAPIServer)
pps.RegisterAPIServer(s, ppsAPIServer)
},
func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
return pfs.RegisterAPIHandler(ctx, mux, clientConn)
},
protoserver.ServeWithHTTPOptions{
ServeOptions: protoserver.ServeOptions{
Version: pachyderm.Version,
},
},
protoserver.ServeEnv{
GRPCPort: appEnv.Port,
},
pkghttp.HandlerEnv{
Port: appEnv.HTTPPort,
},
)
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
discoveryClient, err := getEtcdClient()
if err != nil {
return err
}
address := appEnv.Address
if address == "" {
address, err = netutil.ExternalIP()
if err != nil {
return err
}
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
sharder := shard.NewSharder(
discoveryClient,
appEnv.NumShards,
appEnv.NumReplicas,
"namespace",
)
var driver drive.Driver
switch appEnv.DriverType {
case "btrfs":
driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
if err != nil {
return err
}
default:
return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
}
apiServer := server.NewAPIServer(
route.NewSharder(
appEnv.NumShards,
0,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
go func() {
if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil {
protolog.Printf("Error from sharder.RegisterFrontend %s", err.Error())
}
}()
internalAPIServer := server.NewInternalAPIServer(
route.NewSharder(
appEnv.NumShards,
0,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
go func() {
if err := sharder.Register(nil, address, internalAPIServer); err != nil {
protolog.Printf("Error from sharder.Register %s", err.Error())
}
}()
return protoserver.Serve(
uint16(appEnv.Port),
func(s *grpc.Server) {
pfs.RegisterAPIServer(s, apiServer)
pfs.RegisterInternalAPIServer(s, internalAPIServer)
},
protoserver.ServeOptions{
HTTPPort: uint16(appEnv.HTTPPort),
DebugPort: uint16(appEnv.DebugPort),
Version: pachyderm.Version,
HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
return pfs.RegisterAPIHandler(ctx, mux, clientConn)
},
},
)
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
discoveryClient, err := getEtcdClient(appEnv)
if err != nil {
return err
}
address := appEnv.Address
if address == "" {
address, err = netutil.ExternalIP()
if err != nil {
return err
}
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
sharder := shard.NewSharder(
discoveryClient,
appEnv.NumShards,
0,
"namespace",
)
objdAddress, err := getObjdAddress(appEnv)
if err != nil {
return err
}
clientConn, err := grpc.Dial(objdAddress, grpc.WithInsecure())
if err != nil {
return err
}
objAPIClient := pfs.NewBlockAPIClient(clientConn)
driver, err := drive.NewDriver(objAPIClient)
if err != nil {
return err
}
apiServer := server.NewAPIServer(
route.NewSharder(
appEnv.NumShards,
1,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
go func() {
if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil {
protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
}
}()
internalAPIServer := server.NewInternalAPIServer(
route.NewSharder(
appEnv.NumShards,
1,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
go func() {
if err := sharder.Register(nil, address, internalAPIServer); err != nil {
protolion.Printf("Error from sharder.Register %s", err.Error())
}
}()
return protoserver.ServeWithHTTP(
func(s *grpc.Server) {
pfs.RegisterAPIServer(s, apiServer)
pfs.RegisterInternalAPIServer(s, internalAPIServer)
},
func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
return pfs.RegisterAPIHandler(ctx, mux, clientConn)
},
protoserver.ServeWithHTTPOptions{
ServeOptions: protoserver.ServeOptions{
Version: pachyderm.Version,
},
},
protoserver.ServeEnv{
GRPCPort: appEnv.Port,
},
pkghttp.HandlerEnv{
Port: appEnv.HTTPPort,
},
)
}