func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) *cluster {
realSharder := shard.NewTestSharder(
discoveryClient,
testShardsPerServer*testNumServers,
testNumReplicas,
testNamespace(),
)
sharder := route.NewSharder(
testShardsPerServer*testNumServers,
testNumReplicas,
)
cluster := cluster{
servers: make(map[string]server.APIServer),
internalServers: make(map[string]server.InternalAPIServer),
cancels: make(map[string]chan bool),
internalCancels: make(map[string]chan bool),
cancel: make(chan bool),
realSharder: realSharder,
sharder: sharder,
tb: tb,
}
for address, s := range servers {
cluster.addresses = append(cluster.addresses, address)
router := route.NewRouter(
cluster.realSharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
)
apiServer := server.NewAPIServer(
cluster.sharder,
router,
)
cluster.servers[address] = apiServer
cluster.cancels[address] = make(chan bool)
go func(address string) {
require.Equal(tb, cluster.realSharder.RegisterFrontend(cluster.cancels[address], address, cluster.servers[address]), shard.ErrCancelled)
}(address)
pfs.RegisterAPIServer(s, apiServer)
internalAPIServer := server.NewInternalAPIServer(
cluster.sharder,
router,
getDriver(tb, address),
)
pfs.RegisterInternalAPIServer(s, internalAPIServer)
cluster.internalServers[address] = internalAPIServer
cluster.internalCancels[address] = make(chan bool)
go func(address string) {
require.Equal(tb, cluster.realSharder.Register(cluster.internalCancels[address], address, cluster.internalServers[address]), shard.ErrCancelled)
}(address)
}
return &cluster
}
func (c *cluster) Restart(index int) {
address := c.addresses[index]
c.cancels[address] = make(chan bool)
internalAPIServer := server.NewInternalAPIServer(
c.sharder,
route.NewRouter(
c.realSharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
getDriver(c.tb, address),
)
c.internalServers[address] = internalAPIServer
go func() {
require.Equal(c.tb, c.realSharder.Register(c.cancels[address], address, c.internalServers[address]), shard.ErrCancelled)
}()
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
etcdClient := getEtcdClient(appEnv)
rethinkAPIServer, err := getRethinkAPIServer(appEnv)
if err != nil {
return err
}
kubeClient, err := getKubeClient(appEnv)
if err != nil {
return err
}
address, err := netutil.ExternalIP()
if err != nil {
return err
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
sharder := shard.NewSharder(
etcdClient,
appEnv.NumShards,
appEnv.Namespace,
)
go func() {
if err := sharder.AssignRoles(address, nil); err != nil {
protolion.Printf("Error from sharder.AssignRoles: %s", err.Error())
}
}()
driver, err := drive.NewDriver(address)
if err != nil {
return err
}
apiServer := pfs_server.NewAPIServer(
pfs.NewHasher(
appEnv.NumShards,
1,
),
shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
go func() {
if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{apiServer}); err != nil {
protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
}
}()
internalAPIServer := pfs_server.NewInternalAPIServer(
pfs.NewHasher(
appEnv.NumShards,
1,
),
shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
go func() {
if err := sharder.Register(nil, address, []shard.Server{internalAPIServer}); err != nil {
protolion.Printf("Error from sharder.Register %s", err.Error())
}
}()
ppsAPIServer := pps_server.NewAPIServer(
pps.NewHasher(appEnv.NumShards, appEnv.NumShards),
shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
address,
rethinkAPIServer,
kubeClient,
)
var blockAPIServer pfs.BlockAPIServer
if err := func() error {
bucket, err := ioutil.ReadFile("/amazon-secret/bucket")
if err != nil {
return err
}
id, err := ioutil.ReadFile("/amazon-secret/id")
if err != nil {
return err
}
secret, err := ioutil.ReadFile("/amazon-secret/secret")
if err != nil {
return err
}
token, err := ioutil.ReadFile("/amazon-secret/token")
if err != nil {
return err
}
region, err := ioutil.ReadFile("/amazon-secret/region")
if err != nil {
return err
}
objClient, err := obj.NewAmazonClient(string(bucket), string(id), string(secret), string(token), string(region))
if err != nil {
return err
}
blockAPIServer, err = pfs_server.NewObjBlockAPIServer(appEnv.StorageRoot, objClient)
if err != nil {
return err
}
return nil
}(); err != nil {
protolion.Errorf("failed to create obj backend, falling back to local")
blockAPIServer, err = pfs_server.NewLocalBlockAPIServer(appEnv.StorageRoot)
if err != nil {
return err
}
}
return protoserver.ServeWithHTTP(
func(s *grpc.Server) {
pfs.RegisterAPIServer(s, apiServer)
pfs.RegisterInternalAPIServer(s, internalAPIServer)
pfs.RegisterBlockAPIServer(s, blockAPIServer)
pps.RegisterAPIServer(s, ppsAPIServer)
},
func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
return pfs.RegisterAPIHandler(ctx, mux, clientConn)
},
protoserver.ServeWithHTTPOptions{
ServeOptions: protoserver.ServeOptions{
Version: pachyderm.Version,
},
},
protoserver.ServeEnv{
GRPCPort: appEnv.Port,
},
pkghttp.HandlerEnv{
Port: appEnv.HTTPPort,
},
)
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
discoveryClient, err := getEtcdClient()
if err != nil {
return err
}
address := appEnv.Address
if address == "" {
address, err = netutil.ExternalIP()
if err != nil {
return err
}
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
sharder := shard.NewSharder(
discoveryClient,
appEnv.NumShards,
appEnv.NumReplicas,
"namespace",
)
var driver drive.Driver
switch appEnv.DriverType {
case "obj":
objdAddress, err := getObjdAddress()
if err != nil {
return err
}
clientConn, err := grpc.Dial(objdAddress, grpc.WithInsecure())
if err != nil {
return err
}
objAPIClient := drive.NewAPIClient(clientConn)
driver, err = obj.NewDriver(objAPIClient)
if err != nil {
return err
}
default:
return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
}
apiServer := server.NewAPIServer(
route.NewSharder(
appEnv.NumShards,
1,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
go func() {
if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil {
protolog.Printf("Error from sharder.RegisterFrontend %s", err.Error())
}
}()
internalAPIServer := server.NewInternalAPIServer(
route.NewSharder(
appEnv.NumShards,
1,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
go func() {
if err := sharder.Register(nil, address, internalAPIServer); err != nil {
protolog.Printf("Error from sharder.Register %s", err.Error())
}
}()
return protoserver.Serve(
uint16(appEnv.Port),
func(s *grpc.Server) {
pfs.RegisterAPIServer(s, apiServer)
pfs.RegisterInternalAPIServer(s, internalAPIServer)
},
protoserver.ServeOptions{
HTTPPort: uint16(appEnv.HTTPPort),
DebugPort: uint16(appEnv.DebugPort),
Version: pachyderm.Version,
HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
return pfs.RegisterAPIHandler(ctx, mux, clientConn)
},
},
)
}
func TestWriteAndRead(t *testing.T) {
t.Parallel()
// don't leave goroutines running
var wg sync.WaitGroup
defer wg.Wait()
tmp, err := ioutil.TempDir("", "pachyderm-test-")
if err != nil {
t.Fatalf("tempdir: %v", err)
}
defer func() {
if err := os.RemoveAll(tmp); err != nil {
t.Errorf("cannot remove tempdir: %v", err)
}
}()
// closed on successful termination
quit := make(chan struct{})
defer close(quit)
listener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("cannot listen: %v", err)
}
defer func() {
_ = listener.Close()
}()
// TODO try to share more of this setup code with various main
// functions
localAddress := listener.Addr().String()
srv := grpc.NewServer()
const (
numShards = 1
)
sharder := shard.NewLocalSharder(localAddress, numShards)
hasher := pfs.NewHasher(numShards, 1)
router := shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
localAddress,
)
blockDir := filepath.Join(tmp, "blocks")
blockServer, err := server.NewLocalBlockAPIServer(blockDir)
if err != nil {
t.Fatalf("NewLocalBlockAPIServer: %v", err)
}
pfs.RegisterBlockAPIServer(srv, blockServer)
driver, err := drive.NewDriver(localAddress)
if err != nil {
t.Fatalf("NewDriver: %v", err)
}
apiServer := server.NewAPIServer(
hasher,
router,
)
pfs.RegisterAPIServer(srv, apiServer)
internalAPIServer := server.NewInternalAPIServer(
hasher,
router,
driver,
)
pfs.RegisterInternalAPIServer(srv, internalAPIServer)
wg.Add(1)
go func() {
defer wg.Done()
if err := srv.Serve(listener); err != nil {
select {
case <-quit:
// orderly shutdown
return
default:
t.Errorf("grpc serve: %v", err)
}
}
}()
clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
if err != nil {
t.Fatalf("grpc dial: %v", err)
}
apiClient := pfs.NewAPIClient(clientConn)
mounter := fuse.NewMounter(localAddress, apiClient)
mountpoint := filepath.Join(tmp, "mnt")
if err := os.Mkdir(mountpoint, 0700); err != nil {
t.Fatalf("mkdir mountpoint: %v", err)
}
ready := make(chan bool)
wg.Add(1)
go func() {
defer wg.Done()
if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
t.Errorf("mount and serve: %v", err)
}
}()
<-ready
defer func() {
if err := mounter.Unmount(mountpoint); err != nil {
t.Errorf("unmount: %v", err)
}
}()
const (
repoName = "foo"
)
if err := pfsutil.CreateRepo(apiClient, repoName); err != nil {
t.Fatalf("CreateRepo: %v", err)
}
commit, err := pfsutil.StartCommit(apiClient, repoName, "")
if err != nil {
t.Fatalf("StartCommit: %v", err)
}
const (
greeting = "Hello, world\n"
)
filePath := filepath.Join(mountpoint, repoName, commit.Id, "greeting")
if err := ioutil.WriteFile(filePath, []byte(greeting), 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
buf, err := ioutil.ReadFile(filePath)
if err != nil {
t.Fatalf("ReadFile: %v", err)
}
if g, e := string(buf), greeting; g != e {
t.Errorf("wrong content: %q != %q", g, e)
}
}
func TestCommitFinishedReadDir(t *testing.T) {
t.Parallel()
// don't leave goroutines running
var wg sync.WaitGroup
defer wg.Wait()
tmp, err := ioutil.TempDir("", "pachyderm-test-")
if err != nil {
t.Fatalf("tempdir: %v", err)
}
defer func() {
if err := os.RemoveAll(tmp); err != nil {
t.Errorf("cannot remove tempdir: %v", err)
}
}()
// closed on successful termination
quit := make(chan struct{})
defer close(quit)
listener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("cannot listen: %v", err)
}
defer func() {
_ = listener.Close()
}()
// TODO try to share more of this setup code with various main
// functions
localAddress := listener.Addr().String()
srv := grpc.NewServer()
const (
numShards = 1
)
sharder := shard.NewLocalSharder(localAddress, numShards)
hasher := pfs.NewHasher(numShards, 1)
router := shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
localAddress,
)
blockDir := filepath.Join(tmp, "blocks")
blockServer, err := server.NewLocalBlockAPIServer(blockDir)
if err != nil {
t.Fatalf("NewLocalBlockAPIServer: %v", err)
}
pfs.RegisterBlockAPIServer(srv, blockServer)
driver, err := drive.NewDriver(localAddress)
if err != nil {
t.Fatalf("NewDriver: %v", err)
}
apiServer := server.NewAPIServer(
hasher,
router,
)
pfs.RegisterAPIServer(srv, apiServer)
internalAPIServer := server.NewInternalAPIServer(
hasher,
router,
driver,
)
pfs.RegisterInternalAPIServer(srv, internalAPIServer)
wg.Add(1)
go func() {
defer wg.Done()
if err := srv.Serve(listener); err != nil {
select {
case <-quit:
// orderly shutdown
return
default:
t.Errorf("grpc serve: %v", err)
}
}
}()
clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
if err != nil {
t.Fatalf("grpc dial: %v", err)
}
apiClient := pfs.NewAPIClient(clientConn)
mounter := fuse.NewMounter(localAddress, apiClient)
mountpoint := filepath.Join(tmp, "mnt")
if err := os.Mkdir(mountpoint, 0700); err != nil {
t.Fatalf("mkdir mountpoint: %v", err)
}
ready := make(chan bool)
wg.Add(1)
go func() {
defer wg.Done()
if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
t.Errorf("mount and serve: %v", err)
}
}()
<-ready
defer func() {
if err := mounter.Unmount(mountpoint); err != nil {
t.Errorf("unmount: %v", err)
}
}()
const (
repoName = "foo"
)
if err := pfsutil.CreateRepo(apiClient, repoName); err != nil {
t.Fatalf("CreateRepo: %v", err)
}
commit, err := pfsutil.StartCommit(apiClient, repoName, "")
if err != nil {
t.Fatalf("StartCommit: %v", err)
}
t.Logf("open commit %v", commit.Id)
const (
greetingName = "greeting"
greeting = "Hello, world\n"
greetingPerm = 0644
)
if err := ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.Id, greetingName), []byte(greeting), greetingPerm); err != nil {
t.Fatalf("WriteFile: %v", err)
}
const (
scriptName = "script"
script = "#!/bin/sh\necho foo\n"
scriptPerm = 0750
)
if err := ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.Id, scriptName), []byte(script), scriptPerm); err != nil {
t.Fatalf("WriteFile: %v", err)
}
if err := pfsutil.FinishCommit(apiClient, repoName, commit.Id); err != nil {
t.Fatalf("FinishCommit: %v", err)
}
if err := fstestutil.CheckDir(filepath.Join(mountpoint, repoName, commit.Id), map[string]fstestutil.FileInfoCheck{
greetingName: func(fi os.FileInfo) error {
// TODO respect greetingPerm
if g, e := fi.Mode(), os.FileMode(0666); g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
if g, e := fi.Size(), int64(len(greeting)); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show fileModTime as mtime
// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
scriptName: func(fi os.FileInfo) error {
// TODO respect scriptPerm
if g, e := fi.Mode(), os.FileMode(0666); g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
if g, e := fi.Size(), int64(len(script)); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show fileModTime as mtime
// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
}); err != nil {
t.Errorf("wrong directory content: %v", err)
}
}
func TestRootReadDir(t *testing.T) {
t.Parallel()
// don't leave goroutines running
var wg sync.WaitGroup
defer wg.Wait()
tmp, err := ioutil.TempDir("", "pachyderm-test-")
if err != nil {
t.Fatalf("tempdir: %v", err)
}
defer func() {
if err := os.RemoveAll(tmp); err != nil {
t.Errorf("cannot remove tempdir: %v", err)
}
}()
// closed on successful termination
quit := make(chan struct{})
defer close(quit)
listener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("cannot listen: %v", err)
}
defer func() {
_ = listener.Close()
}()
// TODO try to share more of this setup code with various main
// functions
localAddress := listener.Addr().String()
srv := grpc.NewServer()
const (
numShards = 1
)
sharder := shard.NewLocalSharder(localAddress, numShards)
hasher := pfs.NewHasher(numShards, 1)
router := shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
localAddress,
)
blockDir := filepath.Join(tmp, "blocks")
blockServer, err := server.NewLocalBlockAPIServer(blockDir)
if err != nil {
t.Fatalf("NewLocalBlockAPIServer: %v", err)
}
pfs.RegisterBlockAPIServer(srv, blockServer)
driver, err := drive.NewDriver(localAddress)
if err != nil {
t.Fatalf("NewDriver: %v", err)
}
apiServer := server.NewAPIServer(
hasher,
router,
)
pfs.RegisterAPIServer(srv, apiServer)
internalAPIServer := server.NewInternalAPIServer(
hasher,
router,
driver,
)
pfs.RegisterInternalAPIServer(srv, internalAPIServer)
wg.Add(1)
go func() {
defer wg.Done()
if err := srv.Serve(listener); err != nil {
select {
case <-quit:
// orderly shutdown
return
default:
t.Errorf("grpc serve: %v", err)
}
}
}()
clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
if err != nil {
t.Fatalf("grpc dial: %v", err)
}
apiClient := pfs.NewAPIClient(clientConn)
mounter := fuse.NewMounter(localAddress, apiClient)
mountpoint := filepath.Join(tmp, "mnt")
if err := os.Mkdir(mountpoint, 0700); err != nil {
t.Fatalf("mkdir mountpoint: %v", err)
}
ready := make(chan bool)
wg.Add(1)
go func() {
defer wg.Done()
if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
t.Errorf("mount and serve: %v", err)
}
}()
<-ready
defer func() {
if err := mounter.Unmount(mountpoint); err != nil {
t.Errorf("unmount: %v", err)
}
}()
if err := pfsutil.CreateRepo(apiClient, "one"); err != nil {
t.Fatalf("CreateRepo: %v", err)
}
if err := pfsutil.CreateRepo(apiClient, "two"); err != nil {
t.Fatalf("CreateRepo: %v", err)
}
if err := fstestutil.CheckDir(mountpoint, map[string]fstestutil.FileInfoCheck{
"one": func(fi os.FileInfo) error {
if g, e := fi.Mode(), os.ModeDir|0555; g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
// TODO show repoSize in repo stat?
if g, e := fi.Size(), int64(0); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show RepoInfo.Created as time
// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
"two": func(fi os.FileInfo) error {
if g, e := fi.Mode(), os.ModeDir|0555; g != e {
return fmt.Errorf("wrong mode: %v != %v", g, e)
}
// TODO show repoSize in repo stat?
if g, e := fi.Size(), int64(0); g != e {
t.Errorf("wrong size: %v != %v", g, e)
}
// TODO show RepoInfo.Created as time
// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
// t.Errorf("wrong mtime: %v != %v", g, e)
// }
return nil
},
}); err != nil {
t.Errorf("wrong directory content: %v", err)
}
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
discoveryClient, err := getEtcdClient(appEnv)
if err != nil {
return err
}
address := appEnv.Address
if address == "" {
address, err = netutil.ExternalIP()
if err != nil {
return err
}
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
sharder := shard.NewSharder(
discoveryClient,
appEnv.NumShards,
0,
"namespace",
)
objdAddress, err := getObjdAddress(appEnv)
if err != nil {
return err
}
clientConn, err := grpc.Dial(objdAddress, grpc.WithInsecure())
if err != nil {
return err
}
objAPIClient := pfs.NewBlockAPIClient(clientConn)
driver, err := drive.NewDriver(objAPIClient)
if err != nil {
return err
}
apiServer := server.NewAPIServer(
route.NewSharder(
appEnv.NumShards,
1,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
go func() {
if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil {
protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
}
}()
internalAPIServer := server.NewInternalAPIServer(
route.NewSharder(
appEnv.NumShards,
1,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
go func() {
if err := sharder.Register(nil, address, internalAPIServer); err != nil {
protolion.Printf("Error from sharder.Register %s", err.Error())
}
}()
return protoserver.ServeWithHTTP(
func(s *grpc.Server) {
pfs.RegisterAPIServer(s, apiServer)
pfs.RegisterInternalAPIServer(s, internalAPIServer)
},
func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
return pfs.RegisterAPIHandler(ctx, mux, clientConn)
},
protoserver.ServeWithHTTPOptions{
ServeOptions: protoserver.ServeOptions{
Version: pachyderm.Version,
},
},
protoserver.ServeEnv{
GRPCPort: appEnv.Port,
},
pkghttp.HandlerEnv{
Port: appEnv.HTTPPort,
},
)
}