func testFuse(
t *testing.T,
test func(client client.APIClient, mountpoint string),
) {
// don't leave goroutines running
var wg sync.WaitGroup
defer wg.Wait()
tmp, err := ioutil.TempDir("", "pachyderm-test-")
require.NoError(t, err)
defer func() {
_ = os.RemoveAll(tmp)
}()
// closed on successful termination
quit := make(chan struct{})
defer close(quit)
listener, err := net.Listen("tcp", "localhost:0")
require.NoError(t, err)
defer func() {
_ = listener.Close()
}()
// TODO try to share more of this setup code with various main
// functions
localAddress := listener.Addr().String()
srv := grpc.NewServer()
const (
numShards = 1
)
sharder := shard.NewLocalSharder([]string{localAddress}, numShards)
hasher := pfsserver.NewHasher(numShards, 1)
router := shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
localAddress,
)
blockDir := filepath.Join(tmp, "blocks")
blockServer, err := server.NewLocalBlockAPIServer(blockDir)
require.NoError(t, err)
pfsclient.RegisterBlockAPIServer(srv, blockServer)
driver, err := drive.NewDriver(localAddress)
require.NoError(t, err)
apiServer := server.NewAPIServer(
hasher,
router,
)
pfsclient.RegisterAPIServer(srv, apiServer)
internalAPIServer := server.NewInternalAPIServer(
hasher,
router,
driver,
)
pfsclient.RegisterInternalAPIServer(srv, internalAPIServer)
wg.Add(1)
go func() {
defer wg.Done()
if err := srv.Serve(listener); err != nil {
select {
case <-quit:
// orderly shutdown
return
default:
t.Errorf("grpc serve: %v", err)
}
}
}()
clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
require.NoError(t, err)
apiClient := pfsclient.NewAPIClient(clientConn)
mounter := fuse.NewMounter(localAddress, apiClient)
mountpoint := filepath.Join(tmp, "mnt")
require.NoError(t, os.Mkdir(mountpoint, 0700))
ready := make(chan bool)
wg.Add(1)
go func() {
defer wg.Done()
require.NoError(t, mounter.MountAndCreate(mountpoint, nil, nil, ready, true))
}()
<-ready
defer func() {
_ = mounter.Unmount(mountpoint)
}()
test(client.APIClient{PfsAPIClient: apiClient}, mountpoint)
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
etcdClient := getEtcdClient(appEnv)
if appEnv.Init {
if err := setClusterID(etcdClient); err != nil {
return fmt.Errorf("error connecting to etcd, if this error persists it likely indicates that kubernetes services are not working correctly. See https://github.com/pachyderm/pachyderm/blob/master/SETUP.md#pachd-or-pachd-init-crash-loop-with-error-connecting-to-etcd for more info")
}
if err := persist_server.InitDBs(fmt.Sprintf("%s:28015", appEnv.DatabaseAddress), appEnv.DatabaseName); err != nil && !isDBCreated(err) {
return err
}
return nil
}
if readinessCheck {
//c, err := client.NewInCluster()
c, err := client.NewFromAddress("127.0.0.1:650")
if err != nil {
return err
}
_, err = c.ListRepo(nil)
if err != nil {
return err
}
os.Exit(0)
return nil
}
clusterID, err := getClusterID(etcdClient)
if err != nil {
return err
}
kubeClient, err := getKubeClient(appEnv)
if err != nil {
return err
}
if appEnv.Metrics {
go metrics.ReportMetrics(clusterID, kubeClient)
}
rethinkAPIServer, err := getRethinkAPIServer(appEnv)
if err != nil {
return err
}
address, err := netutil.ExternalIP()
if err != nil {
return err
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
sharder := shard.NewSharder(
etcdClient,
appEnv.NumShards,
appEnv.Namespace,
)
go func() {
if err := sharder.AssignRoles(address, nil); err != nil {
protolion.Printf("Error from sharder.AssignRoles: %s", err.Error())
}
}()
driver, err := drive.NewDriver(address)
if err != nil {
return err
}
apiServer := pfs_server.NewAPIServer(
pfsmodel.NewHasher(
appEnv.NumShards,
1,
),
shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
go func() {
if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{apiServer}); err != nil {
protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
}
}()
internalAPIServer := pfs_server.NewInternalAPIServer(
pfsmodel.NewHasher(
appEnv.NumShards,
1,
),
shard.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
ppsAPIServer := pps_server.NewAPIServer(
ppsserver.NewHasher(appEnv.NumShards, appEnv.NumShards),
address,
kubeClient,
getNamespace(),
)
go func() {
if err := sharder.Register(nil, address, []shard.Server{internalAPIServer, ppsAPIServer}); err != nil {
protolion.Printf("Error from sharder.Register %s", err.Error())
}
}()
blockAPIServer, err := pfs_server.NewBlockAPIServer(appEnv.StorageRoot, appEnv.StorageBackend)
if err != nil {
return err
}
return protoserver.Serve(
func(s *grpc.Server) {
pfsclient.RegisterAPIServer(s, apiServer)
pfsclient.RegisterInternalAPIServer(s, internalAPIServer)
pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
ppsclient.RegisterAPIServer(s, ppsAPIServer)
ppsserver.RegisterInternalJobAPIServer(s, ppsAPIServer)
persist.RegisterAPIServer(s, rethinkAPIServer)
},
protoserver.ServeOptions{
Version: version.Version,
},
protoserver.ServeEnv{
GRPCPort: appEnv.Port,
},
)
}