func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) Cluster {
cluster := cluster{
rolers: make(map[string]role.Roler),
servers: make(map[string]server.CombinedAPIServer),
addresser: route.NewDiscoveryAddresser(
discoveryClient,
testNamespace(),
),
sharder: route.NewSharder(
testShardsPerServer * testNumServers,
),
tb: tb,
}
for address, s := range servers {
combinedAPIServer := server.NewCombinedAPIServer(
cluster.sharder,
route.NewRouter(
cluster.addresser,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
getDriver(tb, address),
)
pfs.RegisterApiServer(s, combinedAPIServer)
pfs.RegisterInternalApiServer(s, combinedAPIServer)
roler := role.NewRoler(cluster.addresser, cluster.sharder, combinedAPIServer, address, testNumReplicas)
go func() { require.Equal(tb, roler.Run(), discovery.ErrCancelled) }()
cluster.addresses = append(cluster.addresses, address)
cluster.rolers[address] = roler
cluster.servers[address] = combinedAPIServer
}
return &cluster
}
func registerFunc(driver drive.Driver, discoveryClient discovery.Client, servers map[string]*grpc.Server) {
addresser := route.NewDiscoveryAddresser(
discoveryClient,
testNamespace(),
)
i := 0
for address := range servers {
for j := 0; j < testShardsPerServer; j++ {
// TODO(pedge): error
_ = addresser.SetMasterAddress((i*testShardsPerServer)+j, address, 0)
}
i++
}
for address, s := range servers {
combinedAPIServer := server.NewCombinedAPIServer(
route.NewSharder(
testShardsPerServer*testNumServers,
),
route.NewRouter(
addresser,
grpcutil.NewDialer(),
address,
),
driver,
)
pfs.RegisterApiServer(s, combinedAPIServer)
pfs.RegisterInternalApiServer(s, combinedAPIServer)
}
}
func runTest(
t *testing.T,
driver drive.Driver,
f func(t *testing.T, apiClient pfs.ApiClient),
) {
grpctest.Run(
t,
testNumServers,
func(servers map[string]*grpc.Server) {
discoveryClient := discovery.NewMockClient()
i := 0
addresses := make([]string, testNumServers)
for address := range servers {
shards := make([]string, testShardsPerServer)
for j := 0; j < testShardsPerServer; j++ {
shards[j] = fmt.Sprintf("%d", (i*testShardsPerServer)+j)
}
_ = discoveryClient.Set(address+"-master", strings.Join(shards, ","))
addresses[i] = address
i++
}
_ = discoveryClient.Set("all-addresses", strings.Join(addresses, ","))
for address, server := range servers {
combinedAPIServer := NewCombinedAPIServer(
route.NewSharder(
testShardsPerServer*testNumServers,
),
route.NewRouter(
route.NewDiscoveryAddresser(
discoveryClient,
),
route.NewDialer(),
address,
),
driver,
)
pfs.RegisterApiServer(server, combinedAPIServer)
pfs.RegisterInternalApiServer(server, combinedAPIServer)
}
},
func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
var clientConn *grpc.ClientConn
for _, c := range clientConns {
clientConn = c
break
}
for _, c := range clientConns {
if c != clientConn {
_ = c.Close()
}
}
f(
t,
pfs.NewApiClient(
clientConn,
),
)
},
)
}
func NewServerGroup(t *testing.T, addresser route.Addresser, numServers int, offset int, numReplicas int) *serverGroup {
sharder := route.NewSharder(testNumShards)
serverGroup := serverGroup{offset: offset}
for i := 0; i < numServers; i++ {
serverGroup.servers = append(serverGroup.servers, newServer(t))
serverGroup.rolers = append(serverGroup.rolers, NewRoler(addresser, sharder, serverGroup.servers[i], fmt.Sprintf("server-%d", i+offset), numReplicas))
}
return &serverGroup
}
func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) *cluster {
realSharder := shard.NewTestSharder(
discoveryClient,
testShardsPerServer*testNumServers,
testNumReplicas,
testNamespace(),
)
sharder := route.NewSharder(
testShardsPerServer*testNumServers,
testNumReplicas,
)
cluster := cluster{
servers: make(map[string]server.APIServer),
internalServers: make(map[string]server.InternalAPIServer),
cancels: make(map[string]chan bool),
internalCancels: make(map[string]chan bool),
cancel: make(chan bool),
realSharder: realSharder,
sharder: sharder,
tb: tb,
}
for address, s := range servers {
cluster.addresses = append(cluster.addresses, address)
router := route.NewRouter(
cluster.realSharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
)
apiServer := server.NewAPIServer(
cluster.sharder,
router,
)
cluster.servers[address] = apiServer
cluster.cancels[address] = make(chan bool)
go func(address string) {
require.Equal(tb, cluster.realSharder.RegisterFrontend(cluster.cancels[address], address, cluster.servers[address]), shard.ErrCancelled)
}(address)
pfs.RegisterAPIServer(s, apiServer)
internalAPIServer := server.NewInternalAPIServer(
cluster.sharder,
router,
getDriver(tb, address),
)
pfs.RegisterInternalAPIServer(s, internalAPIServer)
cluster.internalServers[address] = internalAPIServer
cluster.internalCancels[address] = make(chan bool)
go func(address string) {
require.Equal(tb, cluster.realSharder.Register(cluster.internalCancels[address], address, cluster.internalServers[address]), shard.ErrCancelled)
}(address)
}
return &cluster
}
func (a *internalAPIServer) ListFile(ctx context.Context, request *pfs.ListFileRequest) (response *pfs.FileInfos, err error) {
defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
version, err := a.getVersion(ctx)
if err != nil {
return nil, err
}
shards, err := a.router.GetMasterShards(version)
if err != nil {
return nil, err
}
if request.Shard == nil {
request.Shard = &pfs.Shard{Number: 0, Modulus: 1}
}
sharder := route.NewSharder(request.Shard.Modulus, 0)
var wg sync.WaitGroup
var lock sync.Mutex
var fileInfos []*pfs.FileInfo
seenDirectories := make(map[string]bool)
var loopErr error
for shard := range shards {
wg.Add(1)
go func(shard uint64) {
defer wg.Done()
subFileInfos, err := a.driver.ListFile(request.File, shard)
lock.Lock()
defer lock.Unlock()
if err != nil {
if loopErr == nil {
loopErr = err
}
return
}
for _, fileInfo := range subFileInfos {
if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR {
if seenDirectories[fileInfo.File.Path] {
continue
}
seenDirectories[fileInfo.File.Path] = true
}
if sharder.GetShard(fileInfo.File) == request.Shard.Number {
fileInfos = append(fileInfos, fileInfo)
}
}
}(shard)
}
wg.Wait()
if loopErr != nil {
return nil, loopErr
}
return &pfs.FileInfos{
FileInfo: fileInfos,
}, nil
}
func do() error {
runtime.GOMAXPROCS(runtime.NumCPU())
appEnv := &appEnv{}
if err := env.Populate(appEnv, env.PopulateOptions{}); err != nil {
return err
}
if appEnv.NumShards == 0 {
appEnv.NumShards = defaultNumShards
}
var btrfsAPI btrfs.API
switch appEnv.BtrfsDriverType {
case "exec":
btrfsAPI = btrfs.NewExecAPI()
case "ffi":
fallthrough
default:
btrfsAPI = btrfs.NewFFIAPI()
}
address := fmt.Sprintf("0.0.0.0:%d", appEnv.APIPort)
combinedAPIServer := server.NewCombinedAPIServer(
route.NewSharder(
appEnv.NumShards,
),
route.NewRouter(
route.NewSingleAddresser(
address,
appEnv.NumShards,
),
route.NewDialer(),
address,
),
drive.NewBtrfsDriver(
appEnv.BtrfsRoot,
btrfsAPI,
),
)
server := grpc.NewServer(grpc.MaxConcurrentStreams(math.MaxUint32))
pfs.RegisterApiServer(server, combinedAPIServer)
pfs.RegisterInternalApiServer(server, combinedAPIServer)
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", appEnv.APIPort))
if err != nil {
return err
}
errC := make(chan error)
go func() { errC <- server.Serve(listener) }()
go func() { errC <- http.ListenAndServe(":8080", nil) }()
if appEnv.TracePort != 0 {
go func() { errC <- http.ListenAndServe(fmt.Sprintf(":%d", appEnv.TracePort), nil) }()
}
return <-errC
}
func (a *internalAPIServer) ListFile(ctx context.Context, request *pfs.ListFileRequest) (response *pfs.FileInfos, retErr error) {
defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
version, err := a.getVersion(ctx)
if err != nil {
return nil, err
}
shards, err := a.router.GetMasterShards(version)
if err != nil {
return nil, err
}
if request.Shard == nil {
request.Shard = &pfs.Shard{Number: 0, Modulus: 1}
}
sharder := route.NewSharder(request.Shard.Modulus, 0)
var wg sync.WaitGroup
var lock sync.Mutex
var fileInfos []*pfs.FileInfo
var loopErr error
for shard := range shards {
shard := shard
wg.Add(1)
go func() {
defer wg.Done()
subFileInfos, err := a.driver.ListFile(request.File, shard)
lock.Lock()
defer lock.Unlock()
if err != nil && err != pfs.ErrFileNotFound {
if loopErr == nil {
loopErr = err
}
return
}
for _, fileInfo := range subFileInfos {
if sharder.GetShard(fileInfo.File) == request.Shard.Number ||
fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR {
fileInfos = append(fileInfos, fileInfo)
}
}
}()
}
wg.Wait()
if loopErr != nil {
return nil, loopErr
}
return &pfs.FileInfos{
FileInfo: pfs.ReduceFileInfos(fileInfos),
}, nil
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
discoveryClient, err := getEtcdClient()
if err != nil {
return err
}
address := fmt.Sprintf("0.0.0.0:%d", appEnv.APIPort)
addresser := route.NewDiscoveryAddresser(
discoveryClient,
"namespace",
)
for i := 0; i < appEnv.NumShards; i++ {
if err := addresser.SetMasterAddress(i, address, 0); err != nil {
return err
}
}
var driver drive.Driver
switch appEnv.DriverType {
case "btrfs":
driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
if err != nil {
return err
}
default:
return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
}
combinedAPIServer := server.NewCombinedAPIServer(
route.NewSharder(
appEnv.NumShards,
),
route.NewRouter(
addresser,
grpcutil.NewDialer(),
address,
),
driver,
)
return grpcutil.GrpcDo(
appEnv.APIPort,
appEnv.TracePort,
pachyderm.Version,
func(s *grpc.Server) {
pfs.RegisterApiServer(s, combinedAPIServer)
pfs.RegisterInternalApiServer(s, combinedAPIServer)
},
)
}
func (a *internalAPIServer) ListChange(ctx context.Context, request *pfs.ListChangeRequest) (response *pfs.Changes, err error) {
defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
version, err := a.getVersion(ctx)
if err != nil {
return nil, err
}
shards, err := a.router.GetMasterShards(version)
if err != nil {
return nil, err
}
if request.Shard == nil {
request.Shard = &pfs.Shard{Number: 0, Modulus: 1}
}
sharder := route.NewSharder(request.Shard.Modulus, 0)
var wg sync.WaitGroup
var lock sync.Mutex
var changes []*pfs.Change
var loopErr error
for shard := range shards {
wg.Add(1)
go func(shard uint64) {
defer wg.Done()
subChanges, err := a.driver.ListChange(request.File, request.From, shard)
lock.Lock()
defer lock.Unlock()
if err != nil {
if loopErr == nil {
loopErr = err
}
return
}
for _, change := range subChanges {
if sharder.GetShard(change.File) == request.Shard.Number {
changes = append(changes, change)
}
}
}(shard)
}
wg.Wait()
if loopErr != nil {
return nil, loopErr
}
return &pfs.Changes{
Change: changes,
}, nil
}
func (a *internalAPIServer) ListBlock(ctx context.Context, request *pfs.ListBlockRequest) (response *pfs.BlockInfos, err error) {
defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
version, err := a.getVersion(ctx)
if err != nil {
return nil, err
}
shards, err := a.router.GetMasterShards(version)
if err != nil {
return nil, err
}
if request.Shard == nil {
request.Shard = &pfs.Shard{Number: 0, Modulus: 1}
}
sharder := route.NewSharder(request.Shard.Modulus, 0)
var wg sync.WaitGroup
var lock sync.Mutex
var blockInfos []*pfs.BlockInfo
var loopErr error
for shard := range shards {
wg.Add(1)
go func(shard uint64) {
defer wg.Done()
subBlockInfos, err := a.driver.ListBlock(shard)
lock.Lock()
defer lock.Unlock()
if err != nil {
if loopErr == nil {
loopErr = err
}
return
}
for _, blockInfo := range subBlockInfos {
if request.Shard == nil || sharder.GetBlockShard(blockInfo.Block) == request.Shard.Number {
blockInfos = append(blockInfos, blockInfo)
}
}
}(shard)
}
wg.Wait()
if loopErr != nil {
return nil, loopErr
}
return &pfs.BlockInfos{
BlockInfo: blockInfos,
}, nil
}
func checkBlockWrites(tb testing.TB, apiClient pfs.APIClient, repoName string, commitID string) {
var wg sync.WaitGroup
defer wg.Wait()
for i := 0; i < testSize; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
buffer := bytes.NewBuffer(nil)
sharder := route.NewSharder(testShardsPerServer*testNumServers, testNumReplicas)
block := sharder.GetBlock([]byte(fmt.Sprintf("hello%d", i)))
iErr := pfsutil.GetBlock(apiClient, block.Hash, &pfs.Shard{Number: 0, Modulus: 1}, buffer)
require.NoError(tb, iErr)
// buffer = bytes.NewBuffer(nil)
// require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
// iErr = pfsutil.GetFile(apiClient, repoName, commitID,
// fmt.Sprintf("a/d/file%d", i), 0, math.MaxInt64, buffer)
// require.NoError(tb, iErr)
// require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String())
}()
}
}
func registerFunc(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) error {
addresser := route.NewDiscoveryAddresser(
discoveryClient,
testNamespace(),
)
i := 0
for address := range servers {
for j := 0; j < testShardsPerServer; j++ {
if err := addresser.SetMasterAddress((i*testShardsPerServer)+j, address, 0); err != nil {
return err
}
if err := addresser.SetReplicaAddress((((i+1)%len(servers))*testShardsPerServer)+j, address, 0); err != nil {
return err
}
if err := addresser.SetReplicaAddress((((i+2)%len(servers))*testShardsPerServer)+j, address, 0); err != nil {
return err
}
}
i++
}
for address, s := range servers {
combinedAPIServer := server.NewCombinedAPIServer(
route.NewSharder(
testShardsPerServer*testNumServers,
),
route.NewRouter(
addresser,
grpcutil.NewDialer(),
address,
),
getDriver(tb, address),
)
pfs.RegisterApiServer(s, combinedAPIServer)
pfs.RegisterInternalApiServer(s, combinedAPIServer)
}
return nil
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
discoveryClient, err := getEtcdClient()
if err != nil {
return err
}
address := appEnv.Address
if address == "" {
address, err = netutil.ExternalIP()
if err != nil {
return err
}
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
addresser := route.NewDiscoveryAddresser(
discoveryClient,
"namespace",
)
for i := 0; i < appEnv.NumShards; i++ {
if _, err := addresser.SetMasterAddress(i, route.Address{address, false}); err != nil {
return err
}
}
var driver drive.Driver
switch appEnv.DriverType {
case "btrfs":
driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
if err != nil {
return err
}
default:
return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
}
combinedAPIServer := server.NewCombinedAPIServer(
route.NewSharder(
appEnv.NumShards,
),
route.NewRouter(
addresser,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
return protoserver.Serve(
uint16(appEnv.Port),
func(s *grpc.Server) {
pfs.RegisterApiServer(s, combinedAPIServer)
pfs.RegisterInternalApiServer(s, combinedAPIServer)
},
protoserver.ServeOptions{
HTTPPort: uint16(appEnv.HTTPPort),
TracePort: uint16(appEnv.TracePort),
Version: pachyderm.Version,
HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
return pfs.RegisterApiHandler(ctx, mux, clientConn)
},
},
)
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
discoveryClient, err := getEtcdClient(appEnv)
if err != nil {
return err
}
address := appEnv.Address
if address == "" {
address, err = netutil.ExternalIP()
if err != nil {
return err
}
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
sharder := shard.NewSharder(
discoveryClient,
appEnv.NumShards,
0,
"namespace",
)
objdAddress, err := getObjdAddress(appEnv)
if err != nil {
return err
}
clientConn, err := grpc.Dial(objdAddress, grpc.WithInsecure())
if err != nil {
return err
}
objAPIClient := pfs.NewBlockAPIClient(clientConn)
driver, err := drive.NewDriver(objAPIClient)
if err != nil {
return err
}
apiServer := server.NewAPIServer(
route.NewSharder(
appEnv.NumShards,
1,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
go func() {
if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil {
protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
}
}()
internalAPIServer := server.NewInternalAPIServer(
route.NewSharder(
appEnv.NumShards,
1,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
go func() {
if err := sharder.Register(nil, address, internalAPIServer); err != nil {
protolion.Printf("Error from sharder.Register %s", err.Error())
}
}()
return protoserver.ServeWithHTTP(
func(s *grpc.Server) {
pfs.RegisterAPIServer(s, apiServer)
pfs.RegisterInternalAPIServer(s, internalAPIServer)
},
func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
return pfs.RegisterAPIHandler(ctx, mux, clientConn)
},
protoserver.ServeWithHTTPOptions{
ServeOptions: protoserver.ServeOptions{
Version: pachyderm.Version,
},
},
protoserver.ServeEnv{
GRPCPort: appEnv.Port,
},
pkghttp.HandlerEnv{
Port: appEnv.HTTPPort,
},
)
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
discoveryClient, err := getEtcdClient()
if err != nil {
return err
}
address := appEnv.Address
if address == "" {
address, err = netutil.ExternalIP()
if err != nil {
return err
}
}
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
sharder := shard.NewSharder(
discoveryClient,
appEnv.NumShards,
appEnv.NumReplicas,
"namespace",
)
var driver drive.Driver
switch appEnv.DriverType {
case "btrfs":
driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
if err != nil {
return err
}
default:
return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
}
apiServer := server.NewAPIServer(
route.NewSharder(
appEnv.NumShards,
0,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
go func() {
if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil {
protolog.Printf("Error from sharder.RegisterFrontend %s", err.Error())
}
}()
internalAPIServer := server.NewInternalAPIServer(
route.NewSharder(
appEnv.NumShards,
0,
),
route.NewRouter(
sharder,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
go func() {
if err := sharder.Register(nil, address, internalAPIServer); err != nil {
protolog.Printf("Error from sharder.Register %s", err.Error())
}
}()
return protoserver.Serve(
uint16(appEnv.Port),
func(s *grpc.Server) {
pfs.RegisterAPIServer(s, apiServer)
pfs.RegisterInternalAPIServer(s, internalAPIServer)
},
protoserver.ServeOptions{
HTTPPort: uint16(appEnv.HTTPPort),
DebugPort: uint16(appEnv.DebugPort),
Version: pachyderm.Version,
HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
return pfs.RegisterAPIHandler(ctx, mux, clientConn)
},
},
)
}
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
logrus.Register()
discoveryClient, err := getEtcdClient()
if err != nil {
return err
}
address := appEnv.Address
if address == "" {
address, err = netutil.ExternalIP()
if err != nil {
return err
}
}
sharder := route.NewSharder(appEnv.NumShards, 0)
address = fmt.Sprintf("%s:%d", address, appEnv.Port)
addresser := route.NewDiscoveryAddresser(
discoveryClient,
sharder,
"namespace",
)
var driver drive.Driver
switch appEnv.DriverType {
case "btrfs":
driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
if err != nil {
return err
}
default:
return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
}
apiServer := server.NewAPIServer(
route.NewSharder(
appEnv.NumShards,
0,
),
route.NewRouter(
addresser,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
)
internalAPIServer := server.NewInternalAPIServer(
route.NewSharder(
appEnv.NumShards,
0,
),
route.NewRouter(
addresser,
grpcutil.NewDialer(
grpc.WithInsecure(),
),
address,
),
driver,
)
go func() {
if err := addresser.Register(nil, "id", address, internalAPIServer); err != nil {
log.Print(err)
}
}()
go func() {
if err := addresser.AssignRoles(nil); err != nil {
log.Print(err)
}
}()
// TODO(pedge): no!
trace.AuthRequest = func(_ *http.Request) (bool, bool) {
return true, true
}
return protoserver.Serve(
uint16(appEnv.Port),
func(s *grpc.Server) {
pfs.RegisterApiServer(s, apiServer)
pfs.RegisterInternalApiServer(s, internalAPIServer)
},
protoserver.ServeOptions{
HTTPPort: uint16(appEnv.HTTPPort),
DebugPort: uint16(appEnv.DebugPort),
Version: pachyderm.Version,
HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
return pfs.RegisterApiHandler(ctx, mux, clientConn)
},
},
)
}