func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) Cluster { cluster := cluster{ rolers: make(map[string]role.Roler), servers: make(map[string]server.CombinedAPIServer), addresser: route.NewDiscoveryAddresser( discoveryClient, testNamespace(), ), sharder: route.NewSharder( testShardsPerServer * testNumServers, ), tb: tb, } for address, s := range servers { combinedAPIServer := server.NewCombinedAPIServer( cluster.sharder, route.NewRouter( cluster.addresser, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ), getDriver(tb, address), ) pfs.RegisterApiServer(s, combinedAPIServer) pfs.RegisterInternalApiServer(s, combinedAPIServer) roler := role.NewRoler(cluster.addresser, cluster.sharder, combinedAPIServer, address, testNumReplicas) go func() { require.Equal(tb, roler.Run(), discovery.ErrCancelled) }() cluster.addresses = append(cluster.addresses, address) cluster.rolers[address] = roler cluster.servers[address] = combinedAPIServer } return &cluster }
func registerFunc(driver drive.Driver, discoveryClient discovery.Client, servers map[string]*grpc.Server) { addresser := route.NewDiscoveryAddresser( discoveryClient, testNamespace(), ) i := 0 for address := range servers { for j := 0; j < testShardsPerServer; j++ { // TODO(pedge): error _ = addresser.SetMasterAddress((i*testShardsPerServer)+j, address, 0) } i++ } for address, s := range servers { combinedAPIServer := server.NewCombinedAPIServer( route.NewSharder( testShardsPerServer*testNumServers, ), route.NewRouter( addresser, grpcutil.NewDialer(), address, ), driver, ) pfs.RegisterApiServer(s, combinedAPIServer) pfs.RegisterInternalApiServer(s, combinedAPIServer) } }
func runTest( t *testing.T, driver drive.Driver, f func(t *testing.T, apiClient pfs.ApiClient), ) { grpctest.Run( t, testNumServers, func(servers map[string]*grpc.Server) { discoveryClient := discovery.NewMockClient() i := 0 addresses := make([]string, testNumServers) for address := range servers { shards := make([]string, testShardsPerServer) for j := 0; j < testShardsPerServer; j++ { shards[j] = fmt.Sprintf("%d", (i*testShardsPerServer)+j) } _ = discoveryClient.Set(address+"-master", strings.Join(shards, ",")) addresses[i] = address i++ } _ = discoveryClient.Set("all-addresses", strings.Join(addresses, ",")) for address, server := range servers { combinedAPIServer := NewCombinedAPIServer( route.NewSharder( testShardsPerServer*testNumServers, ), route.NewRouter( route.NewDiscoveryAddresser( discoveryClient, ), route.NewDialer(), address, ), driver, ) pfs.RegisterApiServer(server, combinedAPIServer) pfs.RegisterInternalApiServer(server, combinedAPIServer) } }, func(t *testing.T, clientConns map[string]*grpc.ClientConn) { var clientConn *grpc.ClientConn for _, c := range clientConns { clientConn = c break } for _, c := range clientConns { if c != clientConn { _ = c.Close() } } f( t, pfs.NewApiClient( clientConn, ), ) }, ) }
func NewServerGroup(t *testing.T, addresser route.Addresser, numServers int, offset int, numReplicas int) *serverGroup { sharder := route.NewSharder(testNumShards) serverGroup := serverGroup{offset: offset} for i := 0; i < numServers; i++ { serverGroup.servers = append(serverGroup.servers, newServer(t)) serverGroup.rolers = append(serverGroup.rolers, NewRoler(addresser, sharder, serverGroup.servers[i], fmt.Sprintf("server-%d", i+offset), numReplicas)) } return &serverGroup }
func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) *cluster { realSharder := shard.NewTestSharder( discoveryClient, testShardsPerServer*testNumServers, testNumReplicas, testNamespace(), ) sharder := route.NewSharder( testShardsPerServer*testNumServers, testNumReplicas, ) cluster := cluster{ servers: make(map[string]server.APIServer), internalServers: make(map[string]server.InternalAPIServer), cancels: make(map[string]chan bool), internalCancels: make(map[string]chan bool), cancel: make(chan bool), realSharder: realSharder, sharder: sharder, tb: tb, } for address, s := range servers { cluster.addresses = append(cluster.addresses, address) router := route.NewRouter( cluster.realSharder, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ) apiServer := server.NewAPIServer( cluster.sharder, router, ) cluster.servers[address] = apiServer cluster.cancels[address] = make(chan bool) go func(address string) { require.Equal(tb, cluster.realSharder.RegisterFrontend(cluster.cancels[address], address, cluster.servers[address]), shard.ErrCancelled) }(address) pfs.RegisterAPIServer(s, apiServer) internalAPIServer := server.NewInternalAPIServer( cluster.sharder, router, getDriver(tb, address), ) pfs.RegisterInternalAPIServer(s, internalAPIServer) cluster.internalServers[address] = internalAPIServer cluster.internalCancels[address] = make(chan bool) go func(address string) { require.Equal(tb, cluster.realSharder.Register(cluster.internalCancels[address], address, cluster.internalServers[address]), shard.ErrCancelled) }(address) } return &cluster }
func (a *internalAPIServer) ListFile(ctx context.Context, request *pfs.ListFileRequest) (response *pfs.FileInfos, err error) { defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now()) version, err := a.getVersion(ctx) if err != nil { return nil, err } shards, err := a.router.GetMasterShards(version) if err != nil { return nil, err } if request.Shard == nil { request.Shard = &pfs.Shard{Number: 0, Modulus: 1} } sharder := route.NewSharder(request.Shard.Modulus, 0) var wg sync.WaitGroup var lock sync.Mutex var fileInfos []*pfs.FileInfo seenDirectories := make(map[string]bool) var loopErr error for shard := range shards { wg.Add(1) go func(shard uint64) { defer wg.Done() subFileInfos, err := a.driver.ListFile(request.File, shard) lock.Lock() defer lock.Unlock() if err != nil { if loopErr == nil { loopErr = err } return } for _, fileInfo := range subFileInfos { if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR { if seenDirectories[fileInfo.File.Path] { continue } seenDirectories[fileInfo.File.Path] = true } if sharder.GetShard(fileInfo.File) == request.Shard.Number { fileInfos = append(fileInfos, fileInfo) } } }(shard) } wg.Wait() if loopErr != nil { return nil, loopErr } return &pfs.FileInfos{ FileInfo: fileInfos, }, nil }
func do() error { runtime.GOMAXPROCS(runtime.NumCPU()) appEnv := &appEnv{} if err := env.Populate(appEnv, env.PopulateOptions{}); err != nil { return err } if appEnv.NumShards == 0 { appEnv.NumShards = defaultNumShards } var btrfsAPI btrfs.API switch appEnv.BtrfsDriverType { case "exec": btrfsAPI = btrfs.NewExecAPI() case "ffi": fallthrough default: btrfsAPI = btrfs.NewFFIAPI() } address := fmt.Sprintf("0.0.0.0:%d", appEnv.APIPort) combinedAPIServer := server.NewCombinedAPIServer( route.NewSharder( appEnv.NumShards, ), route.NewRouter( route.NewSingleAddresser( address, appEnv.NumShards, ), route.NewDialer(), address, ), drive.NewBtrfsDriver( appEnv.BtrfsRoot, btrfsAPI, ), ) server := grpc.NewServer(grpc.MaxConcurrentStreams(math.MaxUint32)) pfs.RegisterApiServer(server, combinedAPIServer) pfs.RegisterInternalApiServer(server, combinedAPIServer) listener, err := net.Listen("tcp", fmt.Sprintf(":%d", appEnv.APIPort)) if err != nil { return err } errC := make(chan error) go func() { errC <- server.Serve(listener) }() go func() { errC <- http.ListenAndServe(":8080", nil) }() if appEnv.TracePort != 0 { go func() { errC <- http.ListenAndServe(fmt.Sprintf(":%d", appEnv.TracePort), nil) }() } return <-errC }
func (a *internalAPIServer) ListFile(ctx context.Context, request *pfs.ListFileRequest) (response *pfs.FileInfos, retErr error) { defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now()) version, err := a.getVersion(ctx) if err != nil { return nil, err } shards, err := a.router.GetMasterShards(version) if err != nil { return nil, err } if request.Shard == nil { request.Shard = &pfs.Shard{Number: 0, Modulus: 1} } sharder := route.NewSharder(request.Shard.Modulus, 0) var wg sync.WaitGroup var lock sync.Mutex var fileInfos []*pfs.FileInfo var loopErr error for shard := range shards { shard := shard wg.Add(1) go func() { defer wg.Done() subFileInfos, err := a.driver.ListFile(request.File, shard) lock.Lock() defer lock.Unlock() if err != nil && err != pfs.ErrFileNotFound { if loopErr == nil { loopErr = err } return } for _, fileInfo := range subFileInfos { if sharder.GetShard(fileInfo.File) == request.Shard.Number || fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR { fileInfos = append(fileInfos, fileInfo) } } }() } wg.Wait() if loopErr != nil { return nil, loopErr } return &pfs.FileInfos{ FileInfo: pfs.ReduceFileInfos(fileInfos), }, nil }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) discoveryClient, err := getEtcdClient() if err != nil { return err } address := fmt.Sprintf("0.0.0.0:%d", appEnv.APIPort) addresser := route.NewDiscoveryAddresser( discoveryClient, "namespace", ) for i := 0; i < appEnv.NumShards; i++ { if err := addresser.SetMasterAddress(i, address, 0); err != nil { return err } } var driver drive.Driver switch appEnv.DriverType { case "btrfs": driver, err = btrfs.NewDriver(appEnv.DriverRoot, "") if err != nil { return err } default: return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType) } combinedAPIServer := server.NewCombinedAPIServer( route.NewSharder( appEnv.NumShards, ), route.NewRouter( addresser, grpcutil.NewDialer(), address, ), driver, ) return grpcutil.GrpcDo( appEnv.APIPort, appEnv.TracePort, pachyderm.Version, func(s *grpc.Server) { pfs.RegisterApiServer(s, combinedAPIServer) pfs.RegisterInternalApiServer(s, combinedAPIServer) }, ) }
func (a *internalAPIServer) ListChange(ctx context.Context, request *pfs.ListChangeRequest) (response *pfs.Changes, err error) { defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now()) version, err := a.getVersion(ctx) if err != nil { return nil, err } shards, err := a.router.GetMasterShards(version) if err != nil { return nil, err } if request.Shard == nil { request.Shard = &pfs.Shard{Number: 0, Modulus: 1} } sharder := route.NewSharder(request.Shard.Modulus, 0) var wg sync.WaitGroup var lock sync.Mutex var changes []*pfs.Change var loopErr error for shard := range shards { wg.Add(1) go func(shard uint64) { defer wg.Done() subChanges, err := a.driver.ListChange(request.File, request.From, shard) lock.Lock() defer lock.Unlock() if err != nil { if loopErr == nil { loopErr = err } return } for _, change := range subChanges { if sharder.GetShard(change.File) == request.Shard.Number { changes = append(changes, change) } } }(shard) } wg.Wait() if loopErr != nil { return nil, loopErr } return &pfs.Changes{ Change: changes, }, nil }
func (a *internalAPIServer) ListBlock(ctx context.Context, request *pfs.ListBlockRequest) (response *pfs.BlockInfos, err error) { defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now()) version, err := a.getVersion(ctx) if err != nil { return nil, err } shards, err := a.router.GetMasterShards(version) if err != nil { return nil, err } if request.Shard == nil { request.Shard = &pfs.Shard{Number: 0, Modulus: 1} } sharder := route.NewSharder(request.Shard.Modulus, 0) var wg sync.WaitGroup var lock sync.Mutex var blockInfos []*pfs.BlockInfo var loopErr error for shard := range shards { wg.Add(1) go func(shard uint64) { defer wg.Done() subBlockInfos, err := a.driver.ListBlock(shard) lock.Lock() defer lock.Unlock() if err != nil { if loopErr == nil { loopErr = err } return } for _, blockInfo := range subBlockInfos { if request.Shard == nil || sharder.GetBlockShard(blockInfo.Block) == request.Shard.Number { blockInfos = append(blockInfos, blockInfo) } } }(shard) } wg.Wait() if loopErr != nil { return nil, loopErr } return &pfs.BlockInfos{ BlockInfo: blockInfos, }, nil }
func checkBlockWrites(tb testing.TB, apiClient pfs.APIClient, repoName string, commitID string) { var wg sync.WaitGroup defer wg.Wait() for i := 0; i < testSize; i++ { i := i wg.Add(1) go func() { defer wg.Done() buffer := bytes.NewBuffer(nil) sharder := route.NewSharder(testShardsPerServer*testNumServers, testNumReplicas) block := sharder.GetBlock([]byte(fmt.Sprintf("hello%d", i))) iErr := pfsutil.GetBlock(apiClient, block.Hash, &pfs.Shard{Number: 0, Modulus: 1}, buffer) require.NoError(tb, iErr) // buffer = bytes.NewBuffer(nil) // require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String()) // iErr = pfsutil.GetFile(apiClient, repoName, commitID, // fmt.Sprintf("a/d/file%d", i), 0, math.MaxInt64, buffer) // require.NoError(tb, iErr) // require.Equal(tb, fmt.Sprintf("hello%d", i), buffer.String()) }() } }
func registerFunc(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) error { addresser := route.NewDiscoveryAddresser( discoveryClient, testNamespace(), ) i := 0 for address := range servers { for j := 0; j < testShardsPerServer; j++ { if err := addresser.SetMasterAddress((i*testShardsPerServer)+j, address, 0); err != nil { return err } if err := addresser.SetReplicaAddress((((i+1)%len(servers))*testShardsPerServer)+j, address, 0); err != nil { return err } if err := addresser.SetReplicaAddress((((i+2)%len(servers))*testShardsPerServer)+j, address, 0); err != nil { return err } } i++ } for address, s := range servers { combinedAPIServer := server.NewCombinedAPIServer( route.NewSharder( testShardsPerServer*testNumServers, ), route.NewRouter( addresser, grpcutil.NewDialer(), address, ), getDriver(tb, address), ) pfs.RegisterApiServer(s, combinedAPIServer) pfs.RegisterInternalApiServer(s, combinedAPIServer) } return nil }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) discoveryClient, err := getEtcdClient() if err != nil { return err } address := appEnv.Address if address == "" { address, err = netutil.ExternalIP() if err != nil { return err } } address = fmt.Sprintf("%s:%d", address, appEnv.Port) addresser := route.NewDiscoveryAddresser( discoveryClient, "namespace", ) for i := 0; i < appEnv.NumShards; i++ { if _, err := addresser.SetMasterAddress(i, route.Address{address, false}); err != nil { return err } } var driver drive.Driver switch appEnv.DriverType { case "btrfs": driver, err = btrfs.NewDriver(appEnv.DriverRoot, "") if err != nil { return err } default: return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType) } combinedAPIServer := server.NewCombinedAPIServer( route.NewSharder( appEnv.NumShards, ), route.NewRouter( addresser, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ), driver, ) return protoserver.Serve( uint16(appEnv.Port), func(s *grpc.Server) { pfs.RegisterApiServer(s, combinedAPIServer) pfs.RegisterInternalApiServer(s, combinedAPIServer) }, protoserver.ServeOptions{ HTTPPort: uint16(appEnv.HTTPPort), TracePort: uint16(appEnv.TracePort), Version: pachyderm.Version, HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error { return pfs.RegisterApiHandler(ctx, mux, clientConn) }, }, ) }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) discoveryClient, err := getEtcdClient(appEnv) if err != nil { return err } address := appEnv.Address if address == "" { address, err = netutil.ExternalIP() if err != nil { return err } } address = fmt.Sprintf("%s:%d", address, appEnv.Port) sharder := shard.NewSharder( discoveryClient, appEnv.NumShards, 0, "namespace", ) objdAddress, err := getObjdAddress(appEnv) if err != nil { return err } clientConn, err := grpc.Dial(objdAddress, grpc.WithInsecure()) if err != nil { return err } objAPIClient := pfs.NewBlockAPIClient(clientConn) driver, err := drive.NewDriver(objAPIClient) if err != nil { return err } apiServer := server.NewAPIServer( route.NewSharder( appEnv.NumShards, 1, ), route.NewRouter( sharder, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ), ) go func() { if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil { protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error()) } }() internalAPIServer := server.NewInternalAPIServer( route.NewSharder( appEnv.NumShards, 1, ), route.NewRouter( sharder, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ), driver, ) go func() { if err := sharder.Register(nil, address, internalAPIServer); err != nil { protolion.Printf("Error from sharder.Register %s", err.Error()) } }() return protoserver.ServeWithHTTP( func(s *grpc.Server) { pfs.RegisterAPIServer(s, apiServer) pfs.RegisterInternalAPIServer(s, internalAPIServer) }, func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error { return pfs.RegisterAPIHandler(ctx, mux, clientConn) }, protoserver.ServeWithHTTPOptions{ ServeOptions: protoserver.ServeOptions{ Version: pachyderm.Version, }, }, protoserver.ServeEnv{ GRPCPort: appEnv.Port, }, pkghttp.HandlerEnv{ Port: appEnv.HTTPPort, }, ) }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) discoveryClient, err := getEtcdClient() if err != nil { return err } address := appEnv.Address if address == "" { address, err = netutil.ExternalIP() if err != nil { return err } } address = fmt.Sprintf("%s:%d", address, appEnv.Port) sharder := shard.NewSharder( discoveryClient, appEnv.NumShards, appEnv.NumReplicas, "namespace", ) var driver drive.Driver switch appEnv.DriverType { case "btrfs": driver, err = btrfs.NewDriver(appEnv.DriverRoot, "") if err != nil { return err } default: return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType) } apiServer := server.NewAPIServer( route.NewSharder( appEnv.NumShards, 0, ), route.NewRouter( sharder, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ), ) go func() { if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil { protolog.Printf("Error from sharder.RegisterFrontend %s", err.Error()) } }() internalAPIServer := server.NewInternalAPIServer( route.NewSharder( appEnv.NumShards, 0, ), route.NewRouter( sharder, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ), driver, ) go func() { if err := sharder.Register(nil, address, internalAPIServer); err != nil { protolog.Printf("Error from sharder.Register %s", err.Error()) } }() return protoserver.Serve( uint16(appEnv.Port), func(s *grpc.Server) { pfs.RegisterAPIServer(s, apiServer) pfs.RegisterInternalAPIServer(s, internalAPIServer) }, protoserver.ServeOptions{ HTTPPort: uint16(appEnv.HTTPPort), DebugPort: uint16(appEnv.DebugPort), Version: pachyderm.Version, HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error { return pfs.RegisterAPIHandler(ctx, mux, clientConn) }, }, ) }
func do(appEnvObj interface{}) error { appEnv := appEnvObj.(*appEnv) logrus.Register() discoveryClient, err := getEtcdClient() if err != nil { return err } address := appEnv.Address if address == "" { address, err = netutil.ExternalIP() if err != nil { return err } } sharder := route.NewSharder(appEnv.NumShards, 0) address = fmt.Sprintf("%s:%d", address, appEnv.Port) addresser := route.NewDiscoveryAddresser( discoveryClient, sharder, "namespace", ) var driver drive.Driver switch appEnv.DriverType { case "btrfs": driver, err = btrfs.NewDriver(appEnv.DriverRoot, "") if err != nil { return err } default: return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType) } apiServer := server.NewAPIServer( route.NewSharder( appEnv.NumShards, 0, ), route.NewRouter( addresser, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ), ) internalAPIServer := server.NewInternalAPIServer( route.NewSharder( appEnv.NumShards, 0, ), route.NewRouter( addresser, grpcutil.NewDialer( grpc.WithInsecure(), ), address, ), driver, ) go func() { if err := addresser.Register(nil, "id", address, internalAPIServer); err != nil { log.Print(err) } }() go func() { if err := addresser.AssignRoles(nil); err != nil { log.Print(err) } }() // TODO(pedge): no! trace.AuthRequest = func(_ *http.Request) (bool, bool) { return true, true } return protoserver.Serve( uint16(appEnv.Port), func(s *grpc.Server) { pfs.RegisterApiServer(s, apiServer) pfs.RegisterInternalApiServer(s, internalAPIServer) }, protoserver.ServeOptions{ HTTPPort: uint16(appEnv.HTTPPort), DebugPort: uint16(appEnv.DebugPort), Version: pachyderm.Version, HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error { return pfs.RegisterApiHandler(ctx, mux, clientConn) }, }, ) }