Beispiel #1
0
func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) Cluster {
	sharder := route.NewSharder(
		testShardsPerServer*testNumServers,
		testNumReplicas,
	)
	addresser := route.NewDiscoveryTestAddresser(discoveryClient, sharder, testNamespace())
	cluster := cluster{
		servers:         make(map[string]server.APIServer),
		internalServers: make(map[string]server.InternalAPIServer),
		cancels:         make(map[string]chan bool),
		cancel:          make(chan bool),
		addresser:       addresser,
		sharder:         sharder,
		tb:              tb,
	}
	for address, s := range servers {
		apiServer := server.NewAPIServer(
			cluster.sharder,
			route.NewRouter(
				cluster.addresser,
				grpcutil.NewDialer(
					grpc.WithInsecure(),
				),
				address,
			),
		)
		pfs.RegisterApiServer(s, apiServer)
		internalAPIServer := server.NewInternalAPIServer(
			cluster.sharder,
			route.NewRouter(
				cluster.addresser,
				grpcutil.NewDialer(
					grpc.WithInsecure(),
				),
				address,
			),
			getDriver(tb, address),
		)
		pfs.RegisterInternalApiServer(s, internalAPIServer)
		cluster.addresses = append(cluster.addresses, address)
		cluster.servers[address] = apiServer
		cluster.internalServers[address] = internalAPIServer
		cluster.cancels[address] = make(chan bool)
		go func(address string) {
			require.Equal(tb, cluster.addresser.Register(cluster.cancels[address], address, address, cluster.internalServers[address]), route.ErrCancelled)
		}(address)
	}
	go func() {
		require.Equal(tb, cluster.addresser.AssignRoles(cluster.cancel), route.ErrCancelled)
	}()
	return &cluster
}
Beispiel #2
0
func newCluster(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) Cluster {
	cluster := cluster{
		rolers:  make(map[string]role.Roler),
		servers: make(map[string]server.CombinedAPIServer),
		addresser: route.NewDiscoveryAddresser(
			discoveryClient,
			testNamespace(),
		),
		sharder: route.NewSharder(
			testShardsPerServer * testNumServers,
		),
		tb: tb,
	}
	for address, s := range servers {
		combinedAPIServer := server.NewCombinedAPIServer(
			cluster.sharder,
			route.NewRouter(
				cluster.addresser,
				grpcutil.NewDialer(
					grpc.WithInsecure(),
				),
				address,
			),
			getDriver(tb, address),
		)
		pfs.RegisterApiServer(s, combinedAPIServer)
		pfs.RegisterInternalApiServer(s, combinedAPIServer)
		roler := role.NewRoler(cluster.addresser, cluster.sharder, combinedAPIServer, address, testNumReplicas)
		go func() { require.Equal(tb, roler.Run(), discovery.ErrCancelled) }()
		cluster.addresses = append(cluster.addresses, address)
		cluster.rolers[address] = roler
		cluster.servers[address] = combinedAPIServer
	}
	return &cluster
}
Beispiel #3
0
func registerFunc(driver drive.Driver, discoveryClient discovery.Client, servers map[string]*grpc.Server) {
	addresser := route.NewDiscoveryAddresser(
		discoveryClient,
		testNamespace(),
	)
	i := 0
	for address := range servers {
		for j := 0; j < testShardsPerServer; j++ {
			// TODO(pedge): error
			_ = addresser.SetMasterAddress((i*testShardsPerServer)+j, address, 0)
		}
		i++
	}
	for address, s := range servers {
		combinedAPIServer := server.NewCombinedAPIServer(
			route.NewSharder(
				testShardsPerServer*testNumServers,
			),
			route.NewRouter(
				addresser,
				grpcutil.NewDialer(),
				address,
			),
			driver,
		)
		pfs.RegisterApiServer(s, combinedAPIServer)
		pfs.RegisterInternalApiServer(s, combinedAPIServer)
	}
}
Beispiel #4
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	discoveryClient, err := getEtcdClient()
	if err != nil {
		return err
	}
	address := fmt.Sprintf("0.0.0.0:%d", appEnv.APIPort)
	addresser := route.NewDiscoveryAddresser(
		discoveryClient,
		"namespace",
	)
	for i := 0; i < appEnv.NumShards; i++ {
		if err := addresser.SetMasterAddress(i, address, 0); err != nil {
			return err
		}
	}
	var driver drive.Driver
	switch appEnv.DriverType {
	case "btrfs":
		driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
	}
	combinedAPIServer := server.NewCombinedAPIServer(
		route.NewSharder(
			appEnv.NumShards,
		),
		route.NewRouter(
			addresser,
			grpcutil.NewDialer(),
			address,
		),
		driver,
	)
	return grpcutil.GrpcDo(
		appEnv.APIPort,
		appEnv.TracePort,
		pachyderm.Version,
		func(s *grpc.Server) {
			pfs.RegisterApiServer(s, combinedAPIServer)
			pfs.RegisterInternalApiServer(s, combinedAPIServer)
		},
	)
}
Beispiel #5
0
func (c *cluster) Restart(index int) {
	address := c.addresses[index]
	c.cancels[address] = make(chan bool)
	internalAPIServer := server.NewInternalAPIServer(
		c.sharder,
		route.NewRouter(
			c.realSharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		getDriver(c.tb, address),
	)
	c.internalServers[address] = internalAPIServer
	go func() {
		require.Equal(c.tb, c.realSharder.Register(c.cancels[address], address, c.internalServers[address]), shard.ErrCancelled)
	}()
}
Beispiel #6
0
func registerFunc(tb testing.TB, discoveryClient discovery.Client, servers map[string]*grpc.Server) error {
	addresser := route.NewDiscoveryAddresser(
		discoveryClient,
		testNamespace(),
	)
	i := 0
	for address := range servers {
		for j := 0; j < testShardsPerServer; j++ {
			if err := addresser.SetMasterAddress((i*testShardsPerServer)+j, address, 0); err != nil {
				return err
			}
			if err := addresser.SetReplicaAddress((((i+1)%len(servers))*testShardsPerServer)+j, address, 0); err != nil {
				return err
			}
			if err := addresser.SetReplicaAddress((((i+2)%len(servers))*testShardsPerServer)+j, address, 0); err != nil {
				return err
			}
		}
		i++
	}
	for address, s := range servers {
		combinedAPIServer := server.NewCombinedAPIServer(
			route.NewSharder(
				testShardsPerServer*testNumServers,
			),
			route.NewRouter(
				addresser,
				grpcutil.NewDialer(),
				address,
			),
			getDriver(tb, address),
		)
		pfs.RegisterApiServer(s, combinedAPIServer)
		pfs.RegisterInternalApiServer(s, combinedAPIServer)
	}
	return nil
}
Beispiel #7
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	etcdClient := getEtcdClient(appEnv)
	rethinkAPIServer, err := getRethinkAPIServer(appEnv)
	if err != nil {
		return err
	}
	kubeClient, err := getKubeClient(appEnv)
	if err != nil {
		return err
	}
	address, err := netutil.ExternalIP()
	if err != nil {
		return err
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	sharder := shard.NewSharder(
		etcdClient,
		appEnv.NumShards,
		appEnv.Namespace,
	)
	go func() {
		if err := sharder.AssignRoles(address, nil); err != nil {
			protolion.Printf("Error from sharder.AssignRoles: %s", err.Error())
		}
	}()
	driver, err := drive.NewDriver(address)
	if err != nil {
		return err
	}
	apiServer := pfs_server.NewAPIServer(
		pfs.NewHasher(
			appEnv.NumShards,
			1,
		),
		shard.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
	)
	go func() {
		if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{apiServer}); err != nil {
			protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
		}
	}()
	internalAPIServer := pfs_server.NewInternalAPIServer(
		pfs.NewHasher(
			appEnv.NumShards,
			1,
		),
		shard.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		driver,
	)
	go func() {
		if err := sharder.Register(nil, address, []shard.Server{internalAPIServer}); err != nil {
			protolion.Printf("Error from sharder.Register %s", err.Error())
		}
	}()
	ppsAPIServer := pps_server.NewAPIServer(
		pps.NewHasher(appEnv.NumShards, appEnv.NumShards),
		shard.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		address,
		rethinkAPIServer,
		kubeClient,
	)
	var blockAPIServer pfs.BlockAPIServer
	if err := func() error {
		bucket, err := ioutil.ReadFile("/amazon-secret/bucket")
		if err != nil {
			return err
		}
		id, err := ioutil.ReadFile("/amazon-secret/id")
		if err != nil {
			return err
		}
		secret, err := ioutil.ReadFile("/amazon-secret/secret")
		if err != nil {
			return err
		}
		token, err := ioutil.ReadFile("/amazon-secret/token")
		if err != nil {
			return err
		}
		region, err := ioutil.ReadFile("/amazon-secret/region")
		if err != nil {
			return err
		}
		objClient, err := obj.NewAmazonClient(string(bucket), string(id), string(secret), string(token), string(region))
		if err != nil {
			return err
		}
		blockAPIServer, err = pfs_server.NewObjBlockAPIServer(appEnv.StorageRoot, objClient)
		if err != nil {
			return err
		}
		return nil
	}(); err != nil {
		protolion.Errorf("failed to create obj backend, falling back to local")
		blockAPIServer, err = pfs_server.NewLocalBlockAPIServer(appEnv.StorageRoot)
		if err != nil {
			return err
		}
	}
	return protoserver.ServeWithHTTP(
		func(s *grpc.Server) {
			pfs.RegisterAPIServer(s, apiServer)
			pfs.RegisterInternalAPIServer(s, internalAPIServer)
			pfs.RegisterBlockAPIServer(s, blockAPIServer)
			pps.RegisterAPIServer(s, ppsAPIServer)
		},
		func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
			return pfs.RegisterAPIHandler(ctx, mux, clientConn)
		},
		protoserver.ServeWithHTTPOptions{
			ServeOptions: protoserver.ServeOptions{
				Version: pachyderm.Version,
			},
		},
		protoserver.ServeEnv{
			GRPCPort: appEnv.Port,
		},
		pkghttp.HandlerEnv{
			Port: appEnv.HTTPPort,
		},
	)
}
Beispiel #8
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	discoveryClient, err := getEtcdClient()
	if err != nil {
		return err
	}
	address := appEnv.Address
	if address == "" {
		address, err = netutil.ExternalIP()
		if err != nil {
			return err
		}
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	sharder := shard.NewSharder(
		discoveryClient,
		appEnv.NumShards,
		appEnv.NumReplicas,
		"namespace",
	)
	var driver drive.Driver
	switch appEnv.DriverType {
	case "btrfs":
		driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
	}
	apiServer := server.NewAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			0,
		),
		route.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
	)
	go func() {
		if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil {
			protolog.Printf("Error from sharder.RegisterFrontend %s", err.Error())
		}
	}()
	internalAPIServer := server.NewInternalAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			0,
		),
		route.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		driver,
	)
	go func() {
		if err := sharder.Register(nil, address, internalAPIServer); err != nil {
			protolog.Printf("Error from sharder.Register %s", err.Error())
		}
	}()
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pfs.RegisterAPIServer(s, apiServer)
			pfs.RegisterInternalAPIServer(s, internalAPIServer)
		},
		protoserver.ServeOptions{
			HTTPPort:  uint16(appEnv.HTTPPort),
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
			HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
				return pfs.RegisterAPIHandler(ctx, mux, clientConn)
			},
		},
	)
}
Beispiel #9
0
func TestWriteAndRead(t *testing.T) {
	t.Parallel()

	// don't leave goroutines running
	var wg sync.WaitGroup
	defer wg.Wait()

	tmp, err := ioutil.TempDir("", "pachyderm-test-")
	if err != nil {
		t.Fatalf("tempdir: %v", err)
	}
	defer func() {
		if err := os.RemoveAll(tmp); err != nil {
			t.Errorf("cannot remove tempdir: %v", err)
		}
	}()

	// closed on successful termination
	quit := make(chan struct{})
	defer close(quit)
	listener, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("cannot listen: %v", err)
	}
	defer func() {
		_ = listener.Close()
	}()

	// TODO try to share more of this setup code with various main
	// functions
	localAddress := listener.Addr().String()
	srv := grpc.NewServer()
	const (
		numShards = 1
	)
	sharder := shard.NewLocalSharder(localAddress, numShards)
	hasher := pfs.NewHasher(numShards, 1)
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		localAddress,
	)

	blockDir := filepath.Join(tmp, "blocks")
	blockServer, err := server.NewLocalBlockAPIServer(blockDir)
	if err != nil {
		t.Fatalf("NewLocalBlockAPIServer: %v", err)
	}
	pfs.RegisterBlockAPIServer(srv, blockServer)

	driver, err := drive.NewDriver(localAddress)
	if err != nil {
		t.Fatalf("NewDriver: %v", err)
	}

	apiServer := server.NewAPIServer(
		hasher,
		router,
	)
	pfs.RegisterAPIServer(srv, apiServer)

	internalAPIServer := server.NewInternalAPIServer(
		hasher,
		router,
		driver,
	)
	pfs.RegisterInternalAPIServer(srv, internalAPIServer)

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := srv.Serve(listener); err != nil {
			select {
			case <-quit:
				// orderly shutdown
				return
			default:
				t.Errorf("grpc serve: %v", err)
			}
		}
	}()

	clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
	if err != nil {
		t.Fatalf("grpc dial: %v", err)
	}
	apiClient := pfs.NewAPIClient(clientConn)
	mounter := fuse.NewMounter(localAddress, apiClient)

	mountpoint := filepath.Join(tmp, "mnt")
	if err := os.Mkdir(mountpoint, 0700); err != nil {
		t.Fatalf("mkdir mountpoint: %v", err)
	}

	ready := make(chan bool)
	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
			t.Errorf("mount and serve: %v", err)
		}
	}()

	<-ready
	defer func() {
		if err := mounter.Unmount(mountpoint); err != nil {
			t.Errorf("unmount: %v", err)
		}
	}()

	const (
		repoName = "foo"
	)
	if err := pfsutil.CreateRepo(apiClient, repoName); err != nil {
		t.Fatalf("CreateRepo: %v", err)
	}
	commit, err := pfsutil.StartCommit(apiClient, repoName, "")
	if err != nil {
		t.Fatalf("StartCommit: %v", err)
	}

	const (
		greeting = "Hello, world\n"
	)
	filePath := filepath.Join(mountpoint, repoName, commit.Id, "greeting")

	if err := ioutil.WriteFile(filePath, []byte(greeting), 0644); err != nil {
		t.Fatalf("WriteFile: %v", err)
	}

	buf, err := ioutil.ReadFile(filePath)
	if err != nil {
		t.Fatalf("ReadFile: %v", err)
	}
	if g, e := string(buf), greeting; g != e {
		t.Errorf("wrong content: %q != %q", g, e)
	}
}
Beispiel #10
0
func TestCommitFinishedReadDir(t *testing.T) {
	t.Parallel()

	// don't leave goroutines running
	var wg sync.WaitGroup
	defer wg.Wait()

	tmp, err := ioutil.TempDir("", "pachyderm-test-")
	if err != nil {
		t.Fatalf("tempdir: %v", err)
	}
	defer func() {
		if err := os.RemoveAll(tmp); err != nil {
			t.Errorf("cannot remove tempdir: %v", err)
		}
	}()

	// closed on successful termination
	quit := make(chan struct{})
	defer close(quit)
	listener, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("cannot listen: %v", err)
	}
	defer func() {
		_ = listener.Close()
	}()

	// TODO try to share more of this setup code with various main
	// functions
	localAddress := listener.Addr().String()
	srv := grpc.NewServer()
	const (
		numShards = 1
	)
	sharder := shard.NewLocalSharder(localAddress, numShards)
	hasher := pfs.NewHasher(numShards, 1)
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		localAddress,
	)

	blockDir := filepath.Join(tmp, "blocks")
	blockServer, err := server.NewLocalBlockAPIServer(blockDir)
	if err != nil {
		t.Fatalf("NewLocalBlockAPIServer: %v", err)
	}
	pfs.RegisterBlockAPIServer(srv, blockServer)

	driver, err := drive.NewDriver(localAddress)
	if err != nil {
		t.Fatalf("NewDriver: %v", err)
	}

	apiServer := server.NewAPIServer(
		hasher,
		router,
	)
	pfs.RegisterAPIServer(srv, apiServer)

	internalAPIServer := server.NewInternalAPIServer(
		hasher,
		router,
		driver,
	)
	pfs.RegisterInternalAPIServer(srv, internalAPIServer)

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := srv.Serve(listener); err != nil {
			select {
			case <-quit:
				// orderly shutdown
				return
			default:
				t.Errorf("grpc serve: %v", err)
			}
		}
	}()

	clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
	if err != nil {
		t.Fatalf("grpc dial: %v", err)
	}
	apiClient := pfs.NewAPIClient(clientConn)
	mounter := fuse.NewMounter(localAddress, apiClient)

	mountpoint := filepath.Join(tmp, "mnt")
	if err := os.Mkdir(mountpoint, 0700); err != nil {
		t.Fatalf("mkdir mountpoint: %v", err)
	}

	ready := make(chan bool)
	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
			t.Errorf("mount and serve: %v", err)
		}
	}()

	<-ready
	defer func() {
		if err := mounter.Unmount(mountpoint); err != nil {
			t.Errorf("unmount: %v", err)
		}
	}()

	const (
		repoName = "foo"
	)
	if err := pfsutil.CreateRepo(apiClient, repoName); err != nil {
		t.Fatalf("CreateRepo: %v", err)
	}
	commit, err := pfsutil.StartCommit(apiClient, repoName, "")
	if err != nil {
		t.Fatalf("StartCommit: %v", err)
	}
	t.Logf("open commit %v", commit.Id)

	const (
		greetingName = "greeting"
		greeting     = "Hello, world\n"
		greetingPerm = 0644
	)
	if err := ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.Id, greetingName), []byte(greeting), greetingPerm); err != nil {
		t.Fatalf("WriteFile: %v", err)
	}
	const (
		scriptName = "script"
		script     = "#!/bin/sh\necho foo\n"
		scriptPerm = 0750
	)
	if err := ioutil.WriteFile(filepath.Join(mountpoint, repoName, commit.Id, scriptName), []byte(script), scriptPerm); err != nil {
		t.Fatalf("WriteFile: %v", err)
	}

	if err := pfsutil.FinishCommit(apiClient, repoName, commit.Id); err != nil {
		t.Fatalf("FinishCommit: %v", err)
	}

	if err := fstestutil.CheckDir(filepath.Join(mountpoint, repoName, commit.Id), map[string]fstestutil.FileInfoCheck{
		greetingName: func(fi os.FileInfo) error {
			// TODO respect greetingPerm
			if g, e := fi.Mode(), os.FileMode(0666); g != e {
				return fmt.Errorf("wrong mode: %v != %v", g, e)
			}
			if g, e := fi.Size(), int64(len(greeting)); g != e {
				t.Errorf("wrong size: %v != %v", g, e)
			}
			// TODO show fileModTime as mtime
			// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
			// 	t.Errorf("wrong mtime: %v != %v", g, e)
			// }
			return nil
		},
		scriptName: func(fi os.FileInfo) error {
			// TODO respect scriptPerm
			if g, e := fi.Mode(), os.FileMode(0666); g != e {
				return fmt.Errorf("wrong mode: %v != %v", g, e)
			}
			if g, e := fi.Size(), int64(len(script)); g != e {
				t.Errorf("wrong size: %v != %v", g, e)
			}
			// TODO show fileModTime as mtime
			// if g, e := fi.ModTime().UTC(), fileModTime; g != e {
			// 	t.Errorf("wrong mtime: %v != %v", g, e)
			// }
			return nil
		},
	}); err != nil {
		t.Errorf("wrong directory content: %v", err)
	}
}
Beispiel #11
0
func TestRootReadDir(t *testing.T) {
	t.Parallel()

	// don't leave goroutines running
	var wg sync.WaitGroup
	defer wg.Wait()

	tmp, err := ioutil.TempDir("", "pachyderm-test-")
	if err != nil {
		t.Fatalf("tempdir: %v", err)
	}
	defer func() {
		if err := os.RemoveAll(tmp); err != nil {
			t.Errorf("cannot remove tempdir: %v", err)
		}
	}()

	// closed on successful termination
	quit := make(chan struct{})
	defer close(quit)
	listener, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("cannot listen: %v", err)
	}
	defer func() {
		_ = listener.Close()
	}()

	// TODO try to share more of this setup code with various main
	// functions
	localAddress := listener.Addr().String()
	srv := grpc.NewServer()
	const (
		numShards = 1
	)
	sharder := shard.NewLocalSharder(localAddress, numShards)
	hasher := pfs.NewHasher(numShards, 1)
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		localAddress,
	)

	blockDir := filepath.Join(tmp, "blocks")
	blockServer, err := server.NewLocalBlockAPIServer(blockDir)
	if err != nil {
		t.Fatalf("NewLocalBlockAPIServer: %v", err)
	}
	pfs.RegisterBlockAPIServer(srv, blockServer)

	driver, err := drive.NewDriver(localAddress)
	if err != nil {
		t.Fatalf("NewDriver: %v", err)
	}

	apiServer := server.NewAPIServer(
		hasher,
		router,
	)
	pfs.RegisterAPIServer(srv, apiServer)

	internalAPIServer := server.NewInternalAPIServer(
		hasher,
		router,
		driver,
	)
	pfs.RegisterInternalAPIServer(srv, internalAPIServer)

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := srv.Serve(listener); err != nil {
			select {
			case <-quit:
				// orderly shutdown
				return
			default:
				t.Errorf("grpc serve: %v", err)
			}
		}
	}()

	clientConn, err := grpc.Dial(localAddress, grpc.WithInsecure())
	if err != nil {
		t.Fatalf("grpc dial: %v", err)
	}
	apiClient := pfs.NewAPIClient(clientConn)
	mounter := fuse.NewMounter(localAddress, apiClient)

	mountpoint := filepath.Join(tmp, "mnt")
	if err := os.Mkdir(mountpoint, 0700); err != nil {
		t.Fatalf("mkdir mountpoint: %v", err)
	}

	ready := make(chan bool)
	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := mounter.Mount(mountpoint, nil, nil, ready); err != nil {
			t.Errorf("mount and serve: %v", err)
		}
	}()

	<-ready
	defer func() {
		if err := mounter.Unmount(mountpoint); err != nil {
			t.Errorf("unmount: %v", err)
		}
	}()

	if err := pfsutil.CreateRepo(apiClient, "one"); err != nil {
		t.Fatalf("CreateRepo: %v", err)
	}
	if err := pfsutil.CreateRepo(apiClient, "two"); err != nil {
		t.Fatalf("CreateRepo: %v", err)
	}

	if err := fstestutil.CheckDir(mountpoint, map[string]fstestutil.FileInfoCheck{
		"one": func(fi os.FileInfo) error {
			if g, e := fi.Mode(), os.ModeDir|0555; g != e {
				return fmt.Errorf("wrong mode: %v != %v", g, e)
			}
			// TODO show repoSize in repo stat?
			if g, e := fi.Size(), int64(0); g != e {
				t.Errorf("wrong size: %v != %v", g, e)
			}
			// TODO show RepoInfo.Created as time
			// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
			// 	t.Errorf("wrong mtime: %v != %v", g, e)
			// }
			return nil
		},
		"two": func(fi os.FileInfo) error {
			if g, e := fi.Mode(), os.ModeDir|0555; g != e {
				return fmt.Errorf("wrong mode: %v != %v", g, e)
			}
			// TODO show repoSize in repo stat?
			if g, e := fi.Size(), int64(0); g != e {
				t.Errorf("wrong size: %v != %v", g, e)
			}
			// TODO show RepoInfo.Created as time
			// if g, e := fi.ModTime().UTC(), repoModTime; g != e {
			// 	t.Errorf("wrong mtime: %v != %v", g, e)
			// }
			return nil
		},
	}); err != nil {
		t.Errorf("wrong directory content: %v", err)
	}
}
Beispiel #12
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	logrus.Register()
	discoveryClient, err := getEtcdClient()
	if err != nil {
		return err
	}
	address := appEnv.Address
	if address == "" {
		address, err = netutil.ExternalIP()
		if err != nil {
			return err
		}
	}
	sharder := route.NewSharder(appEnv.NumShards, 0)
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	addresser := route.NewDiscoveryAddresser(
		discoveryClient,
		sharder,
		"namespace",
	)
	var driver drive.Driver
	switch appEnv.DriverType {
	case "btrfs":
		driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
	}
	apiServer := server.NewAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			0,
		),
		route.NewRouter(
			addresser,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
	)
	internalAPIServer := server.NewInternalAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			0,
		),
		route.NewRouter(
			addresser,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		driver,
	)
	go func() {
		if err := addresser.Register(nil, "id", address, internalAPIServer); err != nil {
			log.Print(err)
		}
	}()
	go func() {
		if err := addresser.AssignRoles(nil); err != nil {
			log.Print(err)
		}
	}()
	// TODO(pedge): no!
	trace.AuthRequest = func(_ *http.Request) (bool, bool) {
		return true, true
	}
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pfs.RegisterApiServer(s, apiServer)
			pfs.RegisterInternalApiServer(s, internalAPIServer)
		},
		protoserver.ServeOptions{
			HTTPPort:  uint16(appEnv.HTTPPort),
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
			HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
				return pfs.RegisterApiHandler(ctx, mux, clientConn)
			},
		},
	)
}
Beispiel #13
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	discoveryClient, err := getEtcdClient(appEnv)
	if err != nil {
		return err
	}
	address := appEnv.Address
	if address == "" {
		address, err = netutil.ExternalIP()
		if err != nil {
			return err
		}
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	sharder := shard.NewSharder(
		discoveryClient,
		appEnv.NumShards,
		0,
		"namespace",
	)
	objdAddress, err := getObjdAddress(appEnv)
	if err != nil {
		return err
	}
	clientConn, err := grpc.Dial(objdAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	objAPIClient := pfs.NewBlockAPIClient(clientConn)
	driver, err := drive.NewDriver(objAPIClient)
	if err != nil {
		return err
	}
	apiServer := server.NewAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			1,
		),
		route.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
	)
	go func() {
		if err := sharder.RegisterFrontend(nil, address, apiServer); err != nil {
			protolion.Printf("Error from sharder.RegisterFrontend %s", err.Error())
		}
	}()
	internalAPIServer := server.NewInternalAPIServer(
		route.NewSharder(
			appEnv.NumShards,
			1,
		),
		route.NewRouter(
			sharder,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		driver,
	)
	go func() {
		if err := sharder.Register(nil, address, internalAPIServer); err != nil {
			protolion.Printf("Error from sharder.Register %s", err.Error())
		}
	}()
	return protoserver.ServeWithHTTP(
		func(s *grpc.Server) {
			pfs.RegisterAPIServer(s, apiServer)
			pfs.RegisterInternalAPIServer(s, internalAPIServer)
		},
		func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
			return pfs.RegisterAPIHandler(ctx, mux, clientConn)
		},
		protoserver.ServeWithHTTPOptions{
			ServeOptions: protoserver.ServeOptions{
				Version: pachyderm.Version,
			},
		},
		protoserver.ServeEnv{
			GRPCPort: appEnv.Port,
		},
		pkghttp.HandlerEnv{
			Port: appEnv.HTTPPort,
		},
	)
}
Beispiel #14
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	discoveryClient, err := getEtcdClient()
	if err != nil {
		return err
	}
	address := appEnv.Address
	if address == "" {
		address, err = netutil.ExternalIP()
		if err != nil {
			return err
		}
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	addresser := route.NewDiscoveryAddresser(
		discoveryClient,
		"namespace",
	)
	for i := 0; i < appEnv.NumShards; i++ {
		if _, err := addresser.SetMasterAddress(i, route.Address{address, false}); err != nil {
			return err
		}
	}
	var driver drive.Driver
	switch appEnv.DriverType {
	case "btrfs":
		driver, err = btrfs.NewDriver(appEnv.DriverRoot, "")
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown value for PFS_DRIVER_TYPE: %s", appEnv.DriverType)
	}
	combinedAPIServer := server.NewCombinedAPIServer(
		route.NewSharder(
			appEnv.NumShards,
		),
		route.NewRouter(
			addresser,
			grpcutil.NewDialer(
				grpc.WithInsecure(),
			),
			address,
		),
		driver,
	)
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pfs.RegisterApiServer(s, combinedAPIServer)
			pfs.RegisterInternalApiServer(s, combinedAPIServer)
		},
		protoserver.ServeOptions{
			HTTPPort:  uint16(appEnv.HTTPPort),
			TracePort: uint16(appEnv.TracePort),
			Version:   pachyderm.Version,
			HTTPRegisterFunc: func(ctx context.Context, mux *runtime.ServeMux, clientConn *grpc.ClientConn) error {
				return pfs.RegisterApiHandler(ctx, mux, clientConn)
			},
		},
	)
}