func (a *internalAPIServer) AddShard(shard uint64) error { version, ctx, err := a.versionAndCtx(context.Background()) if err != nil { return err } if version == route.InvalidVersion { return nil } clientConn, err := a.router.GetMasterOrReplicaClientConn(shard, version) if err != nil { return err } repoInfos, err := pfs.NewInternalApiClient(clientConn).ListRepo(ctx, &pfs.ListRepoRequest{}) if err != nil { return err } for _, repoInfo := range repoInfos.RepoInfo { if err := a.driver.CreateRepo(repoInfo.Repo); err != nil { return err } commitInfos, err := pfs.NewInternalApiClient(clientConn).ListCommit(ctx, &pfs.ListCommitRequest{Repo: repoInfo.Repo}) if err != nil { return err } for i := range commitInfos.CommitInfo { commit := commitInfos.CommitInfo[len(commitInfos.CommitInfo)-(i+1)].Commit commitInfo, err := a.driver.InspectCommit(commit, map[uint64]bool{shard: true}) if err != nil { return err } if commitInfo != nil { // we already have the commit so nothing to do continue } pullDiffRequest := &pfs.PullDiffRequest{ Commit: commit, Shard: shard, } pullDiffClient, err := pfs.NewInternalApiClient(clientConn).PullDiff(ctx, pullDiffRequest) if err != nil { return err } diffReader := protostream.NewStreamingBytesReader(pullDiffClient) a.driver.PushDiff(commit, diffReader) } } return nil }
func RunTest( t *testing.T, f func(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient), ) { discoveryClient, err := getEtcdClient() require.NoError(t, err) grpctest.Run( t, testNumServers, func(servers map[string]*grpc.Server) { registerFunc(getDriver(t), discoveryClient, servers) }, func(t *testing.T, clientConns map[string]*grpc.ClientConn) { var clientConn *grpc.ClientConn for _, c := range clientConns { clientConn = c break } f( t, pfs.NewApiClient( clientConn, ), pfs.NewInternalApiClient( clientConn, ), ) }, ) }
func (a *apiServer) CreateRepo(ctx context.Context, request *pfs.CreateRepoRequest) (*google_protobuf.Empty, error) { version, ctx, err := a.versionAndCtx(ctx) if err != nil { return nil, err } clientConns, err := a.router.GetAllClientConns(version) if err != nil { return nil, err } for _, clientConn := range clientConns { if _, err := pfs.NewInternalApiClient(clientConn).CreateRepo(ctx, request); err != nil { return nil, err } } // Create the initial commit if _, err = a.StartCommit(ctx, &pfs.StartCommitRequest{ Parent: nil, Commit: &pfs.Commit{ Repo: request.Repo, Id: InitialCommitID, }, }); err != nil { return nil, err } if _, err = a.FinishCommit(ctx, &pfs.FinishCommitRequest{ Commit: &pfs.Commit{ Repo: request.Repo, Id: InitialCommitID, }, }); err != nil { return nil, err } return emptyInstance, nil }
func (a *internalAPIServer) commitToReplicas(ctx context.Context, commit *pfs.Commit) error { version, err := a.getVersion(ctx) if err != nil { return err } shards, err := a.router.GetMasterShards(version) if err != nil { return err } for shard := range shards { clientConns, err := a.router.GetReplicaClientConns(shard, version) if err != nil { return err } var diff bytes.Buffer if err = a.driver.PullDiff(commit, shard, &diff); err != nil { return err } for _, clientConn := range clientConns { if _, err = pfs.NewInternalApiClient(clientConn).PushDiff( ctx, &pfs.PushDiffRequest{ Commit: commit, Shard: shard, Value: diff.Bytes(), }, ); err != nil { return err } } } return nil }
func RunTest( t *testing.T, f func(*testing.T, pfs.ApiClient, pfs.InternalApiClient, Cluster), ) { discoveryClient, err := getEtcdClient() require.NoError(t, err) var cluster Cluster prototest.RunT( t, testNumServers, func(servers map[string]*grpc.Server) { cluster = registerFunc(t, discoveryClient, servers) }, func(t *testing.T, clientConns map[string]*grpc.ClientConn) { var clientConn *grpc.ClientConn for _, c := range clientConns { clientConn = c break } f( t, pfs.NewApiClient( clientConn, ), pfs.NewInternalApiClient( clientConn, ), cluster, ) }, ) cluster.Shutdown() }
func (a *combinedAPIServer) PushDiff(ctx context.Context, pushDiffRequest *pfs.PushDiffRequest) (*google_protobuf.Empty, error) { clientConn, err := a.getClientConnIfNecessary(int(pushDiffRequest.Shard), false) if err != nil { return nil, err } if clientConn != nil { return pfs.NewInternalApiClient(clientConn).PushDiff(ctx, pushDiffRequest) } return emptyInstance, a.driver.PushDiff(pushDiffRequest.Commit, bytes.NewReader(pushDiffRequest.Value)) }
func (a *apiServer) DeleteFile(ctx context.Context, request *pfs.DeleteFileRequest) (*google_protobuf.Empty, error) { version, ctx, err := a.versionAndCtx(ctx) if err != nil { return nil, err } clientConn, err := a.getClientConnForFile(request.File, version) if err != nil { return nil, err } return pfs.NewInternalApiClient(clientConn).DeleteFile(ctx, request) }
func (a *apiServer) ListRepo(ctx context.Context, request *pfs.ListRepoRequest) (*pfs.RepoInfos, error) { version, ctx, err := a.versionAndCtx(ctx) if err != nil { return nil, err } clientConn, err := a.getClientConn(version) if err != nil { return nil, err } return pfs.NewInternalApiClient(clientConn).ListRepo(ctx, request) }
func (a *apiServer) InspectFile(ctx context.Context, request *pfs.InspectFileRequest) (*pfs.FileInfo, error) { version, ctx, err := a.versionAndCtx(ctx) if err != nil { return nil, err } clientConn, err := a.getClientConnForFile(request.File, version) if err != nil { return nil, err } return pfs.NewInternalApiClient(clientConn).InspectFile(ctx, request) }
func (a *apiServer) GetBlock(request *pfs.GetBlockRequest, apiGetBlockServer pfs.Api_GetBlockServer) (retErr error) { version, ctx, err := a.versionAndCtx(context.Background()) if err != nil { return err } clientConn, err := a.getClientConnForBlock(request.Block, version) if err != nil { return err } blockGetClient, err := pfs.NewInternalApiClient(clientConn).GetBlock(ctx, request) if err != nil { return err } return protostream.RelayFromStreamingBytesClient(blockGetClient, apiGetBlockServer) }
func (a *apiServer) PutBlock(ctx context.Context, request *pfs.PutBlockRequest) (*pfs.Block, error) { version, ctx, err := a.versionAndCtx(ctx) if err != nil { return nil, err } block := a.sharder.GetBlock(request.Value) clientConn, err := a.getClientConnForBlock(block, version) if err != nil { return nil, err } if _, err := pfs.NewInternalApiClient(clientConn).PutBlock(ctx, request); err != nil { return nil, err } return block, nil }
func (a *apiServer) GetFile(request *pfs.GetFileRequest, apiGetFileServer pfs.Api_GetFileServer) error { version, ctx, err := a.versionAndCtx(context.Background()) if err != nil { return err } clientConn, err := a.getClientConnForFile(request.File, version) if err != nil { return err } fileGetClient, err := pfs.NewInternalApiClient(clientConn).GetFile(ctx, request) if err != nil { return err } return protostream.RelayFromStreamingBytesClient(fileGetClient, apiGetFileServer) }
func (a *combinedAPIServer) Master(shard int) error { clientConns, err := a.router.GetReplicaClientConns(shard) if err != nil { return err } for _, clientConn := range clientConns { apiClient := pfs.NewApiClient(clientConn) response, err := apiClient.ListRepositories(context.Background(), &pfs.ListRepositoriesRequest{}) if err != nil { return err } for _, repository := range response.Repository { if err := a.driver.InitRepository(repository, map[int]bool{shard: true}); err != nil { return err } response, err := apiClient.ListCommits(context.Background(), &pfs.ListCommitsRequest{Repository: repository}) if err != nil { return err } localCommitInfo, err := a.driver.ListCommits(repository, shard) if err != nil { return err } for i, commitInfo := range response.CommitInfo { if i < len(localCommitInfo) { if *commitInfo != *localCommitInfo[i] { return fmt.Errorf("divergent data") } continue } pullDiffClient, err := pfs.NewInternalApiClient(clientConn).PullDiff( context.Background(), &pfs.PullDiffRequest{ Commit: commitInfo.Commit, Shard: uint64(shard), }, ) if err != nil { return err } if err := a.driver.PushDiff(commitInfo.Commit, protostream.NewStreamingBytesReader(pullDiffClient)); err != nil { return err } } } } return nil }
func (a *apiServer) FinishCommit(ctx context.Context, request *pfs.FinishCommitRequest) (*google_protobuf.Empty, error) { version, ctx, err := a.versionAndCtx(ctx) if err != nil { return nil, err } clientConns, err := a.router.GetAllClientConns(version) if err != nil { return nil, err } for _, clientConn := range clientConns { if _, err := pfs.NewInternalApiClient(clientConn).FinishCommit(ctx, request); err != nil { return nil, err } } return emptyInstance, nil }
func (a *apiServer) ListFile(ctx context.Context, request *pfs.ListFileRequest) (*pfs.FileInfos, error) { version, ctx, err := a.versionAndCtx(ctx) if err != nil { return nil, err } clientConns, err := a.router.GetAllClientConns(version) if err != nil { return nil, err } var wg sync.WaitGroup var lock sync.Mutex var fileInfos []*pfs.FileInfo seenDirectories := make(map[string]bool) var loopErr error for _, clientConn := range clientConns { wg.Add(1) go func(clientConn *grpc.ClientConn) { defer wg.Done() subFileInfos, err := pfs.NewInternalApiClient(clientConn).ListFile(ctx, request) lock.Lock() defer lock.Unlock() if err != nil { if loopErr == nil { loopErr = err } return } for _, fileInfo := range subFileInfos.FileInfo { if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR { if seenDirectories[fileInfo.File.Path] { continue } seenDirectories[fileInfo.File.Path] = true } fileInfos = append(fileInfos, fileInfo) } }(clientConn) } wg.Wait() if loopErr != nil { return nil, loopErr } return &pfs.FileInfos{ FileInfo: fileInfos, }, nil }
func (a *combinedAPIServer) PullDiff(pullDiffRequest *pfs.PullDiffRequest, apiPullDiffServer pfs.InternalApi_PullDiffServer) error { clientConn, err := a.getClientConnIfNecessary(int(pullDiffRequest.Shard), false) if err != nil { return err } if clientConn != nil { apiPullDiffClient, err := pfs.NewInternalApiClient(clientConn).PullDiff(context.Background(), pullDiffRequest) if err != nil { return err } return protoutil.RelayFromStreamingBytesClient(apiPullDiffClient, apiPullDiffServer) } var buffer bytes.Buffer a.driver.PullDiff(pullDiffRequest.Commit, int(pullDiffRequest.Shard), &buffer) return protoutil.WriteToStreamingBytesServer( &buffer, apiPullDiffServer, ) }
func (a *apiServer) StartCommit(ctx context.Context, request *pfs.StartCommitRequest) (*pfs.Commit, error) { version, ctx, err := a.versionAndCtx(ctx) if err != nil { return nil, err } clientConns, err := a.router.GetAllClientConns(version) if err != nil { return nil, err } if request.Commit == nil { request.Commit = &pfs.Commit{ Repo: request.Parent.Repo, Id: strings.Replace(uuid.NewV4().String(), "-", "", -1), } } for _, clientConn := range clientConns { if _, err := pfs.NewInternalApiClient(clientConn).StartCommit(ctx, request); err != nil { return nil, err } } return request.Commit, nil }
func (a *apiServer) ListChange(ctx context.Context, request *pfs.ListChangeRequest) (*pfs.Changes, error) { version, ctx, err := a.versionAndCtx(ctx) if err != nil { return nil, err } clientConns, err := a.router.GetAllClientConns(version) if err != nil { return nil, err } var wg sync.WaitGroup var lock sync.Mutex var changes []*pfs.Change var loopErr error for _, clientConn := range clientConns { wg.Add(1) go func(clientConn *grpc.ClientConn) { defer wg.Done() subChanges, err := pfs.NewInternalApiClient(clientConn).ListChange(ctx, request) lock.Lock() defer lock.Unlock() if err != nil { if loopErr == nil { loopErr = err } return } for _, change := range subChanges.Change { changes = append(changes, change) } }(clientConn) } wg.Wait() if loopErr != nil { return nil, loopErr } return &pfs.Changes{ Change: changes, }, nil }
func (a *apiServer) ListCommit(ctx context.Context, request *pfs.ListCommitRequest) (*pfs.CommitInfos, error) { version, ctx, err := a.versionAndCtx(ctx) if err != nil { return nil, err } clientConns, err := a.router.GetAllClientConns(version) if err != nil { return nil, err } var wg sync.WaitGroup var lock sync.Mutex var commitInfos [][]*pfs.CommitInfo var loopErr error for _, clientConn := range clientConns { wg.Add(1) go func(clientConn *grpc.ClientConn) { defer wg.Done() subCommitInfos, err := pfs.NewInternalApiClient(clientConn).ListCommit(ctx, request) lock.Lock() defer lock.Unlock() if err != nil { loopErr = err return } commitInfos = append(commitInfos, subCommitInfos.CommitInfo) }(clientConn) } wg.Wait() if loopErr != nil { return nil, loopErr } idToCommitInfo := make(map[string]*pfs.CommitInfo) idToSeenCount := make(map[string]int) for _, subCommitInfos := range commitInfos { for _, subCommitInfo := range subCommitInfos { idToSeenCount[subCommitInfo.Commit.Id] = idToSeenCount[subCommitInfo.Commit.Id] + 1 commitInfo, ok := idToCommitInfo[subCommitInfo.Commit.Id] if !ok { idToCommitInfo[subCommitInfo.Commit.Id] = subCommitInfo continue } if commitInfo.CommitType != pfs.CommitType_COMMIT_TYPE_READ { commitInfo.CommitType = subCommitInfo.CommitType } if prototime.TimestampToTime(subCommitInfo.Started).Before(prototime.TimestampToTime(commitInfo.Started)) { commitInfo.Started = subCommitInfo.Started } if commitInfo.Finished != nil { if subCommitInfo.Finished == nil { commitInfo.Finished = nil } else if prototime.TimestampToTime(subCommitInfo.Finished).After(prototime.TimestampToTime(commitInfo.Finished)) { commitInfo.Finished = subCommitInfo.Finished } } commitInfo.CommitBytes += subCommitInfo.CommitBytes commitInfo.TotalBytes += subCommitInfo.TotalBytes } } var result []*pfs.CommitInfo for _, subCommitInfos := range commitInfos { for _, commitInfo := range subCommitInfos { if idToSeenCount[commitInfo.Commit.Id] == len(commitInfos) { result = append(result, idToCommitInfo[commitInfo.Commit.Id]) } } break } return &pfs.CommitInfos{CommitInfo: result}, nil }
func (a *apiServer) PutFile(putFileServer pfs.Api_PutFileServer) (retErr error) { version, ctx, err := a.versionAndCtx(putFileServer.Context()) if err != nil { return err } defer func() { if err := putFileServer.SendAndClose(emptyInstance); err != nil && retErr == nil { retErr = err } }() request, err := putFileServer.Recv() if err != nil { return err } if strings.HasPrefix(request.File.Path, "/") { // This is a subtle error case, the paths foo and /foo will hash to // different shards but will produce the same change once they get to // those shards due to how path.Join. This can go wrong in a number of // ways so we forbid leading slashes. return fmt.Errorf("pachyderm: leading slash in path: %s", request.File.Path) } if request.FileType == pfs.FileType_FILE_TYPE_DIR { if len(request.Value) > 0 { return fmt.Errorf("PutFileRequest shouldn't have type dir and a value") } clientConns, err := a.router.GetAllClientConns(version) if err != nil { return err } for _, clientConn := range clientConns { putFileClient, err := pfs.NewInternalApiClient(clientConn).PutFile(ctx) if err != nil { return err } if err := putFileClient.Send(request); err != nil { return err } if _, err := putFileClient.CloseAndRecv(); err != nil { return err } } return nil } clientConn, err := a.getClientConnForFile(request.File, version) if err != nil { return err } putFileClient, err := pfs.NewInternalApiClient(clientConn).PutFile(ctx) if err != nil { return err } defer func() { if _, err := putFileClient.CloseAndRecv(); err != nil && retErr == nil { retErr = err } }() if err := putFileClient.Send(request); err != nil { return err } for { request, err := putFileServer.Recv() if err != nil { if err == io.EOF { break } return err } if err := putFileClient.Send(request); err != nil { return err } } if _, err := putFileClient.CloseAndRecv(); err != nil { return err } return nil }