func (a *internalAPIServer) commitToReplicas(ctx context.Context, commit *pfs.Commit) error {
version, err := a.getVersion(ctx)
if err != nil {
return err
}
shards, err := a.router.GetMasterShards(version)
if err != nil {
return err
}
for shard := range shards {
clientConns, err := a.router.GetReplicaClientConns(shard, version)
if err != nil {
return err
}
var diff bytes.Buffer
if err = a.driver.PullDiff(commit, shard, &diff); err != nil {
return err
}
for _, clientConn := range clientConns {
if _, err = pfs.NewInternalAPIClient(clientConn).PushDiff(
ctx,
&pfs.PushDiffRequest{
Commit: commit,
Shard: shard,
Value: diff.Bytes(),
},
); err != nil {
return err
}
}
}
return nil
}
func (a *apiServer) StartCommit(ctx context.Context, request *pfs.StartCommitRequest) (response *pfs.Commit, retErr error) {
defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
if request.Commit == nil {
if request.Parent == nil {
return nil, fmt.Errorf("one of Parent or Commit must be non nil")
}
request.Commit = &pfs.Commit{
Repo: request.Parent.Repo,
Id: uuid.NewWithoutDashes(),
}
}
for _, clientConn := range clientConns {
if _, err := pfs.NewInternalAPIClient(clientConn).StartCommit(ctx, request); err != nil {
return nil, err
}
}
return request.Commit, nil
}
func (a *apiServer) ListCommit(ctx context.Context, request *pfs.ListCommitRequest) (response *pfs.CommitInfos, retErr error) {
defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
var wg sync.WaitGroup
var lock sync.Mutex
var commitInfos []*pfs.CommitInfo
var loopErr error
for _, clientConn := range clientConns {
wg.Add(1)
go func(clientConn *grpc.ClientConn) {
defer wg.Done()
subCommitInfos, err := pfs.NewInternalAPIClient(clientConn).ListCommit(ctx, request)
if err != nil {
if loopErr == nil {
loopErr = err
}
return
}
lock.Lock()
defer lock.Unlock()
commitInfos = append(commitInfos, subCommitInfos.CommitInfo...)
}(clientConn)
}
wg.Wait()
if loopErr != nil {
return nil, loopErr
}
return &pfs.CommitInfos{CommitInfo: pfs.ReduceCommitInfos(commitInfos)}, nil
}
func (a *apiServer) CreateRepo(ctx context.Context, request *pfs.CreateRepoRequest) (*google_protobuf.Empty, error) {
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
if _, err := pfs.NewInternalAPIClient(clientConn).CreateRepo(ctx, request); err != nil {
return nil, err
}
}
// Create the initial commit
if _, err = a.StartCommit(ctx, &pfs.StartCommitRequest{
Parent: nil,
Commit: &pfs.Commit{
Repo: request.Repo,
Id: pfs.InitialCommitID,
},
}); err != nil {
return nil, err
}
if _, err = a.FinishCommit(ctx, &pfs.FinishCommitRequest{
Commit: &pfs.Commit{
Repo: request.Repo,
Id: pfs.InitialCommitID,
},
}); err != nil {
return nil, err
}
return google_protobuf.EmptyInstance, nil
}
func (a *internalAPIServer) AddShard(_shard uint64, version int64) error {
if version == shard.InvalidVersion {
return nil
}
ctx := versionToContext(version, context.Background())
clientConn, err := a.router.GetMasterOrReplicaClientConn(_shard, version)
if err != nil {
return err
}
repoInfos, err := pfs.NewInternalAPIClient(clientConn).ListRepo(ctx, &pfs.ListRepoRequest{})
if err != nil {
return err
}
for _, repoInfo := range repoInfos.RepoInfo {
if err := a.driver.CreateRepo(repoInfo.Repo); err != nil {
return err
}
commitInfos, err := pfs.NewInternalAPIClient(clientConn).ListCommit(ctx, &pfs.ListCommitRequest{Repo: repoInfo.Repo})
if err != nil {
return err
}
for i := range commitInfos.CommitInfo {
commit := commitInfos.CommitInfo[len(commitInfos.CommitInfo)-(i+1)].Commit
commitInfo, err := a.driver.InspectCommit(commit, map[uint64]bool{_shard: true})
if err != nil {
return err
}
if commitInfo != nil {
// we already have the commit so nothing to do
continue
}
pullDiffRequest := &pfs.PullDiffRequest{
Commit: commit,
Shard: _shard,
}
pullDiffClient, err := pfs.NewReplicaAPIClient(clientConn).PullDiff(ctx, pullDiffRequest)
if err != nil {
return err
}
diffReader := protostream.NewStreamingBytesReader(pullDiffClient)
if err := a.driver.PushDiff(commit, _shard, diffReader); err != nil {
return err
}
}
}
return nil
}
func (a *apiServer) ListRepo(ctx context.Context, request *pfs.ListRepoRequest) (*pfs.RepoInfos, error) {
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConn, err := a.getClientConn(a.version)
if err != nil {
return nil, err
}
return pfs.NewInternalAPIClient(clientConn).ListRepo(ctx, request)
}
func (a *apiServer) DeleteFile(ctx context.Context, request *pfs.DeleteFileRequest) (*google_protobuf.Empty, error) {
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConn, err := a.getClientConnForFile(request.File, a.version)
if err != nil {
return nil, err
}
return pfs.NewInternalAPIClient(clientConn).DeleteFile(ctx, request)
}
func (a *apiServer) InspectFile(ctx context.Context, request *pfs.InspectFileRequest) (*pfs.FileInfo, error) {
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConn, err := a.getClientConnForFile(request.File, a.version)
if err != nil {
return nil, err
}
return pfs.NewInternalAPIClient(clientConn).InspectFile(ctx, request)
}
func (a *apiServer) DeleteFile(ctx context.Context, request *pfs.DeleteFileRequest) (response *google_protobuf.Empty, retErr error) {
defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConn, err := a.getClientConnForFile(request.File, a.version)
if err != nil {
return nil, err
}
return pfs.NewInternalAPIClient(clientConn).DeleteFile(ctx, request)
}
func (a *apiServer) ListRepo(ctx context.Context, request *pfs.ListRepoRequest) (response *pfs.RepoInfos, retErr error) {
defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConn, err := a.getClientConn(a.version)
if err != nil {
return nil, err
}
return pfs.NewInternalAPIClient(clientConn).ListRepo(ctx, request)
}
func (a *apiServer) GetFile(request *pfs.GetFileRequest, apiGetFileServer pfs.API_GetFileServer) error {
ctx := versionToContext(a.version, apiGetFileServer.Context())
clientConn, err := a.getClientConnForFile(request.File, a.version)
if err != nil {
return err
}
fileGetClient, err := pfs.NewInternalAPIClient(clientConn).GetFile(ctx, request)
if err != nil {
return err
}
return protostream.RelayFromStreamingBytesClient(fileGetClient, apiGetFileServer)
}
func (a *apiServer) GetBlock(request *pfs.GetBlockRequest, apiGetBlockServer pfs.API_GetBlockServer) (retErr error) {
ctx := versionToContext(a.version, apiGetBlockServer.Context())
clientConn, err := a.getClientConnForBlock(request.Block, a.version)
if err != nil {
return err
}
blockGetClient, err := pfs.NewInternalAPIClient(clientConn).GetBlock(ctx, request)
if err != nil {
return err
}
return protostream.RelayFromStreamingBytesClient(blockGetClient, apiGetBlockServer)
}
func (a *apiServer) GetFile(request *pfs.GetFileRequest, apiGetFileServer pfs.API_GetFileServer) (retErr error) {
defer func(start time.Time) { a.Log(request, google_protobuf.EmptyInstance, retErr, time.Since(start)) }(time.Now())
ctx := versionToContext(a.version, apiGetFileServer.Context())
clientConn, err := a.getClientConnForFile(request.File, a.version)
if err != nil {
return err
}
fileGetClient, err := pfs.NewInternalAPIClient(clientConn).GetFile(ctx, request)
if err != nil {
return err
}
return protostream.RelayFromStreamingBytesClient(fileGetClient, apiGetFileServer)
}
func (a *apiServer) PutBlock(ctx context.Context, request *pfs.PutBlockRequest) (*pfs.Block, error) {
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
block := a.sharder.GetBlock(request.Value)
clientConn, err := a.getClientConnForBlock(block, a.version)
if err != nil {
return nil, err
}
if _, err := pfs.NewInternalAPIClient(clientConn).PutBlock(ctx, request); err != nil {
return nil, err
}
return block, nil
}
func (a *apiServer) FinishCommit(ctx context.Context, request *pfs.FinishCommitRequest) (*google_protobuf.Empty, error) {
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
if _, err := pfs.NewInternalAPIClient(clientConn).FinishCommit(ctx, request); err != nil {
return nil, err
}
}
return google_protobuf.EmptyInstance, nil
}
func (a *apiServer) PutBlock(ctx context.Context, request *pfs.PutBlockRequest) (response *pfs.Block, retErr error) {
defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
block := a.sharder.GetBlock(request.Value)
clientConn, err := a.getClientConnForBlock(block, a.version)
if err != nil {
return nil, err
}
if _, err := pfs.NewInternalAPIClient(clientConn).PutBlock(ctx, request); err != nil {
return nil, err
}
return block, nil
}
func (a *apiServer) CreateRepo(ctx context.Context, request *pfs.CreateRepoRequest) (response *google_protobuf.Empty, retErr error) {
defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
for _, clientConn := range clientConns {
if _, err := pfs.NewInternalAPIClient(clientConn).CreateRepo(ctx, request); err != nil {
return nil, err
}
}
return google_protobuf.EmptyInstance, nil
}
func (a *apiServer) ListFile(ctx context.Context, request *pfs.ListFileRequest) (response *pfs.FileInfos, retErr error) {
defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
var wg sync.WaitGroup
var lock sync.Mutex
var fileInfos []*pfs.FileInfo
seenDirectories := make(map[string]bool)
var loopErr error
for _, clientConn := range clientConns {
wg.Add(1)
go func(clientConn *grpc.ClientConn) {
defer wg.Done()
subFileInfos, err := pfs.NewInternalAPIClient(clientConn).ListFile(ctx, request)
lock.Lock()
defer lock.Unlock()
if err != nil {
if loopErr == nil {
loopErr = err
}
return
}
for _, fileInfo := range subFileInfos.FileInfo {
if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR {
if seenDirectories[fileInfo.File.Path] {
continue
}
seenDirectories[fileInfo.File.Path] = true
}
fileInfos = append(fileInfos, fileInfo)
}
}(clientConn)
}
wg.Wait()
if loopErr != nil {
return nil, loopErr
}
return &pfs.FileInfos{
FileInfo: fileInfos,
}, nil
}
func (a *apiServer) StartCommit(ctx context.Context, request *pfs.StartCommitRequest) (*pfs.Commit, error) {
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
if request.Commit == nil {
request.Commit = &pfs.Commit{
Repo: request.Parent.Repo,
Id: uuid.NewWithoutDashes(),
}
}
for _, clientConn := range clientConns {
if _, err := pfs.NewInternalAPIClient(clientConn).StartCommit(ctx, request); err != nil {
return nil, err
}
}
return request.Commit, nil
}
func (a *apiServer) ListChange(ctx context.Context, request *pfs.ListChangeRequest) (*pfs.Changes, error) {
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
var wg sync.WaitGroup
var lock sync.Mutex
var changes []*pfs.Change
var loopErr error
for _, clientConn := range clientConns {
wg.Add(1)
go func(clientConn *grpc.ClientConn) {
defer wg.Done()
subChanges, err := pfs.NewInternalAPIClient(clientConn).ListChange(ctx, request)
lock.Lock()
defer lock.Unlock()
if err != nil {
if loopErr == nil {
loopErr = err
}
return
}
for _, change := range subChanges.Change {
changes = append(changes, change)
}
}(clientConn)
}
wg.Wait()
if loopErr != nil {
return nil, loopErr
}
return &pfs.Changes{
Change: changes,
}, nil
}
func RunTest(
t *testing.T,
f func(*testing.T, pfs.APIClient, pfs.InternalAPIClient, Cluster),
) {
discoveryClient, err := getEtcdClient()
require.NoError(t, err)
var cluster *cluster
prototest.RunT(
t,
testNumServers,
func(servers map[string]*grpc.Server) {
cluster = registerFunc(t, discoveryClient, servers)
},
func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
var clientConn *grpc.ClientConn
for _, c := range clientConns {
clientConn = c
break
}
go func() {
require.Equal(t, cluster.realSharder.AssignRoles(cluster.cancel), shard.ErrCancelled)
}()
cluster.WaitForAvailability()
f(
t,
pfs.NewAPIClient(
clientConn,
),
pfs.NewInternalAPIClient(
clientConn,
),
cluster,
)
},
)
cluster.Shutdown()
}
func (a *apiServer) ListCommit(ctx context.Context, request *pfs.ListCommitRequest) (response *pfs.CommitInfos, retErr error) {
defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
var wg sync.WaitGroup
var lock sync.Mutex
var commitInfos [][]*pfs.CommitInfo
var loopErr error
for _, clientConn := range clientConns {
wg.Add(1)
go func(clientConn *grpc.ClientConn) {
defer wg.Done()
subCommitInfos, err := pfs.NewInternalAPIClient(clientConn).ListCommit(ctx, request)
lock.Lock()
defer lock.Unlock()
if err != nil {
loopErr = err
return
}
commitInfos = append(commitInfos, subCommitInfos.CommitInfo)
}(clientConn)
}
wg.Wait()
if loopErr != nil {
return nil, loopErr
}
idToCommitInfo := make(map[string]*pfs.CommitInfo)
idToSeenCount := make(map[string]int)
for _, subCommitInfos := range commitInfos {
for _, subCommitInfo := range subCommitInfos {
idToSeenCount[subCommitInfo.Commit.Id] = idToSeenCount[subCommitInfo.Commit.Id] + 1
commitInfo, ok := idToCommitInfo[subCommitInfo.Commit.Id]
if !ok {
idToCommitInfo[subCommitInfo.Commit.Id] = subCommitInfo
continue
}
if commitInfo.CommitType != pfs.CommitType_COMMIT_TYPE_READ {
commitInfo.CommitType = subCommitInfo.CommitType
}
if prototime.TimestampToTime(subCommitInfo.Started).Before(prototime.TimestampToTime(commitInfo.Started)) {
commitInfo.Started = subCommitInfo.Started
}
if commitInfo.Finished != nil {
if subCommitInfo.Finished == nil {
commitInfo.Finished = nil
} else if prototime.TimestampToTime(subCommitInfo.Finished).After(prototime.TimestampToTime(commitInfo.Finished)) {
commitInfo.Finished = subCommitInfo.Finished
}
}
commitInfo.CommitBytes += subCommitInfo.CommitBytes
commitInfo.TotalBytes += subCommitInfo.TotalBytes
}
}
var result []*pfs.CommitInfo
for _, subCommitInfos := range commitInfos {
for _, commitInfo := range subCommitInfos {
if idToSeenCount[commitInfo.Commit.Id] == len(commitInfos) {
result = append(result, idToCommitInfo[commitInfo.Commit.Id])
}
}
break
}
return &pfs.CommitInfos{CommitInfo: result}, nil
}
func (a *apiServer) PutFile(putFileServer pfs.API_PutFileServer) (retErr error) {
ctx := versionToContext(a.version, putFileServer.Context())
defer func() {
if err := putFileServer.SendAndClose(google_protobuf.EmptyInstance); err != nil && retErr == nil {
retErr = err
}
}()
request, err := putFileServer.Recv()
if err != nil {
return err
}
if strings.HasPrefix(request.File.Path, "/") {
// This is a subtle error case, the paths foo and /foo will hash to
// different shards but will produce the same change once they get to
// those shards due to how path.Join. This can go wrong in a number of
// ways so we forbid leading slashes.
return fmt.Errorf("pachyderm: leading slash in path: %s", request.File.Path)
}
if request.FileType == pfs.FileType_FILE_TYPE_DIR {
if len(request.Value) > 0 {
return fmt.Errorf("PutFileRequest shouldn't have type dir and a value")
}
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return err
}
for _, clientConn := range clientConns {
putFileClient, err := pfs.NewInternalAPIClient(clientConn).PutFile(ctx)
if err != nil {
return err
}
if err := putFileClient.Send(request); err != nil {
return err
}
if _, err := putFileClient.CloseAndRecv(); err != nil {
return err
}
}
return nil
}
clientConn, err := a.getClientConnForFile(request.File, a.version)
if err != nil {
return err
}
putFileClient, err := pfs.NewInternalAPIClient(clientConn).PutFile(ctx)
if err != nil {
return err
}
defer func() {
if _, err := putFileClient.CloseAndRecv(); err != nil && retErr == nil {
retErr = err
}
}()
if err := putFileClient.Send(request); err != nil {
return err
}
for {
request, err := putFileServer.Recv()
if err != nil {
if err == io.EOF {
break
}
return err
}
if err := putFileClient.Send(request); err != nil {
return err
}
}
if _, err := putFileClient.CloseAndRecv(); err != nil {
return err
}
return nil
}