// fullCommitProvenance recursively computes the provenance of the commit, // starting with the immediate provenance that was declared when the commit was // created, then our immediate provenance's immediate provenance etc. func (d *driver) fullCommitProvenance(commit *pfs.Commit, repoSet map[string]bool, shards map[uint64]bool) ([]*pfs.Commit, error) { shardToDiffInfo, ok := d.diffs[commit.Repo.Name] if !ok { return nil, pfsserver.NewErrRepoNotFound(commit.Repo.Name) } var result []*pfs.Commit for shard := range shards { diffInfos := shardToDiffInfo[shard] if !ok { return nil, fmt.Errorf("missing shard %d (this is likely a bug)", shard) } diffInfo, ok := diffInfos[commit.ID] if !ok { return nil, fmt.Errorf("missing \"%s\" diff (this is likely a bug)", commit.ID) } for _, provCommit := range diffInfo.Provenance { if !repoSet[provCommit.Repo.Name] { repoSet[provCommit.Repo.Name] = true result = append(result, provCommit) provCommits, err := d.fullCommitProvenance(provCommit, repoSet, shards) if err != nil { return nil, err } result = append(result, provCommits...) } } break // we only need to consider 1 shard } return result, nil }
// inspectRepo assumes that the lock is being held func (d *driver) inspectRepo(repo *pfs.Repo, shards map[uint64]bool) (*pfs.RepoInfo, error) { result := &pfs.RepoInfo{ Repo: repo, } shardToDiffInfo, ok := d.diffs[repo.Name] if !ok { return nil, pfsserver.NewErrRepoNotFound(repo.Name) } for shard := range shards { diffInfos, ok := shardToDiffInfo[shard] if !ok { continue } for _, diffInfo := range diffInfos { diffInfo := diffInfo if diffInfo.Diff.Commit.ID == "" && result.Created == nil { result.Created = diffInfo.Finished } result.SizeBytes += diffInfo.SizeBytes } } provenance, err := d.fullRepoProvenance(repo, shards) if err != nil { return nil, err } result.Provenance = provenance return result, nil }
// canonicalCommit finds the canonical way of referring to a commit func (d *driver) canonicalCommit(commit *pfs.Commit) (*pfs.Commit, error) { if _, ok := d.branches[commit.Repo.Name]; !ok { return nil, pfsserver.NewErrRepoNotFound(commit.Repo.Name) } if commitID, ok := d.branches[commit.Repo.Name][commit.ID]; ok { return client.NewCommit(commit.Repo.Name, commitID), nil } return commit, nil }
func (d *driver) ListCommit(repos []*pfs.Repo, commitType pfs.CommitType, fromCommit []*pfs.Commit, provenance []*pfs.Commit, all bool, shards map[uint64]bool) ([]*pfs.CommitInfo, error) { repoSet := repoSet(repos) var canonicalProvenance []*pfs.Commit for _, provCommit := range provenance { canonicalCommit, err := d.canonicalCommit(provCommit) if err != nil { return nil, err } canonicalProvenance = append(canonicalProvenance, canonicalCommit) } breakCommitIDs := make(map[string]bool) for _, commit := range fromCommit { if !repoSet[commit.Repo.Name] { return nil, fmt.Errorf("Commit %s/%s is from a repo that isn't being listed.", commit.Repo.Name, commit.ID) } breakCommitIDs[commit.ID] = true } d.lock.RLock() defer d.lock.RUnlock() var result []*pfs.CommitInfo for _, repo := range repos { _, ok := d.diffs[repo.Name] if !ok { return nil, pfsserver.NewErrRepoNotFound(repo.Name) } for _, commitID := range d.dags[repo.Name].Leaves() { commit := &pfs.Commit{ Repo: repo, ID: commitID, } for commit != nil && !breakCommitIDs[commit.ID] { // we add this commit to breakCommitIDs so we won't see it twice breakCommitIDs[commit.ID] = true commitInfo, err := d.inspectCommit(commit, shards) if err != nil { return nil, err } commit = commitInfo.ParentCommit if commitInfo.Cancelled && !all { continue } if !MatchProvenance(canonicalProvenance, commitInfo.Provenance) { continue } if commitType != pfs.CommitType_COMMIT_TYPE_NONE && commitType != commitInfo.CommitType { continue } result = append(result, commitInfo) } } } return result, nil }
func (d *driver) StartCommit(repo *pfs.Repo, commitID string, parentID string, branch string, started *google_protobuf.Timestamp, provenance []*pfs.Commit, shards map[uint64]bool) error { d.lock.Lock() defer d.lock.Unlock() // make sure that the parent commit exists if parentID != "" { _, err := d.inspectCommit(client.NewCommit(repo.Name, parentID), shards) if err != nil { return err } } for shard := range shards { if len(provenance) != 0 { diffInfo, ok := d.diffs.get(client.NewDiff(repo.Name, "", shard)) if !ok { return pfsserver.NewErrRepoNotFound(repo.Name) } provRepos := repoSetFromCommits(diffInfo.Provenance) for _, provCommit := range provenance { if !provRepos[provCommit.Repo.Name] { return fmt.Errorf("cannot use %s/%s as provenance, %s is not provenance of %s", provCommit.Repo.Name, provCommit.ID, provCommit.Repo.Name, repo.Name) } } } diffInfo := &pfs.DiffInfo{ Diff: client.NewDiff(repo.Name, commitID, shard), Started: started, Appends: make(map[string]*pfs.Append), Branch: branch, Provenance: provenance, } if branch != "" { parentCommit, err := d.branchParent(client.NewCommit(repo.Name, commitID), branch) if err != nil { return err } if parentCommit != nil && parentID != "" { return fmt.Errorf("branch %s already exists as %s, can't create with %s as parent", branch, parentCommit.ID, parentID) } diffInfo.ParentCommit = parentCommit } if diffInfo.ParentCommit == nil && parentID != "" { diffInfo.ParentCommit = client.NewCommit(repo.Name, parentID) } if err := d.insertDiffInfo(diffInfo); err != nil { return err } } d.commitConds[commitID] = sync.NewCond(&d.lock) return nil }
func (d *driver) ListBranch(repo *pfs.Repo, shards map[uint64]bool) ([]*pfs.CommitInfo, error) { var result []*pfs.CommitInfo _, ok := d.branches[repo.Name] if !ok { return nil, pfsserver.NewErrRepoNotFound(repo.Name) } for commitID := range d.branches[repo.Name] { commitInfo, err := d.inspectCommit(client.NewCommit(repo.Name, commitID), shards) if err != nil { return nil, err } result = append(result, commitInfo) } return result, nil }
func (d diffMap) insert(diffInfo *pfs.DiffInfo) error { diff := diffInfo.Diff shardMap, ok := d[diff.Commit.Repo.Name] if !ok { return pfsserver.NewErrRepoNotFound(diff.Commit.Repo.Name) } commitMap, ok := shardMap[diff.Shard] if !ok { commitMap = make(map[string]*pfs.DiffInfo) shardMap[diff.Shard] = commitMap } if _, ok = commitMap[diff.Commit.ID]; ok { return fmt.Errorf("commit %s/%s already exists", diff.Commit.Repo.Name, diff.Commit.ID) } commitMap[diff.Commit.ID] = diffInfo return nil }
func (d *driver) DeleteRepo(repo *pfs.Repo, shards map[uint64]bool) error { // Make sure that this repo is not the provenance of any other repo repoInfos, err := d.ListRepo([]*pfs.Repo{repo}, shards) if err != nil { return err } var diffInfos []*pfs.DiffInfo err = func() error { d.lock.Lock() defer d.lock.Unlock() if _, ok := d.diffs[repo.Name]; !ok { return pfsserver.NewErrRepoNotFound(repo.Name) } if len(repoInfos) > 0 { var repoNames []string for _, repoInfo := range repoInfos { repoNames = append(repoNames, repoInfo.Repo.Name) } return fmt.Errorf("cannot delete repo %v; it's the provenance of the following repos: %v", repo.Name, repoNames) } for shard := range shards { for _, diffInfo := range d.diffs[repo.Name][shard] { diffInfos = append(diffInfos, diffInfo) } } delete(d.diffs, repo.Name) return nil }() if err != nil { return err } blockClient, err := d.getBlockClient() if err != nil { return err } errCh := make(chan error, 1) var wg sync.WaitGroup for _, diffInfo := range diffInfos { diffInfo := diffInfo wg.Add(1) go func() { defer wg.Done() if _, err := blockClient.DeleteDiff( context.Background(), &pfs.DeleteDiffRequest{Diff: diffInfo.Diff}, ); err != nil { select { case errCh <- err: default: } return } }() } wg.Wait() select { case err := <-errCh: return err default: } return nil }