// fullCommitProvenance recursively computes the provenance of the commit,
// starting with the immediate provenance that was declared when the commit was
// created, then our immediate provenance's immediate provenance etc.
func (d *driver) fullCommitProvenance(commit *pfs.Commit, repoSet map[string]bool,
shards map[uint64]bool) ([]*pfs.Commit, error) {
shardToDiffInfo, ok := d.diffs[commit.Repo.Name]
if !ok {
return nil, pfsserver.NewErrRepoNotFound(commit.Repo.Name)
}
var result []*pfs.Commit
for shard := range shards {
diffInfos := shardToDiffInfo[shard]
if !ok {
return nil, fmt.Errorf("missing shard %d (this is likely a bug)", shard)
}
diffInfo, ok := diffInfos[commit.ID]
if !ok {
return nil, fmt.Errorf("missing \"%s\" diff (this is likely a bug)", commit.ID)
}
for _, provCommit := range diffInfo.Provenance {
if !repoSet[provCommit.Repo.Name] {
repoSet[provCommit.Repo.Name] = true
result = append(result, provCommit)
provCommits, err := d.fullCommitProvenance(provCommit, repoSet, shards)
if err != nil {
return nil, err
}
result = append(result, provCommits...)
}
}
break // we only need to consider 1 shard
}
return result, nil
}
// inspectRepo assumes that the lock is being held
func (d *driver) inspectRepo(repo *pfs.Repo, shards map[uint64]bool) (*pfs.RepoInfo, error) {
result := &pfs.RepoInfo{
Repo: repo,
}
shardToDiffInfo, ok := d.diffs[repo.Name]
if !ok {
return nil, pfsserver.NewErrRepoNotFound(repo.Name)
}
for shard := range shards {
diffInfos, ok := shardToDiffInfo[shard]
if !ok {
continue
}
for _, diffInfo := range diffInfos {
diffInfo := diffInfo
if diffInfo.Diff.Commit.ID == "" && result.Created == nil {
result.Created = diffInfo.Finished
}
result.SizeBytes += diffInfo.SizeBytes
}
}
provenance, err := d.fullRepoProvenance(repo, shards)
if err != nil {
return nil, err
}
result.Provenance = provenance
return result, nil
}
// canonicalCommit finds the canonical way of referring to a commit
func (d *driver) canonicalCommit(commit *pfs.Commit) (*pfs.Commit, error) {
if _, ok := d.branches[commit.Repo.Name]; !ok {
return nil, pfsserver.NewErrRepoNotFound(commit.Repo.Name)
}
if commitID, ok := d.branches[commit.Repo.Name][commit.ID]; ok {
return client.NewCommit(commit.Repo.Name, commitID), nil
}
return commit, nil
}
func (d *driver) ListCommit(repos []*pfs.Repo, commitType pfs.CommitType, fromCommit []*pfs.Commit,
provenance []*pfs.Commit, all bool, shards map[uint64]bool) ([]*pfs.CommitInfo, error) {
repoSet := repoSet(repos)
var canonicalProvenance []*pfs.Commit
for _, provCommit := range provenance {
canonicalCommit, err := d.canonicalCommit(provCommit)
if err != nil {
return nil, err
}
canonicalProvenance = append(canonicalProvenance, canonicalCommit)
}
breakCommitIDs := make(map[string]bool)
for _, commit := range fromCommit {
if !repoSet[commit.Repo.Name] {
return nil, fmt.Errorf("Commit %s/%s is from a repo that isn't being listed.", commit.Repo.Name, commit.ID)
}
breakCommitIDs[commit.ID] = true
}
d.lock.RLock()
defer d.lock.RUnlock()
var result []*pfs.CommitInfo
for _, repo := range repos {
_, ok := d.diffs[repo.Name]
if !ok {
return nil, pfsserver.NewErrRepoNotFound(repo.Name)
}
for _, commitID := range d.dags[repo.Name].Leaves() {
commit := &pfs.Commit{
Repo: repo,
ID: commitID,
}
for commit != nil && !breakCommitIDs[commit.ID] {
// we add this commit to breakCommitIDs so we won't see it twice
breakCommitIDs[commit.ID] = true
commitInfo, err := d.inspectCommit(commit, shards)
if err != nil {
return nil, err
}
commit = commitInfo.ParentCommit
if commitInfo.Cancelled && !all {
continue
}
if !MatchProvenance(canonicalProvenance, commitInfo.Provenance) {
continue
}
if commitType != pfs.CommitType_COMMIT_TYPE_NONE &&
commitType != commitInfo.CommitType {
continue
}
result = append(result, commitInfo)
}
}
}
return result, nil
}
func (d *driver) StartCommit(repo *pfs.Repo, commitID string, parentID string, branch string,
started *google_protobuf.Timestamp, provenance []*pfs.Commit, shards map[uint64]bool) error {
d.lock.Lock()
defer d.lock.Unlock()
// make sure that the parent commit exists
if parentID != "" {
_, err := d.inspectCommit(client.NewCommit(repo.Name, parentID), shards)
if err != nil {
return err
}
}
for shard := range shards {
if len(provenance) != 0 {
diffInfo, ok := d.diffs.get(client.NewDiff(repo.Name, "", shard))
if !ok {
return pfsserver.NewErrRepoNotFound(repo.Name)
}
provRepos := repoSetFromCommits(diffInfo.Provenance)
for _, provCommit := range provenance {
if !provRepos[provCommit.Repo.Name] {
return fmt.Errorf("cannot use %s/%s as provenance, %s is not provenance of %s",
provCommit.Repo.Name, provCommit.ID, provCommit.Repo.Name, repo.Name)
}
}
}
diffInfo := &pfs.DiffInfo{
Diff: client.NewDiff(repo.Name, commitID, shard),
Started: started,
Appends: make(map[string]*pfs.Append),
Branch: branch,
Provenance: provenance,
}
if branch != "" {
parentCommit, err := d.branchParent(client.NewCommit(repo.Name, commitID), branch)
if err != nil {
return err
}
if parentCommit != nil && parentID != "" {
return fmt.Errorf("branch %s already exists as %s, can't create with %s as parent",
branch, parentCommit.ID, parentID)
}
diffInfo.ParentCommit = parentCommit
}
if diffInfo.ParentCommit == nil && parentID != "" {
diffInfo.ParentCommit = client.NewCommit(repo.Name, parentID)
}
if err := d.insertDiffInfo(diffInfo); err != nil {
return err
}
}
d.commitConds[commitID] = sync.NewCond(&d.lock)
return nil
}
func (d *driver) ListBranch(repo *pfs.Repo, shards map[uint64]bool) ([]*pfs.CommitInfo, error) {
var result []*pfs.CommitInfo
_, ok := d.branches[repo.Name]
if !ok {
return nil, pfsserver.NewErrRepoNotFound(repo.Name)
}
for commitID := range d.branches[repo.Name] {
commitInfo, err := d.inspectCommit(client.NewCommit(repo.Name, commitID), shards)
if err != nil {
return nil, err
}
result = append(result, commitInfo)
}
return result, nil
}
func (d diffMap) insert(diffInfo *pfs.DiffInfo) error {
diff := diffInfo.Diff
shardMap, ok := d[diff.Commit.Repo.Name]
if !ok {
return pfsserver.NewErrRepoNotFound(diff.Commit.Repo.Name)
}
commitMap, ok := shardMap[diff.Shard]
if !ok {
commitMap = make(map[string]*pfs.DiffInfo)
shardMap[diff.Shard] = commitMap
}
if _, ok = commitMap[diff.Commit.ID]; ok {
return fmt.Errorf("commit %s/%s already exists", diff.Commit.Repo.Name, diff.Commit.ID)
}
commitMap[diff.Commit.ID] = diffInfo
return nil
}
func (d *driver) DeleteRepo(repo *pfs.Repo, shards map[uint64]bool) error {
// Make sure that this repo is not the provenance of any other repo
repoInfos, err := d.ListRepo([]*pfs.Repo{repo}, shards)
if err != nil {
return err
}
var diffInfos []*pfs.DiffInfo
err = func() error {
d.lock.Lock()
defer d.lock.Unlock()
if _, ok := d.diffs[repo.Name]; !ok {
return pfsserver.NewErrRepoNotFound(repo.Name)
}
if len(repoInfos) > 0 {
var repoNames []string
for _, repoInfo := range repoInfos {
repoNames = append(repoNames, repoInfo.Repo.Name)
}
return fmt.Errorf("cannot delete repo %v; it's the provenance of the following repos: %v", repo.Name, repoNames)
}
for shard := range shards {
for _, diffInfo := range d.diffs[repo.Name][shard] {
diffInfos = append(diffInfos, diffInfo)
}
}
delete(d.diffs, repo.Name)
return nil
}()
if err != nil {
return err
}
blockClient, err := d.getBlockClient()
if err != nil {
return err
}
errCh := make(chan error, 1)
var wg sync.WaitGroup
for _, diffInfo := range diffInfos {
diffInfo := diffInfo
wg.Add(1)
go func() {
defer wg.Done()
if _, err := blockClient.DeleteDiff(
context.Background(),
&pfs.DeleteDiffRequest{Diff: diffInfo.Diff},
); err != nil {
select {
case errCh <- err:
default:
}
return
}
}()
}
wg.Wait()
select {
case err := <-errCh:
return err
default:
}
return nil
}