Example #1
0
// fullCommitProvenance recursively computes the provenance of the commit,
// starting with the immediate provenance that was declared when the commit was
// created, then our immediate provenance's immediate provenance etc.
func (d *driver) fullCommitProvenance(commit *pfs.Commit, repoSet map[string]bool,
	shards map[uint64]bool) ([]*pfs.Commit, error) {
	shardToDiffInfo, ok := d.diffs[commit.Repo.Name]
	if !ok {
		return nil, pfsserver.NewErrRepoNotFound(commit.Repo.Name)
	}
	var result []*pfs.Commit
	for shard := range shards {
		diffInfos := shardToDiffInfo[shard]
		if !ok {
			return nil, fmt.Errorf("missing shard %d (this is likely a bug)", shard)
		}
		diffInfo, ok := diffInfos[commit.ID]
		if !ok {
			return nil, fmt.Errorf("missing \"%s\" diff (this is likely a bug)", commit.ID)
		}
		for _, provCommit := range diffInfo.Provenance {
			if !repoSet[provCommit.Repo.Name] {
				repoSet[provCommit.Repo.Name] = true
				result = append(result, provCommit)
				provCommits, err := d.fullCommitProvenance(provCommit, repoSet, shards)
				if err != nil {
					return nil, err
				}
				result = append(result, provCommits...)
			}
		}
		break // we only need to consider 1 shard
	}
	return result, nil
}
Example #2
0
// inspectRepo assumes that the lock is being held
func (d *driver) inspectRepo(repo *pfs.Repo, shards map[uint64]bool) (*pfs.RepoInfo, error) {
	result := &pfs.RepoInfo{
		Repo: repo,
	}
	shardToDiffInfo, ok := d.diffs[repo.Name]
	if !ok {
		return nil, pfsserver.NewErrRepoNotFound(repo.Name)
	}
	for shard := range shards {
		diffInfos, ok := shardToDiffInfo[shard]
		if !ok {
			continue
		}
		for _, diffInfo := range diffInfos {
			diffInfo := diffInfo
			if diffInfo.Diff.Commit.ID == "" && result.Created == nil {
				result.Created = diffInfo.Finished
			}
			result.SizeBytes += diffInfo.SizeBytes
		}
	}
	provenance, err := d.fullRepoProvenance(repo, shards)
	if err != nil {
		return nil, err
	}
	result.Provenance = provenance
	return result, nil
}
Example #3
0
// canonicalCommit finds the canonical way of referring to a commit
func (d *driver) canonicalCommit(commit *pfs.Commit) (*pfs.Commit, error) {
	if _, ok := d.branches[commit.Repo.Name]; !ok {
		return nil, pfsserver.NewErrRepoNotFound(commit.Repo.Name)
	}
	if commitID, ok := d.branches[commit.Repo.Name][commit.ID]; ok {
		return client.NewCommit(commit.Repo.Name, commitID), nil
	}
	return commit, nil
}
Example #4
0
func (d *driver) ListCommit(repos []*pfs.Repo, commitType pfs.CommitType, fromCommit []*pfs.Commit,
	provenance []*pfs.Commit, all bool, shards map[uint64]bool) ([]*pfs.CommitInfo, error) {
	repoSet := repoSet(repos)
	var canonicalProvenance []*pfs.Commit
	for _, provCommit := range provenance {
		canonicalCommit, err := d.canonicalCommit(provCommit)
		if err != nil {
			return nil, err
		}
		canonicalProvenance = append(canonicalProvenance, canonicalCommit)
	}
	breakCommitIDs := make(map[string]bool)
	for _, commit := range fromCommit {
		if !repoSet[commit.Repo.Name] {
			return nil, fmt.Errorf("Commit %s/%s is from a repo that isn't being listed.", commit.Repo.Name, commit.ID)
		}
		breakCommitIDs[commit.ID] = true
	}
	d.lock.RLock()
	defer d.lock.RUnlock()
	var result []*pfs.CommitInfo
	for _, repo := range repos {
		_, ok := d.diffs[repo.Name]
		if !ok {
			return nil, pfsserver.NewErrRepoNotFound(repo.Name)
		}
		for _, commitID := range d.dags[repo.Name].Leaves() {
			commit := &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			}
			for commit != nil && !breakCommitIDs[commit.ID] {
				// we add this commit to breakCommitIDs so we won't see it twice
				breakCommitIDs[commit.ID] = true
				commitInfo, err := d.inspectCommit(commit, shards)
				if err != nil {
					return nil, err
				}
				commit = commitInfo.ParentCommit
				if commitInfo.Cancelled && !all {
					continue
				}
				if !MatchProvenance(canonicalProvenance, commitInfo.Provenance) {
					continue
				}
				if commitType != pfs.CommitType_COMMIT_TYPE_NONE &&
					commitType != commitInfo.CommitType {
					continue
				}
				result = append(result, commitInfo)
			}
		}
	}
	return result, nil
}
Example #5
0
func (d *driver) StartCommit(repo *pfs.Repo, commitID string, parentID string, branch string,
	started *google_protobuf.Timestamp, provenance []*pfs.Commit, shards map[uint64]bool) error {
	d.lock.Lock()
	defer d.lock.Unlock()

	// make sure that the parent commit exists
	if parentID != "" {
		_, err := d.inspectCommit(client.NewCommit(repo.Name, parentID), shards)
		if err != nil {
			return err
		}
	}

	for shard := range shards {
		if len(provenance) != 0 {
			diffInfo, ok := d.diffs.get(client.NewDiff(repo.Name, "", shard))
			if !ok {
				return pfsserver.NewErrRepoNotFound(repo.Name)
			}
			provRepos := repoSetFromCommits(diffInfo.Provenance)
			for _, provCommit := range provenance {
				if !provRepos[provCommit.Repo.Name] {
					return fmt.Errorf("cannot use %s/%s as provenance, %s is not provenance of %s",
						provCommit.Repo.Name, provCommit.ID, provCommit.Repo.Name, repo.Name)
				}
			}
		}
		diffInfo := &pfs.DiffInfo{
			Diff:       client.NewDiff(repo.Name, commitID, shard),
			Started:    started,
			Appends:    make(map[string]*pfs.Append),
			Branch:     branch,
			Provenance: provenance,
		}
		if branch != "" {
			parentCommit, err := d.branchParent(client.NewCommit(repo.Name, commitID), branch)
			if err != nil {
				return err
			}
			if parentCommit != nil && parentID != "" {
				return fmt.Errorf("branch %s already exists as %s, can't create with %s as parent",
					branch, parentCommit.ID, parentID)
			}
			diffInfo.ParentCommit = parentCommit
		}
		if diffInfo.ParentCommit == nil && parentID != "" {
			diffInfo.ParentCommit = client.NewCommit(repo.Name, parentID)
		}
		if err := d.insertDiffInfo(diffInfo); err != nil {
			return err
		}
	}
	d.commitConds[commitID] = sync.NewCond(&d.lock)
	return nil
}
Example #6
0
func (d *driver) ListBranch(repo *pfs.Repo, shards map[uint64]bool) ([]*pfs.CommitInfo, error) {
	var result []*pfs.CommitInfo

	_, ok := d.branches[repo.Name]
	if !ok {
		return nil, pfsserver.NewErrRepoNotFound(repo.Name)
	}

	for commitID := range d.branches[repo.Name] {
		commitInfo, err := d.inspectCommit(client.NewCommit(repo.Name, commitID), shards)
		if err != nil {
			return nil, err
		}
		result = append(result, commitInfo)
	}
	return result, nil
}
Example #7
0
func (d diffMap) insert(diffInfo *pfs.DiffInfo) error {
	diff := diffInfo.Diff
	shardMap, ok := d[diff.Commit.Repo.Name]
	if !ok {
		return pfsserver.NewErrRepoNotFound(diff.Commit.Repo.Name)
	}
	commitMap, ok := shardMap[diff.Shard]
	if !ok {
		commitMap = make(map[string]*pfs.DiffInfo)
		shardMap[diff.Shard] = commitMap
	}
	if _, ok = commitMap[diff.Commit.ID]; ok {
		return fmt.Errorf("commit %s/%s already exists", diff.Commit.Repo.Name, diff.Commit.ID)
	}
	commitMap[diff.Commit.ID] = diffInfo
	return nil
}
Example #8
0
func (d *driver) DeleteRepo(repo *pfs.Repo, shards map[uint64]bool) error {
	// Make sure that this repo is not the provenance of any other repo
	repoInfos, err := d.ListRepo([]*pfs.Repo{repo}, shards)
	if err != nil {
		return err
	}

	var diffInfos []*pfs.DiffInfo
	err = func() error {
		d.lock.Lock()
		defer d.lock.Unlock()
		if _, ok := d.diffs[repo.Name]; !ok {
			return pfsserver.NewErrRepoNotFound(repo.Name)
		}
		if len(repoInfos) > 0 {
			var repoNames []string
			for _, repoInfo := range repoInfos {
				repoNames = append(repoNames, repoInfo.Repo.Name)
			}
			return fmt.Errorf("cannot delete repo %v; it's the provenance of the following repos: %v", repo.Name, repoNames)
		}

		for shard := range shards {
			for _, diffInfo := range d.diffs[repo.Name][shard] {
				diffInfos = append(diffInfos, diffInfo)
			}
		}
		delete(d.diffs, repo.Name)
		return nil
	}()
	if err != nil {
		return err
	}

	blockClient, err := d.getBlockClient()
	if err != nil {
		return err
	}
	errCh := make(chan error, 1)
	var wg sync.WaitGroup
	for _, diffInfo := range diffInfos {
		diffInfo := diffInfo
		wg.Add(1)
		go func() {
			defer wg.Done()
			if _, err := blockClient.DeleteDiff(
				context.Background(),
				&pfs.DeleteDiffRequest{Diff: diffInfo.Diff},
			); err != nil {
				select {
				case errCh <- err:
				default:
				}
				return
			}
		}()
	}
	wg.Wait()
	select {
	case err := <-errCh:
		return err
	default:
	}
	return nil
}