// Fetch recent objects based on config
func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) bool {
fetchconf := lfs.Config.FetchPruneConfig()
if fetchconf.FetchRecentRefsDays == 0 && fetchconf.FetchRecentCommitsDays == 0 {
return true
}
ok := true
// Make a list of what unique commits we've already fetched for to avoid duplicating work
uniqueRefShas := make(map[string]string, len(alreadyFetchedRefs))
for _, ref := range alreadyFetchedRefs {
uniqueRefShas[ref.Sha] = ref.Name
}
// First find any other recent refs
if fetchconf.FetchRecentRefsDays > 0 {
Print("Fetching recent branches within %v days", fetchconf.FetchRecentRefsDays)
refsSince := time.Now().AddDate(0, 0, -fetchconf.FetchRecentRefsDays)
refs, err := git.RecentBranches(refsSince, fetchconf.FetchRecentRefsIncludeRemotes, lfs.Config.CurrentRemote)
if err != nil {
Panic(err, "Could not scan for recent refs")
}
for _, ref := range refs {
// Don't fetch for the same SHA twice
if prevRefName, ok := uniqueRefShas[ref.Sha]; ok {
if ref.Name != prevRefName {
tracerx.Printf("Skipping fetch for %v, already fetched via %v", ref.Name, prevRefName)
}
} else {
uniqueRefShas[ref.Sha] = ref.Name
Print("Fetching %v", ref.Name)
k := fetchRef(ref.Sha, include, exclude)
ok = ok && k
}
}
}
// For every unique commit we've fetched, check recent commits too
if fetchconf.FetchRecentCommitsDays > 0 {
for commit, refName := range uniqueRefShas {
// We measure from the last commit at the ref
summ, err := git.GetCommitSummary(commit)
if err != nil {
Error("Couldn't scan commits at %v: %v", refName, err)
continue
}
Print("Fetching changes within %v days of %v", fetchconf.FetchRecentCommitsDays, refName)
commitsSince := summ.CommitDate.AddDate(0, 0, -fetchconf.FetchRecentCommitsDays)
k := fetchPreviousVersions(commit, commitsSince, include, exclude)
ok = ok && k
}
}
return ok
}
// Background task, must call waitg.Done() once at end
func pruneTaskGetRetainedCurrentAndRecentRefs(retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
defer waitg.Done()
// We actually increment the waitg in this func since we kick off sub-goroutines
// Make a list of what unique commits to keep, & search backward from
commits := lfs.NewStringSet()
// Do current first
ref, err := git.CurrentRef()
if err != nil {
errorChan <- err
return
}
commits.Add(ref.Sha)
waitg.Add(1)
go pruneTaskGetRetainedAtRef(ref.Sha, retainChan, errorChan, waitg)
// Now recent
fetchconf := lfs.Config.FetchPruneConfig()
if fetchconf.FetchRecentRefsDays > 0 {
pruneRefDays := fetchconf.FetchRecentRefsDays + fetchconf.PruneOffsetDays
tracerx.Printf("PRUNE: Retaining non-HEAD refs within %d (%d+%d) days", pruneRefDays, fetchconf.FetchRecentRefsDays, fetchconf.PruneOffsetDays)
refsSince := time.Now().AddDate(0, 0, -pruneRefDays)
// Keep all recent refs including any recent remote branches
refs, err := git.RecentBranches(refsSince, fetchconf.FetchRecentRefsIncludeRemotes, "")
if err != nil {
Panic(err, "Could not scan for recent refs")
}
for _, ref := range refs {
if commits.Add(ref.Sha) {
// A new commit
waitg.Add(1)
go pruneTaskGetRetainedAtRef(ref.Sha, retainChan, errorChan, waitg)
}
}
}
// For every unique commit we've fetched, check recent commits too
// Only if we're fetching recent commits, otherwise only keep at refs
if fetchconf.FetchRecentCommitsDays > 0 {
pruneCommitDays := fetchconf.FetchRecentCommitsDays + fetchconf.PruneOffsetDays
for commit := range commits.Iter() {
// We measure from the last commit at the ref
summ, err := git.GetCommitSummary(commit)
if err != nil {
errorChan <- fmt.Errorf("Couldn't scan commits at %v: %v", commit, err)
continue
}
commitsSince := summ.CommitDate.AddDate(0, 0, -pruneCommitDays)
waitg.Add(1)
go pruneTaskGetPreviousVersionsOfRef(commit, commitsSince, retainChan, errorChan, waitg)
}
}
}
func (repo *Repo) AddCommits(inputs []*CommitInput) []*CommitOutput {
if repo.Settings.RepoType == RepoTypeBare {
repo.callback.Fatalf("Cannot use AddCommits on a bare repo; clone it & push changes instead")
}
// Change to repo working dir
oldwd, err := os.Getwd()
if err != nil {
repo.callback.Fatalf("Can't get cwd %v", err)
}
err = os.Chdir(repo.Path)
if err != nil {
repo.callback.Fatalf("Can't chdir to repo %v", err)
}
// Used to check whether we need to checkout another commit before
lastBranch := "master"
outputs := make([]*CommitOutput, 0, len(inputs))
for i, input := range inputs {
output := &CommitOutput{}
// first, are we on the correct branch
if len(input.ParentBranches) > 0 {
if input.ParentBranches[0] != lastBranch {
RunGitCommand(repo.callback, true, "checkout", input.ParentBranches[0])
lastBranch = input.ParentBranches[0]
}
}
// Is this a merge?
if len(input.ParentBranches) > 1 {
// Always take the *other* side in a merge so we adopt changes
// also don't automatically commit, we'll do that below
args := []string{"merge", "--no-ff", "--no-commit", "--strategy-option=theirs"}
args = append(args, input.ParentBranches[1:]...)
RunGitCommand(repo.callback, false, args...)
} else if input.NewBranch != "" {
RunGitCommand(repo.callback, true, "checkout", "-b", input.NewBranch)
lastBranch = input.NewBranch
}
// Any files to write?
for fi, infile := range input.Files {
inputData := infile.DataReader
if inputData == nil && infile.Data != "" {
inputData = strings.NewReader(infile.Data)
}
if inputData == nil {
// Different data for each file but deterministic
inputData = NewPlaceholderDataReader(int64(i*fi), infile.Size)
}
cleaned, err := lfs.PointerClean(inputData, infile.Filename, infile.Size, nil)
if err != nil {
repo.callback.Errorf("Error creating pointer file: %v", err)
continue
}
// this only created the temp file, move to final location
tmpfile := cleaned.Filename
mediafile, err := lfs.LocalMediaPath(cleaned.Oid)
if err != nil {
repo.callback.Errorf("Unable to get local media path: %v", err)
continue
}
if _, err := os.Stat(mediafile); err != nil {
if err := os.Rename(tmpfile, mediafile); err != nil {
repo.callback.Errorf("Unable to move %s to %s: %v", tmpfile, mediafile, err)
continue
}
}
output.Files = append(output.Files, cleaned.Pointer)
// Write pointer to local filename for adding (not using clean filter)
os.MkdirAll(filepath.Dir(infile.Filename), 0755)
f, err := os.Create(infile.Filename)
if err != nil {
repo.callback.Errorf("Error creating pointer file: %v", err)
continue
}
_, err = cleaned.Pointer.Encode(f)
if err != nil {
f.Close()
repo.callback.Errorf("Error encoding pointer file: %v", err)
continue
}
f.Close() // early close in a loop, don't defer
RunGitCommand(repo.callback, true, "add", infile.Filename)
}
// Now commit
err = commitAtDate(input.CommitDate, input.CommitterName, input.CommitterEmail,
fmt.Sprintf("Test commit %d", i))
if err != nil {
repo.callback.Fatalf("Error committing: %v", err)
}
commit, err := git.GetCommitSummary("HEAD")
if err != nil {
repo.callback.Fatalf("Error determining commit SHA: %v", err)
}
// tags
for _, tag := range input.Tags {
// Use annotated tags, assume full release tags (also tag objects have edge cases)
RunGitCommand(repo.callback, true, "tag", "-a", "-m", "Added tag", tag)
}
output.Sha = commit.Sha
output.Parents = commit.Parents
outputs = append(outputs, output)
}
// Restore cwd
err = os.Chdir(oldwd)
if err != nil {
repo.callback.Fatalf("Can't restore old cwd %v", err)
}
return outputs
}