// Fetch recent objects based on config func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) bool { fetchconf := lfs.Config.FetchPruneConfig() if fetchconf.FetchRecentRefsDays == 0 && fetchconf.FetchRecentCommitsDays == 0 { return true } ok := true // Make a list of what unique commits we've already fetched for to avoid duplicating work uniqueRefShas := make(map[string]string, len(alreadyFetchedRefs)) for _, ref := range alreadyFetchedRefs { uniqueRefShas[ref.Sha] = ref.Name } // First find any other recent refs if fetchconf.FetchRecentRefsDays > 0 { Print("Fetching recent branches within %v days", fetchconf.FetchRecentRefsDays) refsSince := time.Now().AddDate(0, 0, -fetchconf.FetchRecentRefsDays) refs, err := git.RecentBranches(refsSince, fetchconf.FetchRecentRefsIncludeRemotes, lfs.Config.CurrentRemote) if err != nil { Panic(err, "Could not scan for recent refs") } for _, ref := range refs { // Don't fetch for the same SHA twice if prevRefName, ok := uniqueRefShas[ref.Sha]; ok { if ref.Name != prevRefName { tracerx.Printf("Skipping fetch for %v, already fetched via %v", ref.Name, prevRefName) } } else { uniqueRefShas[ref.Sha] = ref.Name Print("Fetching %v", ref.Name) k := fetchRef(ref.Sha, include, exclude) ok = ok && k } } } // For every unique commit we've fetched, check recent commits too if fetchconf.FetchRecentCommitsDays > 0 { for commit, refName := range uniqueRefShas { // We measure from the last commit at the ref summ, err := git.GetCommitSummary(commit) if err != nil { Error("Couldn't scan commits at %v: %v", refName, err) continue } Print("Fetching changes within %v days of %v", fetchconf.FetchRecentCommitsDays, refName) commitsSince := summ.CommitDate.AddDate(0, 0, -fetchconf.FetchRecentCommitsDays) k := fetchPreviousVersions(commit, commitsSince, include, exclude) ok = ok && k } } return ok }
// Background task, must call waitg.Done() once at end func pruneTaskGetRetainedCurrentAndRecentRefs(retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) { defer waitg.Done() // We actually increment the waitg in this func since we kick off sub-goroutines // Make a list of what unique commits to keep, & search backward from commits := lfs.NewStringSet() // Do current first ref, err := git.CurrentRef() if err != nil { errorChan <- err return } commits.Add(ref.Sha) waitg.Add(1) go pruneTaskGetRetainedAtRef(ref.Sha, retainChan, errorChan, waitg) // Now recent fetchconf := lfs.Config.FetchPruneConfig() if fetchconf.FetchRecentRefsDays > 0 { pruneRefDays := fetchconf.FetchRecentRefsDays + fetchconf.PruneOffsetDays tracerx.Printf("PRUNE: Retaining non-HEAD refs within %d (%d+%d) days", pruneRefDays, fetchconf.FetchRecentRefsDays, fetchconf.PruneOffsetDays) refsSince := time.Now().AddDate(0, 0, -pruneRefDays) // Keep all recent refs including any recent remote branches refs, err := git.RecentBranches(refsSince, fetchconf.FetchRecentRefsIncludeRemotes, "") if err != nil { Panic(err, "Could not scan for recent refs") } for _, ref := range refs { if commits.Add(ref.Sha) { // A new commit waitg.Add(1) go pruneTaskGetRetainedAtRef(ref.Sha, retainChan, errorChan, waitg) } } } // For every unique commit we've fetched, check recent commits too // Only if we're fetching recent commits, otherwise only keep at refs if fetchconf.FetchRecentCommitsDays > 0 { pruneCommitDays := fetchconf.FetchRecentCommitsDays + fetchconf.PruneOffsetDays for commit := range commits.Iter() { // We measure from the last commit at the ref summ, err := git.GetCommitSummary(commit) if err != nil { errorChan <- fmt.Errorf("Couldn't scan commits at %v: %v", commit, err) continue } commitsSince := summ.CommitDate.AddDate(0, 0, -pruneCommitDays) waitg.Add(1) go pruneTaskGetPreviousVersionsOfRef(commit, commitsSince, retainChan, errorChan, waitg) } } }
func (repo *Repo) AddCommits(inputs []*CommitInput) []*CommitOutput { if repo.Settings.RepoType == RepoTypeBare { repo.callback.Fatalf("Cannot use AddCommits on a bare repo; clone it & push changes instead") } // Change to repo working dir oldwd, err := os.Getwd() if err != nil { repo.callback.Fatalf("Can't get cwd %v", err) } err = os.Chdir(repo.Path) if err != nil { repo.callback.Fatalf("Can't chdir to repo %v", err) } // Used to check whether we need to checkout another commit before lastBranch := "master" outputs := make([]*CommitOutput, 0, len(inputs)) for i, input := range inputs { output := &CommitOutput{} // first, are we on the correct branch if len(input.ParentBranches) > 0 { if input.ParentBranches[0] != lastBranch { RunGitCommand(repo.callback, true, "checkout", input.ParentBranches[0]) lastBranch = input.ParentBranches[0] } } // Is this a merge? if len(input.ParentBranches) > 1 { // Always take the *other* side in a merge so we adopt changes // also don't automatically commit, we'll do that below args := []string{"merge", "--no-ff", "--no-commit", "--strategy-option=theirs"} args = append(args, input.ParentBranches[1:]...) RunGitCommand(repo.callback, false, args...) } else if input.NewBranch != "" { RunGitCommand(repo.callback, true, "checkout", "-b", input.NewBranch) lastBranch = input.NewBranch } // Any files to write? for fi, infile := range input.Files { inputData := infile.DataReader if inputData == nil && infile.Data != "" { inputData = strings.NewReader(infile.Data) } if inputData == nil { // Different data for each file but deterministic inputData = NewPlaceholderDataReader(int64(i*fi), infile.Size) } cleaned, err := lfs.PointerClean(inputData, infile.Filename, infile.Size, nil) if err != nil { repo.callback.Errorf("Error creating pointer file: %v", err) continue } // this only created the temp file, move to final location tmpfile := cleaned.Filename mediafile, err := lfs.LocalMediaPath(cleaned.Oid) if err != nil { repo.callback.Errorf("Unable to get local media path: %v", err) continue } if _, err := os.Stat(mediafile); err != nil { if err := os.Rename(tmpfile, mediafile); err != nil { repo.callback.Errorf("Unable to move %s to %s: %v", tmpfile, mediafile, err) continue } } output.Files = append(output.Files, cleaned.Pointer) // Write pointer to local filename for adding (not using clean filter) os.MkdirAll(filepath.Dir(infile.Filename), 0755) f, err := os.Create(infile.Filename) if err != nil { repo.callback.Errorf("Error creating pointer file: %v", err) continue } _, err = cleaned.Pointer.Encode(f) if err != nil { f.Close() repo.callback.Errorf("Error encoding pointer file: %v", err) continue } f.Close() // early close in a loop, don't defer RunGitCommand(repo.callback, true, "add", infile.Filename) } // Now commit err = commitAtDate(input.CommitDate, input.CommitterName, input.CommitterEmail, fmt.Sprintf("Test commit %d", i)) if err != nil { repo.callback.Fatalf("Error committing: %v", err) } commit, err := git.GetCommitSummary("HEAD") if err != nil { repo.callback.Fatalf("Error determining commit SHA: %v", err) } // tags for _, tag := range input.Tags { // Use annotated tags, assume full release tags (also tag objects have edge cases) RunGitCommand(repo.callback, true, "tag", "-a", "-m", "Added tag", tag) } output.Sha = commit.Sha output.Parents = commit.Parents outputs = append(outputs, output) } // Restore cwd err = os.Chdir(oldwd) if err != nil { repo.callback.Fatalf("Can't restore old cwd %v", err) } return outputs }