func TestAllCurrentObjectsSome(t *testing.T) { repo := test.NewRepo(t) repo.Pushd() defer func() { repo.Popd() repo.Cleanup() }() // We're not testing commits here, just storage, so just create a single // commit input with lots of files to generate many oids numFiles := 20 files := make([]*test.FileInput, 0, numFiles) for i := 0; i < numFiles; i++ { // Must be >=16 bytes for each file to be unique files = append(files, &test.FileInput{Filename: fmt.Sprintf("file%d.txt", i), Size: 30}) } inputs := []*test.CommitInput{ {Files: files}, } outputs := repo.AddCommits(inputs) expected := make([]*lfs.Pointer, 0, numFiles) for _, f := range outputs[0].Files { expected = append(expected, f) } actualObjects := lfs.AllObjects() actual := make([]*lfs.Pointer, len(actualObjects)) for idx, f := range actualObjects { actual[idx] = lfs.NewPointer(f.Oid, f.Size, nil) } // sort to ensure comparison is equal sort.Sort(test.PointersByOid(expected)) sort.Sort(test.PointersByOid(actual)) assert.Equal(t, expected, actual, "Oids from disk should be the same as in commits") }
func prune(verifyRemote, dryRun, verbose bool) { localObjects := make([]localstorage.Object, 0, 100) retainedObjects := lfs.NewStringSetWithCapacity(100) var reachableObjects lfs.StringSet var taskwait sync.WaitGroup // Add all the base funcs to the waitgroup before starting them, in case // one completes really fast & hits 0 unexpectedly // each main process can Add() to the wg itself if it subdivides the task taskwait.Add(4) // 1..4: localObjects, current & recent refs, unpushed, worktree if verifyRemote { taskwait.Add(1) // 5 } progressChan := make(PruneProgressChan, 100) // Collect errors errorChan := make(chan error, 10) var errorwait sync.WaitGroup errorwait.Add(1) var taskErrors []error go pruneTaskCollectErrors(&taskErrors, errorChan, &errorwait) // Populate the single list of local objects go pruneTaskGetLocalObjects(&localObjects, progressChan, &taskwait) // Now find files to be retained from many sources retainChan := make(chan string, 100) go pruneTaskGetRetainedCurrentAndRecentRefs(retainChan, errorChan, &taskwait) go pruneTaskGetRetainedUnpushed(retainChan, errorChan, &taskwait) go pruneTaskGetRetainedWorktree(retainChan, errorChan, &taskwait) if verifyRemote { reachableObjects = lfs.NewStringSetWithCapacity(100) go pruneTaskGetReachableObjects(&reachableObjects, errorChan, &taskwait) } // Now collect all the retained objects, on separate wait var retainwait sync.WaitGroup retainwait.Add(1) go pruneTaskCollectRetained(&retainedObjects, retainChan, progressChan, &retainwait) // Report progress var progresswait sync.WaitGroup progresswait.Add(1) go pruneTaskDisplayProgress(progressChan, &progresswait) taskwait.Wait() // wait for subtasks close(retainChan) // triggers retain collector to end now all tasks have retainwait.Wait() // make sure all retained objects added close(errorChan) // triggers error collector to end now all tasks have errorwait.Wait() // make sure all errors have been processed pruneCheckErrors(taskErrors) prunableObjects := make([]string, 0, len(localObjects)/2) // Build list of prunables (also queue for verify at same time if applicable) var verifyQueue *lfs.TransferQueue var verifiedObjects lfs.StringSet var totalSize int64 var verboseOutput bytes.Buffer if verifyRemote { lfs.Config.CurrentRemote = lfs.Config.FetchPruneConfig().PruneRemoteName // build queue now, no estimates or progress output verifyQueue = lfs.NewDownloadCheckQueue(0, 0, true) verifiedObjects = lfs.NewStringSetWithCapacity(len(localObjects) / 2) } for _, file := range localObjects { if !retainedObjects.Contains(file.Oid) { prunableObjects = append(prunableObjects, file.Oid) totalSize += file.Size if verbose { // Save up verbose output for the end, spinner still going verboseOutput.WriteString(fmt.Sprintf(" * %v (%v)\n", file.Oid, humanizeBytes(file.Size))) } if verifyRemote { tracerx.Printf("VERIFYING: %v", file.Oid) pointer := lfs.NewPointer(file.Oid, file.Size, nil) verifyQueue.Add(lfs.NewDownloadCheckable(&lfs.WrappedPointer{Pointer: pointer})) } } } if verifyRemote { // this channel is filled with oids for which Check() succeeded & Transfer() was called verifyc := verifyQueue.Watch() var verifywait sync.WaitGroup verifywait.Add(1) go func() { for oid := range verifyc { verifiedObjects.Add(oid) tracerx.Printf("VERIFIED: %v", oid) progressChan <- PruneProgress{PruneProgressTypeVerify, 1} } verifywait.Done() }() verifyQueue.Wait() verifywait.Wait() close(progressChan) // after verify (uses spinner) but before check progresswait.Wait() pruneCheckVerified(prunableObjects, reachableObjects, verifiedObjects) } else { close(progressChan) progresswait.Wait() } if len(prunableObjects) == 0 { Print("Nothing to prune") return } if dryRun { Print("%d files would be pruned (%v)", len(prunableObjects), humanizeBytes(totalSize)) if verbose { Print(verboseOutput.String()) } } else { Print("Pruning %d files, (%v)", len(prunableObjects), humanizeBytes(totalSize)) if verbose { Print(verboseOutput.String()) } pruneDeleteFiles(prunableObjects) } }
func pointerCommand(cmd *cobra.Command, args []string) { comparing := false something := false buildOid := "" compareOid := "" if len(pointerCompare) > 0 || pointerStdin { comparing = true } if len(pointerFile) > 0 { something = true buildFile, err := os.Open(pointerFile) if err != nil { Error(err.Error()) os.Exit(1) } oidHash := sha256.New() size, err := io.Copy(oidHash, buildFile) buildFile.Close() if err != nil { Error(err.Error()) os.Exit(1) } ptr := lfs.NewPointer(hex.EncodeToString(oidHash.Sum(nil)), size, nil) fmt.Fprintf(os.Stderr, "Git LFS pointer for %s\n\n", pointerFile) buf := &bytes.Buffer{} lfs.EncodePointer(io.MultiWriter(os.Stdout, buf), ptr) if comparing { buildOid = gitHashObject(buf.Bytes()) fmt.Fprintf(os.Stderr, "\nGit blob OID: %s\n\n", buildOid) } } else { comparing = false } if len(pointerCompare) > 0 || pointerStdin { something = true compFile, err := pointerReader() if err != nil { Error(err.Error()) os.Exit(1) } buf := &bytes.Buffer{} tee := io.TeeReader(compFile, buf) _, err = lfs.DecodePointer(tee) compFile.Close() pointerName := "STDIN" if !pointerStdin { pointerName = pointerCompare } fmt.Fprintf(os.Stderr, "Pointer from %s\n\n", pointerName) if err != nil { Error(err.Error()) os.Exit(1) } fmt.Fprintf(os.Stderr, buf.String()) if comparing { compareOid = gitHashObject(buf.Bytes()) fmt.Fprintf(os.Stderr, "\nGit blob OID: %s\n", compareOid) } } if comparing && buildOid != compareOid { fmt.Fprintf(os.Stderr, "\nPointers do not match\n") os.Exit(1) } if !something { Error("Nothing to do!") os.Exit(1) } }