func TestAllCurrentObjectsSome(t *testing.T) {
repo := test.NewRepo(t)
repo.Pushd()
defer func() {
repo.Popd()
repo.Cleanup()
}()
// We're not testing commits here, just storage, so just create a single
// commit input with lots of files to generate many oids
numFiles := 20
files := make([]*test.FileInput, 0, numFiles)
for i := 0; i < numFiles; i++ {
// Must be >=16 bytes for each file to be unique
files = append(files, &test.FileInput{Filename: fmt.Sprintf("file%d.txt", i), Size: 30})
}
inputs := []*test.CommitInput{
{Files: files},
}
outputs := repo.AddCommits(inputs)
expected := make([]*lfs.Pointer, 0, numFiles)
for _, f := range outputs[0].Files {
expected = append(expected, f)
}
actualObjects := lfs.AllObjects()
actual := make([]*lfs.Pointer, len(actualObjects))
for idx, f := range actualObjects {
actual[idx] = lfs.NewPointer(f.Oid, f.Size, nil)
}
// sort to ensure comparison is equal
sort.Sort(test.PointersByOid(expected))
sort.Sort(test.PointersByOid(actual))
assert.Equal(t, expected, actual, "Oids from disk should be the same as in commits")
}
func prune(verifyRemote, dryRun, verbose bool) {
localObjects := make([]localstorage.Object, 0, 100)
retainedObjects := lfs.NewStringSetWithCapacity(100)
var reachableObjects lfs.StringSet
var taskwait sync.WaitGroup
// Add all the base funcs to the waitgroup before starting them, in case
// one completes really fast & hits 0 unexpectedly
// each main process can Add() to the wg itself if it subdivides the task
taskwait.Add(4) // 1..4: localObjects, current & recent refs, unpushed, worktree
if verifyRemote {
taskwait.Add(1) // 5
}
progressChan := make(PruneProgressChan, 100)
// Collect errors
errorChan := make(chan error, 10)
var errorwait sync.WaitGroup
errorwait.Add(1)
var taskErrors []error
go pruneTaskCollectErrors(&taskErrors, errorChan, &errorwait)
// Populate the single list of local objects
go pruneTaskGetLocalObjects(&localObjects, progressChan, &taskwait)
// Now find files to be retained from many sources
retainChan := make(chan string, 100)
go pruneTaskGetRetainedCurrentAndRecentRefs(retainChan, errorChan, &taskwait)
go pruneTaskGetRetainedUnpushed(retainChan, errorChan, &taskwait)
go pruneTaskGetRetainedWorktree(retainChan, errorChan, &taskwait)
if verifyRemote {
reachableObjects = lfs.NewStringSetWithCapacity(100)
go pruneTaskGetReachableObjects(&reachableObjects, errorChan, &taskwait)
}
// Now collect all the retained objects, on separate wait
var retainwait sync.WaitGroup
retainwait.Add(1)
go pruneTaskCollectRetained(&retainedObjects, retainChan, progressChan, &retainwait)
// Report progress
var progresswait sync.WaitGroup
progresswait.Add(1)
go pruneTaskDisplayProgress(progressChan, &progresswait)
taskwait.Wait() // wait for subtasks
close(retainChan) // triggers retain collector to end now all tasks have
retainwait.Wait() // make sure all retained objects added
close(errorChan) // triggers error collector to end now all tasks have
errorwait.Wait() // make sure all errors have been processed
pruneCheckErrors(taskErrors)
prunableObjects := make([]string, 0, len(localObjects)/2)
// Build list of prunables (also queue for verify at same time if applicable)
var verifyQueue *lfs.TransferQueue
var verifiedObjects lfs.StringSet
var totalSize int64
var verboseOutput bytes.Buffer
if verifyRemote {
lfs.Config.CurrentRemote = lfs.Config.FetchPruneConfig().PruneRemoteName
// build queue now, no estimates or progress output
verifyQueue = lfs.NewDownloadCheckQueue(0, 0, true)
verifiedObjects = lfs.NewStringSetWithCapacity(len(localObjects) / 2)
}
for _, file := range localObjects {
if !retainedObjects.Contains(file.Oid) {
prunableObjects = append(prunableObjects, file.Oid)
totalSize += file.Size
if verbose {
// Save up verbose output for the end, spinner still going
verboseOutput.WriteString(fmt.Sprintf(" * %v (%v)\n", file.Oid, humanizeBytes(file.Size)))
}
if verifyRemote {
tracerx.Printf("VERIFYING: %v", file.Oid)
pointer := lfs.NewPointer(file.Oid, file.Size, nil)
verifyQueue.Add(lfs.NewDownloadCheckable(&lfs.WrappedPointer{Pointer: pointer}))
}
}
}
if verifyRemote {
// this channel is filled with oids for which Check() succeeded & Transfer() was called
verifyc := verifyQueue.Watch()
var verifywait sync.WaitGroup
verifywait.Add(1)
go func() {
for oid := range verifyc {
verifiedObjects.Add(oid)
tracerx.Printf("VERIFIED: %v", oid)
progressChan <- PruneProgress{PruneProgressTypeVerify, 1}
}
verifywait.Done()
}()
verifyQueue.Wait()
verifywait.Wait()
close(progressChan) // after verify (uses spinner) but before check
progresswait.Wait()
pruneCheckVerified(prunableObjects, reachableObjects, verifiedObjects)
} else {
close(progressChan)
progresswait.Wait()
}
if len(prunableObjects) == 0 {
Print("Nothing to prune")
return
}
if dryRun {
Print("%d files would be pruned (%v)", len(prunableObjects), humanizeBytes(totalSize))
if verbose {
Print(verboseOutput.String())
}
} else {
Print("Pruning %d files, (%v)", len(prunableObjects), humanizeBytes(totalSize))
if verbose {
Print(verboseOutput.String())
}
pruneDeleteFiles(prunableObjects)
}
}
func pointerCommand(cmd *cobra.Command, args []string) {
comparing := false
something := false
buildOid := ""
compareOid := ""
if len(pointerCompare) > 0 || pointerStdin {
comparing = true
}
if len(pointerFile) > 0 {
something = true
buildFile, err := os.Open(pointerFile)
if err != nil {
Error(err.Error())
os.Exit(1)
}
oidHash := sha256.New()
size, err := io.Copy(oidHash, buildFile)
buildFile.Close()
if err != nil {
Error(err.Error())
os.Exit(1)
}
ptr := lfs.NewPointer(hex.EncodeToString(oidHash.Sum(nil)), size, nil)
fmt.Fprintf(os.Stderr, "Git LFS pointer for %s\n\n", pointerFile)
buf := &bytes.Buffer{}
lfs.EncodePointer(io.MultiWriter(os.Stdout, buf), ptr)
if comparing {
buildOid = gitHashObject(buf.Bytes())
fmt.Fprintf(os.Stderr, "\nGit blob OID: %s\n\n", buildOid)
}
} else {
comparing = false
}
if len(pointerCompare) > 0 || pointerStdin {
something = true
compFile, err := pointerReader()
if err != nil {
Error(err.Error())
os.Exit(1)
}
buf := &bytes.Buffer{}
tee := io.TeeReader(compFile, buf)
_, err = lfs.DecodePointer(tee)
compFile.Close()
pointerName := "STDIN"
if !pointerStdin {
pointerName = pointerCompare
}
fmt.Fprintf(os.Stderr, "Pointer from %s\n\n", pointerName)
if err != nil {
Error(err.Error())
os.Exit(1)
}
fmt.Fprintf(os.Stderr, buf.String())
if comparing {
compareOid = gitHashObject(buf.Bytes())
fmt.Fprintf(os.Stderr, "\nGit blob OID: %s\n", compareOid)
}
}
if comparing && buildOid != compareOid {
fmt.Fprintf(os.Stderr, "\nPointers do not match\n")
os.Exit(1)
}
if !something {
Error("Nothing to do!")
os.Exit(1)
}
}