Exemple #1
0
func TestAllCurrentObjectsSome(t *testing.T) {
	repo := test.NewRepo(t)
	repo.Pushd()
	defer func() {
		repo.Popd()
		repo.Cleanup()
	}()

	// We're not testing commits here, just storage, so just create a single
	// commit input with lots of files to generate many oids
	numFiles := 20
	files := make([]*test.FileInput, 0, numFiles)
	for i := 0; i < numFiles; i++ {
		// Must be >=16 bytes for each file to be unique
		files = append(files, &test.FileInput{Filename: fmt.Sprintf("file%d.txt", i), Size: 30})
	}

	inputs := []*test.CommitInput{
		{Files: files},
	}

	outputs := repo.AddCommits(inputs)

	expected := make([]*lfs.Pointer, 0, numFiles)
	for _, f := range outputs[0].Files {
		expected = append(expected, f)
	}

	actualObjects := lfs.AllObjects()
	actual := make([]*lfs.Pointer, len(actualObjects))
	for idx, f := range actualObjects {
		actual[idx] = lfs.NewPointer(f.Oid, f.Size, nil)
	}

	// sort to ensure comparison is equal
	sort.Sort(test.PointersByOid(expected))
	sort.Sort(test.PointersByOid(actual))
	assert.Equal(t, expected, actual, "Oids from disk should be the same as in commits")

}
Exemple #2
0
func prune(verifyRemote, dryRun, verbose bool) {
	localObjects := make([]localstorage.Object, 0, 100)
	retainedObjects := lfs.NewStringSetWithCapacity(100)
	var reachableObjects lfs.StringSet
	var taskwait sync.WaitGroup

	// Add all the base funcs to the waitgroup before starting them, in case
	// one completes really fast & hits 0 unexpectedly
	// each main process can Add() to the wg itself if it subdivides the task
	taskwait.Add(4) // 1..4: localObjects, current & recent refs, unpushed, worktree
	if verifyRemote {
		taskwait.Add(1) // 5
	}

	progressChan := make(PruneProgressChan, 100)

	// Collect errors
	errorChan := make(chan error, 10)
	var errorwait sync.WaitGroup
	errorwait.Add(1)
	var taskErrors []error
	go pruneTaskCollectErrors(&taskErrors, errorChan, &errorwait)

	// Populate the single list of local objects
	go pruneTaskGetLocalObjects(&localObjects, progressChan, &taskwait)

	// Now find files to be retained from many sources
	retainChan := make(chan string, 100)

	go pruneTaskGetRetainedCurrentAndRecentRefs(retainChan, errorChan, &taskwait)
	go pruneTaskGetRetainedUnpushed(retainChan, errorChan, &taskwait)
	go pruneTaskGetRetainedWorktree(retainChan, errorChan, &taskwait)
	if verifyRemote {
		reachableObjects = lfs.NewStringSetWithCapacity(100)
		go pruneTaskGetReachableObjects(&reachableObjects, errorChan, &taskwait)
	}

	// Now collect all the retained objects, on separate wait
	var retainwait sync.WaitGroup
	retainwait.Add(1)
	go pruneTaskCollectRetained(&retainedObjects, retainChan, progressChan, &retainwait)

	// Report progress
	var progresswait sync.WaitGroup
	progresswait.Add(1)
	go pruneTaskDisplayProgress(progressChan, &progresswait)

	taskwait.Wait()   // wait for subtasks
	close(retainChan) // triggers retain collector to end now all tasks have
	retainwait.Wait() // make sure all retained objects added

	close(errorChan) // triggers error collector to end now all tasks have
	errorwait.Wait() // make sure all errors have been processed
	pruneCheckErrors(taskErrors)

	prunableObjects := make([]string, 0, len(localObjects)/2)

	// Build list of prunables (also queue for verify at same time if applicable)
	var verifyQueue *lfs.TransferQueue
	var verifiedObjects lfs.StringSet
	var totalSize int64
	var verboseOutput bytes.Buffer
	if verifyRemote {
		lfs.Config.CurrentRemote = lfs.Config.FetchPruneConfig().PruneRemoteName
		// build queue now, no estimates or progress output
		verifyQueue = lfs.NewDownloadCheckQueue(0, 0, true)
		verifiedObjects = lfs.NewStringSetWithCapacity(len(localObjects) / 2)
	}
	for _, file := range localObjects {
		if !retainedObjects.Contains(file.Oid) {
			prunableObjects = append(prunableObjects, file.Oid)
			totalSize += file.Size
			if verbose {
				// Save up verbose output for the end, spinner still going
				verboseOutput.WriteString(fmt.Sprintf(" * %v (%v)\n", file.Oid, humanizeBytes(file.Size)))
			}
			if verifyRemote {
				tracerx.Printf("VERIFYING: %v", file.Oid)
				pointer := lfs.NewPointer(file.Oid, file.Size, nil)
				verifyQueue.Add(lfs.NewDownloadCheckable(&lfs.WrappedPointer{Pointer: pointer}))
			}
		}
	}

	if verifyRemote {
		// this channel is filled with oids for which Check() succeeded & Transfer() was called
		verifyc := verifyQueue.Watch()
		var verifywait sync.WaitGroup
		verifywait.Add(1)
		go func() {
			for oid := range verifyc {
				verifiedObjects.Add(oid)
				tracerx.Printf("VERIFIED: %v", oid)
				progressChan <- PruneProgress{PruneProgressTypeVerify, 1}
			}
			verifywait.Done()
		}()
		verifyQueue.Wait()
		verifywait.Wait()
		close(progressChan) // after verify (uses spinner) but before check
		progresswait.Wait()
		pruneCheckVerified(prunableObjects, reachableObjects, verifiedObjects)
	} else {
		close(progressChan)
		progresswait.Wait()
	}

	if len(prunableObjects) == 0 {
		Print("Nothing to prune")
		return
	}
	if dryRun {
		Print("%d files would be pruned (%v)", len(prunableObjects), humanizeBytes(totalSize))
		if verbose {
			Print(verboseOutput.String())
		}
	} else {
		Print("Pruning %d files, (%v)", len(prunableObjects), humanizeBytes(totalSize))
		if verbose {
			Print(verboseOutput.String())
		}
		pruneDeleteFiles(prunableObjects)
	}

}
Exemple #3
0
func pointerCommand(cmd *cobra.Command, args []string) {
	comparing := false
	something := false
	buildOid := ""
	compareOid := ""

	if len(pointerCompare) > 0 || pointerStdin {
		comparing = true
	}

	if len(pointerFile) > 0 {
		something = true
		buildFile, err := os.Open(pointerFile)
		if err != nil {
			Error(err.Error())
			os.Exit(1)
		}

		oidHash := sha256.New()
		size, err := io.Copy(oidHash, buildFile)
		buildFile.Close()

		if err != nil {
			Error(err.Error())
			os.Exit(1)
		}

		ptr := lfs.NewPointer(hex.EncodeToString(oidHash.Sum(nil)), size, nil)
		fmt.Fprintf(os.Stderr, "Git LFS pointer for %s\n\n", pointerFile)
		buf := &bytes.Buffer{}
		lfs.EncodePointer(io.MultiWriter(os.Stdout, buf), ptr)

		if comparing {
			buildOid = gitHashObject(buf.Bytes())
			fmt.Fprintf(os.Stderr, "\nGit blob OID: %s\n\n", buildOid)
		}
	} else {
		comparing = false
	}

	if len(pointerCompare) > 0 || pointerStdin {
		something = true
		compFile, err := pointerReader()
		if err != nil {
			Error(err.Error())
			os.Exit(1)
		}

		buf := &bytes.Buffer{}
		tee := io.TeeReader(compFile, buf)
		_, err = lfs.DecodePointer(tee)
		compFile.Close()

		pointerName := "STDIN"
		if !pointerStdin {
			pointerName = pointerCompare
		}
		fmt.Fprintf(os.Stderr, "Pointer from %s\n\n", pointerName)

		if err != nil {
			Error(err.Error())
			os.Exit(1)
		}

		fmt.Fprintf(os.Stderr, buf.String())
		if comparing {
			compareOid = gitHashObject(buf.Bytes())
			fmt.Fprintf(os.Stderr, "\nGit blob OID: %s\n", compareOid)
		}
	}

	if comparing && buildOid != compareOid {
		fmt.Fprintf(os.Stderr, "\nPointers do not match\n")
		os.Exit(1)
	}

	if !something {
		Error("Nothing to do!")
		os.Exit(1)
	}
}