示例#1
0
func pruneTaskCollectRetained(outRetainedObjects *lfs.StringSet, retainChan chan string,
	progressChan PruneProgressChan, retainwait *sync.WaitGroup) {

	defer retainwait.Done()

	for oid := range retainChan {
		if outRetainedObjects.Add(oid) {
			progressChan <- PruneProgress{PruneProgressTypeRetain, 1}
		}
	}

}
示例#2
0
// Background task, must call waitg.Done() once at end
func pruneTaskGetReachableObjects(outObjectSet *lfs.StringSet, errorChan chan error, waitg *sync.WaitGroup) {
	defer waitg.Done()

	// converts to `git rev-list --all`
	// We only pick up objects in real commits and not the reflog
	opts := lfs.NewScanRefsOptions()
	opts.ScanMode = lfs.ScanAllMode
	opts.SkipDeletedBlobs = false

	pointerchan, err := lfs.ScanRefsToChan("", "", opts)
	if err != nil {
		errorChan <- fmt.Errorf("Error scanning for reachable objects: %v", err)
		return
	}

	for p := range pointerchan {
		outObjectSet.Add(p.Oid)
	}

}
示例#3
0
func pruneCheckVerified(prunableObjects []string, reachableObjects, verifiedObjects lfs.StringSet) {
	// There's no issue if an object is not reachable and missing, only if reachable & missing
	var problems bytes.Buffer
	for _, oid := range prunableObjects {
		// Test verified first as most likely reachable
		if !verifiedObjects.Contains(oid) {
			if reachableObjects.Contains(oid) {
				problems.WriteString(fmt.Sprintf(" * %v\n", oid))
			} else {
				// Just to indicate why it doesn't matter that we didn't verify
				tracerx.Printf("UNREACHABLE: %v", oid)
			}
		}
	}
	// technically we could still prune the other oids, but this indicates a
	// more serious issue because the local state implies that these can be
	// deleted but that's incorrect; bad state has occurred somehow, might need
	// push --all to resolve
	if problems.Len() > 0 {
		Exit("Abort: these objects to be pruned are missing on remote:\n%v", problems.String())
	}
}
示例#4
0
func prune(verifyRemote, dryRun, verbose bool) {
	localObjects := make([]localstorage.Object, 0, 100)
	retainedObjects := lfs.NewStringSetWithCapacity(100)
	var reachableObjects lfs.StringSet
	var taskwait sync.WaitGroup

	// Add all the base funcs to the waitgroup before starting them, in case
	// one completes really fast & hits 0 unexpectedly
	// each main process can Add() to the wg itself if it subdivides the task
	taskwait.Add(4) // 1..4: localObjects, current & recent refs, unpushed, worktree
	if verifyRemote {
		taskwait.Add(1) // 5
	}

	progressChan := make(PruneProgressChan, 100)

	// Collect errors
	errorChan := make(chan error, 10)
	var errorwait sync.WaitGroup
	errorwait.Add(1)
	var taskErrors []error
	go pruneTaskCollectErrors(&taskErrors, errorChan, &errorwait)

	// Populate the single list of local objects
	go pruneTaskGetLocalObjects(&localObjects, progressChan, &taskwait)

	// Now find files to be retained from many sources
	retainChan := make(chan string, 100)

	go pruneTaskGetRetainedCurrentAndRecentRefs(retainChan, errorChan, &taskwait)
	go pruneTaskGetRetainedUnpushed(retainChan, errorChan, &taskwait)
	go pruneTaskGetRetainedWorktree(retainChan, errorChan, &taskwait)
	if verifyRemote {
		reachableObjects = lfs.NewStringSetWithCapacity(100)
		go pruneTaskGetReachableObjects(&reachableObjects, errorChan, &taskwait)
	}

	// Now collect all the retained objects, on separate wait
	var retainwait sync.WaitGroup
	retainwait.Add(1)
	go pruneTaskCollectRetained(&retainedObjects, retainChan, progressChan, &retainwait)

	// Report progress
	var progresswait sync.WaitGroup
	progresswait.Add(1)
	go pruneTaskDisplayProgress(progressChan, &progresswait)

	taskwait.Wait()   // wait for subtasks
	close(retainChan) // triggers retain collector to end now all tasks have
	retainwait.Wait() // make sure all retained objects added

	close(errorChan) // triggers error collector to end now all tasks have
	errorwait.Wait() // make sure all errors have been processed
	pruneCheckErrors(taskErrors)

	prunableObjects := make([]string, 0, len(localObjects)/2)

	// Build list of prunables (also queue for verify at same time if applicable)
	var verifyQueue *lfs.TransferQueue
	var verifiedObjects lfs.StringSet
	var totalSize int64
	var verboseOutput bytes.Buffer
	if verifyRemote {
		lfs.Config.CurrentRemote = lfs.Config.FetchPruneConfig().PruneRemoteName
		// build queue now, no estimates or progress output
		verifyQueue = lfs.NewDownloadCheckQueue(0, 0, true)
		verifiedObjects = lfs.NewStringSetWithCapacity(len(localObjects) / 2)
	}
	for _, file := range localObjects {
		if !retainedObjects.Contains(file.Oid) {
			prunableObjects = append(prunableObjects, file.Oid)
			totalSize += file.Size
			if verbose {
				// Save up verbose output for the end, spinner still going
				verboseOutput.WriteString(fmt.Sprintf(" * %v (%v)\n", file.Oid, humanizeBytes(file.Size)))
			}
			if verifyRemote {
				tracerx.Printf("VERIFYING: %v", file.Oid)
				pointer := lfs.NewPointer(file.Oid, file.Size, nil)
				verifyQueue.Add(lfs.NewDownloadCheckable(&lfs.WrappedPointer{Pointer: pointer}))
			}
		}
	}

	if verifyRemote {
		// this channel is filled with oids for which Check() succeeded & Transfer() was called
		verifyc := verifyQueue.Watch()
		var verifywait sync.WaitGroup
		verifywait.Add(1)
		go func() {
			for oid := range verifyc {
				verifiedObjects.Add(oid)
				tracerx.Printf("VERIFIED: %v", oid)
				progressChan <- PruneProgress{PruneProgressTypeVerify, 1}
			}
			verifywait.Done()
		}()
		verifyQueue.Wait()
		verifywait.Wait()
		close(progressChan) // after verify (uses spinner) but before check
		progresswait.Wait()
		pruneCheckVerified(prunableObjects, reachableObjects, verifiedObjects)
	} else {
		close(progressChan)
		progresswait.Wait()
	}

	if len(prunableObjects) == 0 {
		Print("Nothing to prune")
		return
	}
	if dryRun {
		Print("%d files would be pruned (%v)", len(prunableObjects), humanizeBytes(totalSize))
		if verbose {
			Print(verboseOutput.String())
		}
	} else {
		Print("Pruning %d files, (%v)", len(prunableObjects), humanizeBytes(totalSize))
		if verbose {
			Print(verboseOutput.String())
		}
		pruneDeleteFiles(prunableObjects)
	}

}
示例#5
0
func prePushRef(left, right string) {
	// Just use scanner here
	scanOpt := lfs.NewScanRefsOptions()
	scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
	scanOpt.RemoteName = lfs.Config.CurrentRemote

	pointers, err := lfs.ScanRefs(left, right, scanOpt)
	if err != nil {
		Panic(err, "Error scanning for Git LFS files")
	}

	totalSize := int64(0)
	for _, p := range pointers {
		totalSize += p.Size
	}

	// Objects to skip because they're missing locally but on server
	var skipObjects lfs.StringSet

	if !prePushDryRun {
		// Do this as a pre-flight check since upload queue starts immediately
		skipObjects = prePushCheckForMissingObjects(pointers)
	}

	uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, prePushDryRun)

	for _, pointer := range pointers {
		if prePushDryRun {
			Print("push %s => %s", pointer.Oid, pointer.Name)
			continue
		}

		if skipObjects.Contains(pointer.Oid) {
			// object missing locally but on server, don't bother
			continue
		}

		u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
		if err != nil {
			if lfs.IsCleanPointerError(err) {
				Exit(prePushMissingErrMsg, pointer.Name, lfs.ErrorGetContext(err, "pointer").(*lfs.Pointer).Oid)
			} else if Debugging || lfs.IsFatalError(err) {
				Panic(err, err.Error())
			} else {
				Exit(err.Error())
			}
		}

		uploadQueue.Add(u)
	}

	if !prePushDryRun {
		uploadQueue.Wait()
		for _, err := range uploadQueue.Errors() {
			if Debugging || lfs.IsFatalError(err) {
				LoggedError(err, err.Error())
			} else {
				Error(err.Error())
			}
		}

		if len(uploadQueue.Errors()) > 0 {
			os.Exit(2)
		}
	}

}