Exemple #1
0
// This checks the given slice of pointers that don't exist in .git/lfs/objects
// against the server. Anything the server already has does not need to be
// uploaded again.
func (c *uploadContext) checkMissing(missing []*lfs.WrappedPointer, missingSize int64) {
	numMissing := len(missing)
	if numMissing == 0 {
		return
	}

	checkQueue := lfs.NewDownloadCheckQueue(numMissing, missingSize)
	for _, p := range missing {
		checkQueue.Add(lfs.NewDownloadable(p))
	}

	// this channel is filled with oids for which Check() succeeded & Transfer() was called
	transferc := checkQueue.Watch()
	done := make(chan int)
	go func() {
		for oid := range transferc {
			c.SetUploaded(oid)
		}
		done <- 1
	}()

	// Currently this is needed to flush the batch but is not enough to sync transferc completely
	checkQueue.Wait()
	<-done
}
Exemple #2
0
// Fetch and report completion of each OID to a channel (optional, pass nil to skip)
func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) {

	totalSize := int64(0)
	for _, p := range pointers {
		totalSize += p.Size
	}
	q := lfs.NewDownloadQueue(len(pointers), totalSize, false)

	for _, p := range pointers {
		// Only add to download queue if local file is not the right size already
		// This avoids previous case of over-reporting a requirement for files we already have
		// which would only be skipped by PointerSmudgeObject later
		passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude)
		if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter {
			q.Add(lfs.NewDownloadable(p))
		} else {
			// If we already have it, or it won't be fetched
			// report it to chan immediately to support pull/checkout
			if out != nil {
				out <- p
			}

		}
	}

	if out != nil {
		dlwatch := q.Watch()

		go func() {
			// fetch only reports single OID, but OID *might* be referenced by multiple
			// WrappedPointers if same content is at multiple paths, so map oid->slice
			oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers))
			for _, pointer := range pointers {
				plist := oidToPointers[pointer.Oid]
				oidToPointers[pointer.Oid] = append(plist, pointer)
			}

			for oid := range dlwatch {
				plist, ok := oidToPointers[oid]
				if !ok {
					continue
				}
				for _, p := range plist {
					out <- p
				}
			}
			close(out)
		}()

	}
	processQueue := time.Now()
	q.Wait()
	tracerx.PerformanceSince("process queue", processQueue)
}
Exemple #3
0
func prune(fetchPruneConfig config.FetchPruneConfig, verifyRemote, dryRun, verbose bool) {
	localObjects := make([]localstorage.Object, 0, 100)
	retainedObjects := tools.NewStringSetWithCapacity(100)
	var reachableObjects tools.StringSet
	var taskwait sync.WaitGroup

	// Add all the base funcs to the waitgroup before starting them, in case
	// one completes really fast & hits 0 unexpectedly
	// each main process can Add() to the wg itself if it subdivides the task
	taskwait.Add(4) // 1..4: localObjects, current & recent refs, unpushed, worktree
	if verifyRemote {
		taskwait.Add(1) // 5
	}

	progressChan := make(PruneProgressChan, 100)

	// Collect errors
	errorChan := make(chan error, 10)
	var errorwait sync.WaitGroup
	errorwait.Add(1)
	var taskErrors []error
	go pruneTaskCollectErrors(&taskErrors, errorChan, &errorwait)

	// Populate the single list of local objects
	go pruneTaskGetLocalObjects(&localObjects, progressChan, &taskwait)

	// Now find files to be retained from many sources
	retainChan := make(chan string, 100)

	go pruneTaskGetRetainedCurrentAndRecentRefs(fetchPruneConfig, retainChan, errorChan, &taskwait)
	go pruneTaskGetRetainedUnpushed(fetchPruneConfig, retainChan, errorChan, &taskwait)
	go pruneTaskGetRetainedWorktree(retainChan, errorChan, &taskwait)
	if verifyRemote {
		reachableObjects = tools.NewStringSetWithCapacity(100)
		go pruneTaskGetReachableObjects(&reachableObjects, errorChan, &taskwait)
	}

	// Now collect all the retained objects, on separate wait
	var retainwait sync.WaitGroup
	retainwait.Add(1)
	go pruneTaskCollectRetained(&retainedObjects, retainChan, progressChan, &retainwait)

	// Report progress
	var progresswait sync.WaitGroup
	progresswait.Add(1)
	go pruneTaskDisplayProgress(progressChan, &progresswait)

	taskwait.Wait()   // wait for subtasks
	close(retainChan) // triggers retain collector to end now all tasks have
	retainwait.Wait() // make sure all retained objects added

	close(errorChan) // triggers error collector to end now all tasks have
	errorwait.Wait() // make sure all errors have been processed
	pruneCheckErrors(taskErrors)

	prunableObjects := make([]string, 0, len(localObjects)/2)

	// Build list of prunables (also queue for verify at same time if applicable)
	var verifyQueue *lfs.TransferQueue
	var verifiedObjects tools.StringSet
	var totalSize int64
	var verboseOutput bytes.Buffer
	var verifyc chan string

	if verifyRemote {
		cfg.CurrentRemote = fetchPruneConfig.PruneRemoteName
		// build queue now, no estimates or progress output
		verifyQueue = lfs.NewDownloadCheckQueue(0, 0)
		verifiedObjects = tools.NewStringSetWithCapacity(len(localObjects) / 2)

		// this channel is filled with oids for which Check() succeeded & Transfer() was called
		verifyc = verifyQueue.Watch()
	}

	for _, file := range localObjects {
		if !retainedObjects.Contains(file.Oid) {
			prunableObjects = append(prunableObjects, file.Oid)
			totalSize += file.Size
			if verbose {
				// Save up verbose output for the end, spinner still going
				verboseOutput.WriteString(fmt.Sprintf(" * %v (%v)\n", file.Oid, humanizeBytes(file.Size)))
			}

			if verifyRemote {
				tracerx.Printf("VERIFYING: %v", file.Oid)
				pointer := lfs.NewPointer(file.Oid, file.Size, nil)
				verifyQueue.Add(lfs.NewDownloadable(&lfs.WrappedPointer{Pointer: pointer}))
			}
		}
	}

	if verifyRemote {
		var verifywait sync.WaitGroup
		verifywait.Add(1)
		go func() {
			for oid := range verifyc {
				verifiedObjects.Add(oid)
				tracerx.Printf("VERIFIED: %v", oid)
				progressChan <- PruneProgress{PruneProgressTypeVerify, 1}
			}
			verifywait.Done()
		}()
		verifyQueue.Wait()
		verifywait.Wait()
		close(progressChan) // after verify (uses spinner) but before check
		progresswait.Wait()
		pruneCheckVerified(prunableObjects, reachableObjects, verifiedObjects)
	} else {
		close(progressChan)
		progresswait.Wait()
	}

	if len(prunableObjects) == 0 {
		Print("Nothing to prune")
		return
	}
	if dryRun {
		Print("%d files would be pruned (%v)", len(prunableObjects), humanizeBytes(totalSize))
		if verbose {
			Print(verboseOutput.String())
		}
	} else {
		Print("Pruning %d files, (%v)", len(prunableObjects), humanizeBytes(totalSize))
		if verbose {
			Print(verboseOutput.String())
		}
		pruneDeleteFiles(prunableObjects)
	}

}
Exemple #4
0
func fetchCommand(cmd *cobra.Command, args []string) {
	var ref string
	var err error

	if len(args) == 1 {
		ref = args[0]
	} else {
		ref, err = git.CurrentRef()
		if err != nil {
			Panic(err, "Could not fetch")
		}
	}

	pointers, err := lfs.ScanRefs(ref, "")
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	q := lfs.NewDownloadQueue(lfs.Config.ConcurrentTransfers(), len(pointers))

	for _, p := range pointers {
		q.Add(lfs.NewDownloadable(p))
	}

	target, err := git.ResolveRef(ref)
	if err != nil {
		Panic(err, "Could not resolve git ref")
	}

	current, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not fetch the current git ref")
	}

	if target == current {
		// We just downloaded the files for the current ref, we can copy them into
		// the working directory and update the git index. We're doing this in a
		// goroutine so they can be copied as they come in, for efficiency.
		watch := q.Watch()

		go func() {
			files := make(map[string]*lfs.WrappedPointer, len(pointers))
			for _, pointer := range pointers {
				files[pointer.Oid] = pointer
			}

			// Fire up the update-index command
			cmd := exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
			stdin, err := cmd.StdinPipe()
			if err != nil {
				Panic(err, "Could not update the index")
			}

			if err := cmd.Start(); err != nil {
				Panic(err, "Could not update the index")
			}

			// As files come in, write them to the wd and update the index
			for oid := range watch {
				pointer, ok := files[oid]
				if !ok {
					continue
				}

				file, err := os.Create(pointer.Name)
				if err != nil {
					Panic(err, "Could not create working directory file")
				}

				if err := lfs.PointerSmudge(file, pointer.Pointer, pointer.Name, nil); err != nil {
					Panic(err, "Could not write working directory file")
				}
				file.Close()

				stdin.Write([]byte(pointer.Name + "\n"))
			}

			stdin.Close()
			if err := cmd.Wait(); err != nil {
				Panic(err, "Error updating the git index")
			}
		}()

		processQueue := time.Now()
		q.Process()
		tracerx.PerformanceSince("process queue", processQueue)
	}
}
Exemple #5
0
// Fetch and report completion of each OID to a channel (optional, pass nil to skip)
// Returns true if all completed with no errors, false if errors were written to stderr/log
func fetchAndReportToChan(allpointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) bool {
	// Lazily initialize the current remote.
	if len(cfg.CurrentRemote) == 0 {
		// Actively find the default remote, don't just assume origin
		defaultRemote, err := git.DefaultRemote()
		if err != nil {
			Exit("No default remote")
		}
		cfg.CurrentRemote = defaultRemote
	}

	ready, pointers, totalSize := readyAndMissingPointers(allpointers, include, exclude)
	q := lfs.NewDownloadQueue(len(pointers), totalSize, false)

	if out != nil {
		// If we already have it, or it won't be fetched
		// report it to chan immediately to support pull/checkout
		for _, p := range ready {
			out <- p
		}

		dlwatch := q.Watch()

		go func() {
			// fetch only reports single OID, but OID *might* be referenced by multiple
			// WrappedPointers if same content is at multiple paths, so map oid->slice
			oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers))
			for _, pointer := range pointers {
				plist := oidToPointers[pointer.Oid]
				oidToPointers[pointer.Oid] = append(plist, pointer)
			}

			for oid := range dlwatch {
				plist, ok := oidToPointers[oid]
				if !ok {
					continue
				}
				for _, p := range plist {
					out <- p
				}
			}
			close(out)
		}()
	}

	for _, p := range pointers {
		tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
		q.Add(lfs.NewDownloadable(p))
	}

	processQueue := time.Now()
	q.Wait()
	tracerx.PerformanceSince("process queue", processQueue)

	ok := true
	for _, err := range q.Errors() {
		ok = false
		FullError(err)
	}
	return ok
}