func checkoutWithIncludeExclude(include []string, exclude []string) { ref, err := git.CurrentRef() if err != nil { Panic(err, "Could not checkout") } pointers, err := lfs.ScanTree(ref) if err != nil { Panic(err, "Could not scan for Git LFS files") } var wait sync.WaitGroup wait.Add(1) c := make(chan *lfs.WrappedPointer) go func() { checkoutWithChan(c) wait.Done() }() for _, pointer := range pointers { if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) { c <- pointer } } close(c) wait.Wait() }
func smudgeCommand(cmd *cobra.Command, args []string) { requireStdin("This command should be run by the Git 'smudge' filter") lfs.InstallHooks(false) // keeps the initial buffer from lfs.DecodePointer b := &bytes.Buffer{} r := io.TeeReader(os.Stdin, b) ptr, err := lfs.DecodePointer(r) if err != nil { mr := io.MultiReader(b, os.Stdin) _, err := io.Copy(os.Stdout, mr) if err != nil { Panic(err, "Error writing data to stdout:") } return } if smudgeInfo { localPath, err := lfs.LocalMediaPath(ptr.Oid) if err != nil { Exit(err.Error()) } stat, err := os.Stat(localPath) if err != nil { Print("%d --", ptr.Size) } else { Print("%d %s", stat.Size(), localPath) } return } filename := smudgeFilename(args, err) cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1) if err != nil { Error(err.Error()) } cfg := lfs.Config download := lfs.FilenamePassesIncludeExcludeFilter(filename, cfg.FetchIncludePaths(), cfg.FetchExcludePaths()) if smudgeSkip || lfs.Config.GetenvBool("GIT_LFS_SKIP_SMUDGE", false) { download = false } err = ptr.Smudge(os.Stdout, filename, download, cb) if file != nil { file.Close() } if err != nil { ptr.Encode(os.Stdout) // Download declined error is ok to skip if we weren't requesting download if !(lfs.IsDownloadDeclinedError(err) && !download) { LoggedError(err, "Error accessing media: %s (%s)", filename, ptr.Oid) os.Exit(2) } } }
func readyAndMissingPointers(allpointers []*lfs.WrappedPointer, include, exclude []string) ([]*lfs.WrappedPointer, []*lfs.WrappedPointer, int64) { size := int64(0) seen := make(map[string]bool, len(allpointers)) missing := make([]*lfs.WrappedPointer, 0, len(allpointers)) ready := make([]*lfs.WrappedPointer, 0, len(allpointers)) for _, p := range allpointers { // Filtered out by --include or --exclude if !lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude) { continue } // no need to download the same object multiple times if seen[p.Oid] { continue } seen[p.Oid] = true // no need to download objects that exist locally already lfs.LinkOrCopyFromReference(p.Oid, p.Size) if lfs.ObjectExistsOfSize(p.Oid, p.Size) { ready = append(ready, p) continue } missing = append(missing, p) size += p.Size } return ready, missing, size }
// Fetch and report completion of each OID to a channel (optional, pass nil to skip) func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) { totalSize := int64(0) for _, p := range pointers { totalSize += p.Size } q := lfs.NewDownloadQueue(len(pointers), totalSize, false) for _, p := range pointers { // Only add to download queue if local file is not the right size already // This avoids previous case of over-reporting a requirement for files we already have // which would only be skipped by PointerSmudgeObject later passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude) if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter { q.Add(lfs.NewDownloadable(p)) } else { // If we already have it, or it won't be fetched // report it to chan immediately to support pull/checkout if out != nil { out <- p } } } if out != nil { dlwatch := q.Watch() go func() { // fetch only reports single OID, but OID *might* be referenced by multiple // WrappedPointers if same content is at multiple paths, so map oid->slice oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers)) for _, pointer := range pointers { plist := oidToPointers[pointer.Oid] oidToPointers[pointer.Oid] = append(plist, pointer) } for oid := range dlwatch { plist, ok := oidToPointers[oid] if !ok { continue } for _, p := range plist { out <- p } } close(out) }() } processQueue := time.Now() q.Wait() tracerx.PerformanceSince("process queue", processQueue) }
func smudgeCommand(cmd *cobra.Command, args []string) { requireStdin("This command should be run by the Git 'smudge' filter") lfs.InstallHooks(false) b := &bytes.Buffer{} r := io.TeeReader(os.Stdin, b) ptr, err := lfs.DecodePointer(r) if err != nil { mr := io.MultiReader(b, os.Stdin) _, err := io.Copy(os.Stdout, mr) if err != nil { Panic(err, "Error writing data to stdout:") } return } if smudgeInfo { localPath, err := lfs.LocalMediaPath(ptr.Oid) if err != nil { Exit(err.Error()) } stat, err := os.Stat(localPath) if err != nil { Print("%d --", ptr.Size) } else { Print("%d %s", stat.Size(), localPath) } return } filename := smudgeFilename(args, err) cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1) if err != nil { Error(err.Error()) } cfg := lfs.Config download := lfs.FilenamePassesIncludeExcludeFilter(filename, cfg.FetchIncludePaths(), cfg.FetchExcludePaths()) err = ptr.Smudge(os.Stdout, filename, download, cb) if file != nil { file.Close() } if err != nil { ptr.Encode(os.Stdout) LoggedError(err, "Error accessing media: %s (%s)", filename, ptr.Oid) } }
func checkoutWithIncludeExclude(include []string, exclude []string) { ref, err := git.CurrentRef() if err != nil { Panic(err, "Could not checkout") } pointers, err := lfs.ScanTree(ref.Sha) if err != nil { Panic(err, "Could not scan for Git LFS files") } var wait sync.WaitGroup wait.Add(1) c := make(chan *lfs.WrappedPointer, 1) go func() { checkoutWithChan(c) wait.Done() }() // Count bytes for progress var totalBytes int64 for _, pointer := range pointers { totalBytes += pointer.Size } logPath, _ := cfg.Os.Get("GIT_LFS_PROGRESS") progress := progress.NewProgressMeter(len(pointers), totalBytes, false, logPath) progress.Start() totalBytes = 0 for _, pointer := range pointers { totalBytes += pointer.Size if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) { progress.Add(pointer.Name) c <- pointer // not strictly correct (parallel) but we don't have a callback & it's just local // plus only 1 slot in channel so it'll block & be close progress.TransferBytes("checkout", pointer.Name, pointer.Size, totalBytes, int(pointer.Size)) progress.FinishTransfer(pointer.Name) } else { progress.Skip(pointer.Size) } } close(c) wait.Wait() progress.Finish() }
func checkoutFromFetchChan(include []string, exclude []string, in chan *lfs.WrappedPointer) { ref, err := git.CurrentRef() if err != nil { Panic(err, "Could not checkout") } // Need to ScanTree to identify multiple files with the same content (fetch will only report oids once) pointers, err := lfs.ScanTree(ref.Sha) if err != nil { Panic(err, "Could not scan for Git LFS files") } // Map oid to multiple pointers mapping := make(map[string][]*lfs.WrappedPointer) for _, pointer := range pointers { if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) { mapping[pointer.Oid] = append(mapping[pointer.Oid], pointer) } } // Launch git update-index c := make(chan *lfs.WrappedPointer) var wait sync.WaitGroup wait.Add(1) go func() { checkoutWithChan(c) wait.Done() }() // Feed it from in, which comes from fetch for p := range in { // Add all of the files for this oid for _, fp := range mapping[p.Oid] { c <- fp } } close(c) wait.Wait() }