func checkoutWithIncludeExclude(include []string, exclude []string) {
ref, err := git.CurrentRef()
if err != nil {
Panic(err, "Could not checkout")
}
pointers, err := lfs.ScanTree(ref)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
var wait sync.WaitGroup
wait.Add(1)
c := make(chan *lfs.WrappedPointer)
go func() {
checkoutWithChan(c)
wait.Done()
}()
for _, pointer := range pointers {
if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
c <- pointer
}
}
close(c)
wait.Wait()
}
func smudgeCommand(cmd *cobra.Command, args []string) {
requireStdin("This command should be run by the Git 'smudge' filter")
lfs.InstallHooks(false)
// keeps the initial buffer from lfs.DecodePointer
b := &bytes.Buffer{}
r := io.TeeReader(os.Stdin, b)
ptr, err := lfs.DecodePointer(r)
if err != nil {
mr := io.MultiReader(b, os.Stdin)
_, err := io.Copy(os.Stdout, mr)
if err != nil {
Panic(err, "Error writing data to stdout:")
}
return
}
if smudgeInfo {
localPath, err := lfs.LocalMediaPath(ptr.Oid)
if err != nil {
Exit(err.Error())
}
stat, err := os.Stat(localPath)
if err != nil {
Print("%d --", ptr.Size)
} else {
Print("%d %s", stat.Size(), localPath)
}
return
}
filename := smudgeFilename(args, err)
cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
if err != nil {
Error(err.Error())
}
cfg := lfs.Config
download := lfs.FilenamePassesIncludeExcludeFilter(filename, cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
if smudgeSkip || lfs.Config.GetenvBool("GIT_LFS_SKIP_SMUDGE", false) {
download = false
}
err = ptr.Smudge(os.Stdout, filename, download, cb)
if file != nil {
file.Close()
}
if err != nil {
ptr.Encode(os.Stdout)
// Download declined error is ok to skip if we weren't requesting download
if !(lfs.IsDownloadDeclinedError(err) && !download) {
LoggedError(err, "Error accessing media: %s (%s)", filename, ptr.Oid)
os.Exit(2)
}
}
}
func readyAndMissingPointers(allpointers []*lfs.WrappedPointer, include, exclude []string) ([]*lfs.WrappedPointer, []*lfs.WrappedPointer, int64) {
size := int64(0)
seen := make(map[string]bool, len(allpointers))
missing := make([]*lfs.WrappedPointer, 0, len(allpointers))
ready := make([]*lfs.WrappedPointer, 0, len(allpointers))
for _, p := range allpointers {
// Filtered out by --include or --exclude
if !lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude) {
continue
}
// no need to download the same object multiple times
if seen[p.Oid] {
continue
}
seen[p.Oid] = true
// no need to download objects that exist locally already
lfs.LinkOrCopyFromReference(p.Oid, p.Size)
if lfs.ObjectExistsOfSize(p.Oid, p.Size) {
ready = append(ready, p)
continue
}
missing = append(missing, p)
size += p.Size
}
return ready, missing, size
}
// Fetch and report completion of each OID to a channel (optional, pass nil to skip)
func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) {
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
q := lfs.NewDownloadQueue(len(pointers), totalSize, false)
for _, p := range pointers {
// Only add to download queue if local file is not the right size already
// This avoids previous case of over-reporting a requirement for files we already have
// which would only be skipped by PointerSmudgeObject later
passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude)
if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter {
q.Add(lfs.NewDownloadable(p))
} else {
// If we already have it, or it won't be fetched
// report it to chan immediately to support pull/checkout
if out != nil {
out <- p
}
}
}
if out != nil {
dlwatch := q.Watch()
go func() {
// fetch only reports single OID, but OID *might* be referenced by multiple
// WrappedPointers if same content is at multiple paths, so map oid->slice
oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers))
for _, pointer := range pointers {
plist := oidToPointers[pointer.Oid]
oidToPointers[pointer.Oid] = append(plist, pointer)
}
for oid := range dlwatch {
plist, ok := oidToPointers[oid]
if !ok {
continue
}
for _, p := range plist {
out <- p
}
}
close(out)
}()
}
processQueue := time.Now()
q.Wait()
tracerx.PerformanceSince("process queue", processQueue)
}
func smudgeCommand(cmd *cobra.Command, args []string) {
requireStdin("This command should be run by the Git 'smudge' filter")
lfs.InstallHooks(false)
b := &bytes.Buffer{}
r := io.TeeReader(os.Stdin, b)
ptr, err := lfs.DecodePointer(r)
if err != nil {
mr := io.MultiReader(b, os.Stdin)
_, err := io.Copy(os.Stdout, mr)
if err != nil {
Panic(err, "Error writing data to stdout:")
}
return
}
if smudgeInfo {
localPath, err := lfs.LocalMediaPath(ptr.Oid)
if err != nil {
Exit(err.Error())
}
stat, err := os.Stat(localPath)
if err != nil {
Print("%d --", ptr.Size)
} else {
Print("%d %s", stat.Size(), localPath)
}
return
}
filename := smudgeFilename(args, err)
cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
if err != nil {
Error(err.Error())
}
cfg := lfs.Config
download := lfs.FilenamePassesIncludeExcludeFilter(filename, cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
err = ptr.Smudge(os.Stdout, filename, download, cb)
if file != nil {
file.Close()
}
if err != nil {
ptr.Encode(os.Stdout)
LoggedError(err, "Error accessing media: %s (%s)", filename, ptr.Oid)
}
}
func checkoutWithIncludeExclude(include []string, exclude []string) {
ref, err := git.CurrentRef()
if err != nil {
Panic(err, "Could not checkout")
}
pointers, err := lfs.ScanTree(ref.Sha)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
var wait sync.WaitGroup
wait.Add(1)
c := make(chan *lfs.WrappedPointer, 1)
go func() {
checkoutWithChan(c)
wait.Done()
}()
// Count bytes for progress
var totalBytes int64
for _, pointer := range pointers {
totalBytes += pointer.Size
}
logPath, _ := cfg.Os.Get("GIT_LFS_PROGRESS")
progress := progress.NewProgressMeter(len(pointers), totalBytes, false, logPath)
progress.Start()
totalBytes = 0
for _, pointer := range pointers {
totalBytes += pointer.Size
if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
progress.Add(pointer.Name)
c <- pointer
// not strictly correct (parallel) but we don't have a callback & it's just local
// plus only 1 slot in channel so it'll block & be close
progress.TransferBytes("checkout", pointer.Name, pointer.Size, totalBytes, int(pointer.Size))
progress.FinishTransfer(pointer.Name)
} else {
progress.Skip(pointer.Size)
}
}
close(c)
wait.Wait()
progress.Finish()
}
func checkoutFromFetchChan(include []string, exclude []string, in chan *lfs.WrappedPointer) {
ref, err := git.CurrentRef()
if err != nil {
Panic(err, "Could not checkout")
}
// Need to ScanTree to identify multiple files with the same content (fetch will only report oids once)
pointers, err := lfs.ScanTree(ref.Sha)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
// Map oid to multiple pointers
mapping := make(map[string][]*lfs.WrappedPointer)
for _, pointer := range pointers {
if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
mapping[pointer.Oid] = append(mapping[pointer.Oid], pointer)
}
}
// Launch git update-index
c := make(chan *lfs.WrappedPointer)
var wait sync.WaitGroup
wait.Add(1)
go func() {
checkoutWithChan(c)
wait.Done()
}()
// Feed it from in, which comes from fetch
for p := range in {
// Add all of the files for this oid
for _, fp := range mapping[p.Oid] {
c <- fp
}
}
close(c)
wait.Wait()
}