// Fetch all binaries for a given ref (that we don't have already)
func fetchRef(ref string, include, exclude []string) {
pointers, err := lfs.ScanRefs(ref, "", nil)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
fetchPointers(pointers, include, exclude)
}
func uploadsBetweenRefs(left string, right string) *lfs.TransferQueue {
// Just use scanner here
pointers, err := lfs.ScanRefs(left, right, nil)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, pushDryRun)
for i, pointer := range pointers {
if pushDryRun {
Print("push %s", pointer.Name)
continue
}
tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name)
if wErr != nil {
if Debugging || wErr.Panic {
Panic(wErr.Err, wErr.Error())
} else {
Exit(wErr.Error())
}
}
uploadQueue.Add(u)
}
return uploadQueue
}
func pointersToFetchForRef(ref string) ([]*lfs.WrappedPointer, error) {
// Use SkipDeletedBlobs to avoid fetching ALL previous versions of modified files
opts := lfs.NewScanRefsOptions()
opts.ScanMode = lfs.ScanRefsMode
opts.SkipDeletedBlobs = true
return lfs.ScanRefs(ref, "", opts)
}
func uploadsBetweenRefAndRemote(ctx *uploadContext, refnames []string) {
tracerx.Printf("Upload refs %v to remote %v", refnames, lfs.Config.CurrentRemote)
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = lfs.Config.CurrentRemote
if pushAll {
scanOpt.ScanMode = lfs.ScanRefsMode
}
refs, err := refsByNames(refnames)
if err != nil {
Error(err.Error())
Exit("Error getting local refs.")
}
for _, ref := range refs {
pointers, err := lfs.ScanRefs(ref.Name, "", scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files in the %q ref", ref.Name)
}
upload(ctx, pointers)
}
}
func uploadsBetweenRefs(left string, right string) *lfs.TransferQueue {
tracerx.Printf("Upload between %v and %v", left, right)
// Just use scanner here
pointers, err := lfs.ScanRefs(left, right, nil)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
return uploadPointers(pointers)
}
func fetchRefToChan(ref string, include, exclude []string) chan *lfs.WrappedPointer {
c := make(chan *lfs.WrappedPointer)
pointers, err := lfs.ScanRefs(ref, "", nil)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
go fetchAndReportToChan(pointers, include, exclude, c)
return c
}
func uploadsBetweenRefAndRemote(ref, remote string) *lfs.TransferQueue {
tracerx.Printf("Upload between %v and remote %v", ref, remote)
scanOpt := &lfs.ScanRefsOptions{ScanMode: lfs.ScanLeftToRemoteMode, RemoteName: remote}
pointers, err := lfs.ScanRefs(ref, "", scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
return uploadPointers(pointers)
}
func uploadsBetweenRefs(ctx *uploadContext, left string, right string) {
tracerx.Printf("Upload between %v and %v", left, right)
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanRefsMode
scanOpt.RemoteName = lfs.Config.CurrentRemote
pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
upload(ctx, pointers)
}
// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
// two arguments on the command line:
//
// 1. Name of the remote to which the push is being done
// 2. URL to which the push is being done
//
// The hook receives commit information on stdin in the form:
// <local ref> <local sha1> <remote ref> <remote sha1>
//
// In the typical case, prePushCommand will get a list of git objects being
// pushed by using the following:
//
// git rev-list --objects <local sha1> ^<remote sha1>
//
// If any of those git objects are associated with Git LFS objects, those
// objects will be pushed to the Git LFS API.
//
// In the case of pushing a new branch, the list of git objects will be all of
// the git objects in this branch.
//
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {
if len(args) == 0 {
Print("This should be run through Git's pre-push hook. Run `git lfs update` to install it.")
os.Exit(1)
}
requireGitVersion()
// Remote is first arg
if err := git.ValidateRemote(args[0]); err != nil {
Exit("Invalid remote name %q", args[0])
}
cfg.CurrentRemote = args[0]
ctx := newUploadContext(prePushDryRun)
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = cfg.CurrentRemote
// We can be passed multiple lines of refs
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if len(line) == 0 {
continue
}
left, right := decodeRefs(line)
if left == prePushDeleteBranch {
continue
}
pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
upload(ctx, pointers)
}
}
func uploadsBetweenRefAndRemote(remote string, refs []string) *lfs.TransferQueue {
tracerx.Printf("Upload refs %v to remote %v", remote, refs)
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = remote
if pushAll {
if len(refs) == 0 {
pointers := scanAll()
Print("Pushing objects...")
return uploadPointers(pointers)
} else {
scanOpt.ScanMode = lfs.ScanRefsMode
}
}
// keep a unique set of pointers
oidPointerMap := make(map[string]*lfs.WrappedPointer)
for _, ref := range refs {
pointers, err := lfs.ScanRefs(ref, "", scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files in the %q ref", ref)
}
for _, p := range pointers {
oidPointerMap[p.Oid] = p
}
}
i := 0
pointers := make([]*lfs.WrappedPointer, len(oidPointerMap))
for _, pointer := range oidPointerMap {
pointers[i] = pointer
i += 1
}
return uploadPointers(pointers)
}
func lsFilesCommand(cmd *cobra.Command, args []string) {
var ref string
var err error
if len(args) == 1 {
ref = args[0]
} else {
ref, err = git.CurrentRef()
if err != nil {
Panic(err, "Could not ls-files")
}
}
pointers, err := lfs.ScanRefs(ref, "")
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
for _, p := range pointers {
Print(p.Name)
}
}
func lsFilesCommand(cmd *cobra.Command, args []string) {
var ref string
var err error
if len(args) == 1 {
ref = args[0]
} else {
fullref, err := git.CurrentRef()
if err != nil {
Panic(err, "Could not ls-files")
}
ref = fullref.Sha
}
scanOpt := &lfs.ScanRefsOptions{SkipDeletedBlobs: true}
pointers, err := lfs.ScanRefs(ref, "", scanOpt)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
for _, p := range pointers {
Print(p.Name)
}
}
// pushCommand pushes local objects to a Git LFS server. It takes two
// arguments:
//
// `<remote> <remote ref>`
//
// Both a remote name ("origin") or a remote URL are accepted.
//
// pushCommand calculates the git objects to send by looking comparing the range
// of commits between the local and remote git servers.
func pushCommand(cmd *cobra.Command, args []string) {
var left, right string
if len(args) == 0 {
Print("Specify a remote and a remote branch name (`git lfs push origin master`)")
os.Exit(1)
}
lfs.Config.CurrentRemote = args[0]
if useStdin {
requireStdin("Run this command from the Git pre-push hook, or leave the --stdin flag off.")
// called from a pre-push hook! Update the existing pre-push hook if it's
// one that git-lfs set.
lfs.InstallHooks(false)
refsData, err := ioutil.ReadAll(os.Stdin)
if err != nil {
Panic(err, "Error reading refs on stdin")
}
if len(refsData) == 0 {
return
}
left, right = decodeRefs(string(refsData))
if left == pushDeleteBranch {
return
}
} else {
var remoteArg, refArg string
if len(args) < 1 {
Print("Usage: git lfs push --dry-run <remote> [ref]")
return
}
remoteArg = args[0]
if len(args) == 2 {
refArg = args[1]
}
localRef, err := git.CurrentRef()
if err != nil {
Panic(err, "Error getting local ref")
}
left = localRef
remoteRef, err := git.LsRemote(remoteArg, refArg)
if err != nil {
Panic(err, "Error getting remote ref")
}
if remoteRef != "" {
right = "^" + strings.Split(remoteRef, "\t")[0]
}
}
// Just use scanner here
pointers, err := lfs.ScanRefs(left, right)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
uploadQueue := lfs.NewUploadQueue(lfs.Config.ConcurrentTransfers(), len(pointers))
for i, pointer := range pointers {
if pushDryRun {
Print("push %s", pointer.Name)
continue
}
tracerx.Printf("checking_asset: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name, i+1, len(pointers))
if wErr != nil {
if Debugging || wErr.Panic {
Panic(wErr.Err, wErr.Error())
} else {
Exit(wErr.Error())
}
}
uploadQueue.Add(u)
}
if !pushDryRun {
uploadQueue.Process()
for _, err := range uploadQueue.Errors() {
if Debugging || err.Panic {
LoggedError(err.Err, err.Error())
} else {
Error(err.Error())
}
}
if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
}
}
func doFsck() (bool, error) {
requireInRepo()
ref, err := git.CurrentRef()
if err != nil {
return false, err
}
// The LFS scanner methods return unexported *lfs.wrappedPointer objects.
// All we care about is the pointer OID and file name
pointerIndex := make(map[string]string)
pointers, err := lfs.ScanRefs(ref.Sha, "", nil)
if err != nil {
return false, err
}
for _, p := range pointers {
pointerIndex[p.Oid] = p.Name
}
// TODO(zeroshirts): do we want to look for LFS stuff in past commits?
p2, err := lfs.ScanIndex()
if err != nil {
return false, err
}
for _, p := range p2 {
pointerIndex[p.Oid] = p.Name
}
ok := true
for oid, name := range pointerIndex {
path := lfs.LocalMediaPathReadOnly(oid)
Debug("Examining %v (%v)", name, path)
f, err := os.Open(path)
if pErr, pOk := err.(*os.PathError); pOk {
Print("Object %s (%s) could not be checked: %s", name, oid, pErr.Err)
ok = false
continue
}
if err != nil {
return false, err
}
oidHash := sha256.New()
_, err = io.Copy(oidHash, f)
f.Close()
if err != nil {
return false, err
}
recalculatedOid := hex.EncodeToString(oidHash.Sum(nil))
if recalculatedOid != oid {
ok = false
Print("Object %s (%s) is corrupt", name, oid)
if fsckDryRun {
continue
}
badDir := filepath.Join(config.LocalGitStorageDir, "lfs", "bad")
if err := os.MkdirAll(badDir, 0755); err != nil {
return false, err
}
badFile := filepath.Join(badDir, oid)
if err := os.Rename(path, badFile); err != nil {
return false, err
}
Print(" moved to %s", badFile)
}
}
return ok, nil
}
// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
// two arguments on the command line:
//
// 1. Name of the remote to which the push is being done
// 2. URL to which the push is being done
//
// The hook receives commit information on stdin in the form:
// <local ref> <local sha1> <remote ref> <remote sha1>
//
// In the typical case, prePushCommand will get a list of git objects being
// pushed by using the following:
//
// git rev-list --objects <local sha1> ^<remote sha1>
//
// If any of those git objects are associated with Git LFS objects, those
// objects will be pushed to the Git LFS API.
//
// In the case of pushing a new branch, the list of git objects will be all of
// the git objects in this branch.
//
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {
var left, right string
if len(args) == 0 {
Print("This should be run through Git's pre-push hook. Run `git lfs update` to install it.")
os.Exit(1)
}
lfs.Config.CurrentRemote = args[0]
refsData, err := ioutil.ReadAll(os.Stdin)
if err != nil {
Panic(err, "Error reading refs on stdin")
}
if len(refsData) == 0 {
return
}
left, right = decodeRefs(string(refsData))
if left == prePushDeleteBranch {
return
}
// Just use scanner here
pointers, err := lfs.ScanRefs(left, right, nil)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
uploadQueue := lfs.NewUploadQueue(lfs.Config.ConcurrentTransfers(), len(pointers))
for _, pointer := range pointers {
if prePushDryRun {
Print("push %s", pointer.Name)
continue
}
u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name)
if wErr != nil {
if cleanPointerErr, ok := wErr.Err.(*lfs.CleanedPointerError); ok {
Exit("%s is an LFS pointer to %s, which does not exist in .git/lfs/objects.\n\nRun 'git lfs fsck' to verify Git LFS objects.",
pointer.Name, cleanPointerErr.Pointer.Oid)
} else if Debugging || wErr.Panic {
Panic(wErr.Err, wErr.Error())
} else {
Exit(wErr.Error())
}
}
uploadQueue.Add(u)
}
if !prePushDryRun {
uploadQueue.Process()
for _, err := range uploadQueue.Errors() {
if Debugging || err.Panic {
LoggedError(err.Err, err.Error())
} else {
Error(err.Error())
}
}
if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
}
}
func prePushRef(left, right string) {
// Just use scanner here
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = lfs.Config.CurrentRemote
pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
// Objects to skip because they're missing locally but on server
var skipObjects lfs.StringSet
if !prePushDryRun {
// Do this as a pre-flight check since upload queue starts immediately
skipObjects = prePushCheckForMissingObjects(pointers)
}
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, prePushDryRun)
for _, pointer := range pointers {
if prePushDryRun {
Print("push %s => %s", pointer.Oid, pointer.Name)
continue
}
if skipObjects.Contains(pointer.Oid) {
// object missing locally but on server, don't bother
continue
}
u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
if err != nil {
if lfs.IsCleanPointerError(err) {
Exit(prePushMissingErrMsg, pointer.Name, lfs.ErrorGetContext(err, "pointer").(*lfs.Pointer).Oid)
} else if Debugging || lfs.IsFatalError(err) {
Panic(err, err.Error())
} else {
Exit(err.Error())
}
}
uploadQueue.Add(u)
}
if !prePushDryRun {
uploadQueue.Wait()
for _, err := range uploadQueue.Errors() {
if Debugging || lfs.IsFatalError(err) {
LoggedError(err, err.Error())
} else {
Error(err.Error())
}
}
if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
}
}
func statusCommand(cmd *cobra.Command, args []string) {
requireInRepo()
ref, err := git.CurrentRef()
if err != nil {
Panic(err, "Could not get the current ref")
}
stagedPointers, err := lfs.ScanIndex()
if err != nil {
Panic(err, "Could not scan staging for Git LFS objects")
}
if porcelain {
for _, p := range stagedPointers {
switch p.Status {
case "R", "C":
Print("%s %s -> %s %d", p.Status, p.SrcName, p.Name, p.Size)
case "M":
Print(" %s %s %d", p.Status, p.Name, p.Size)
default:
Print("%s %s %d", p.Status, p.Name, p.Size)
}
}
return
}
Print("On branch %s", ref.Name)
remoteRef, err := git.CurrentRemoteRef()
if err == nil {
pointers, err := lfs.ScanRefs(ref.Sha, "^"+remoteRef.Sha, nil)
if err != nil {
Panic(err, "Could not scan for Git LFS objects")
}
Print("Git LFS objects to be pushed to %s:\n", remoteRef.Name)
for _, p := range pointers {
Print("\t%s (%s)", p.Name, humanizeBytes(p.Size))
}
}
Print("\nGit LFS objects to be committed:\n")
for _, p := range stagedPointers {
switch p.Status {
case "R", "C":
Print("\t%s -> %s (%s)", p.SrcName, p.Name, humanizeBytes(p.Size))
case "M":
default:
Print("\t%s (%s)", p.Name, humanizeBytes(p.Size))
}
}
Print("\nGit LFS objects not staged for commit:\n")
for _, p := range stagedPointers {
if p.Status == "M" {
Print("\t%s", p.Name)
}
}
Print("")
}
// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
// two arguments on the command line:
//
// 1. Name of the remote to which the push is being done
// 2. URL to which the push is being done
//
// The hook receives commit information on stdin in the form:
// <local ref> <local sha1> <remote ref> <remote sha1>
//
// In the typical case, prePushCommand will get a list of git objects being
// pushed by using the following:
//
// git rev-list --objects <local sha1> ^<remote sha1>
//
// If any of those git objects are associated with Git LFS objects, those
// objects will be pushed to the Git LFS API.
//
// In the case of pushing a new branch, the list of git objects will be all of
// the git objects in this branch.
//
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {
var left, right string
if len(args) == 0 {
Print("This should be run through Git's pre-push hook. Run `git lfs update` to install it.")
os.Exit(1)
}
lfs.Config.CurrentRemote = args[0]
refsData, err := ioutil.ReadAll(os.Stdin)
if err != nil {
Panic(err, "Error reading refs on stdin")
}
if len(refsData) == 0 {
return
}
left, right = decodeRefs(string(refsData))
if left == prePushDeleteBranch {
return
}
// Just use scanner here
scanOpt := &lfs.ScanRefsOptions{ScanMode: lfs.ScanLeftToRemoteMode, RemoteName: lfs.Config.CurrentRemote}
pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
// Objects to skip because they're missing locally but on server
var skipObjects map[string]struct{}
if !prePushDryRun {
// Do this as a pre-flight check since upload queue starts immediately
skipObjects = prePushCheckForMissingObjects(pointers)
}
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, prePushDryRun)
for _, pointer := range pointers {
if prePushDryRun {
Print("push %s [%s]", pointer.Name, pointer.Oid)
continue
}
if _, skip := skipObjects[pointer.Oid]; skip {
// object missing locally but on server, don't bother
continue
}
u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
if err != nil {
if lfs.IsCleanPointerError(err) {
Exit(prePushMissingErrMsg, pointer.Name, lfs.ErrorGetContext(err, "pointer").(*lfs.Pointer).Oid)
} else if Debugging || lfs.IsFatalError(err) {
Panic(err, err.Error())
} else {
Exit(err.Error())
}
}
uploadQueue.Add(u)
}
if !prePushDryRun {
uploadQueue.Wait()
for _, err := range uploadQueue.Errors() {
if Debugging || lfs.IsFatalError(err) {
LoggedError(err, err.Error())
} else {
Error(err.Error())
}
}
if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
}
}
func fetchCommand(cmd *cobra.Command, args []string) {
var ref string
var err error
if len(args) == 1 {
ref = args[0]
} else {
ref, err = git.CurrentRef()
if err != nil {
Panic(err, "Could not fetch")
}
}
pointers, err := lfs.ScanRefs(ref, "")
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
q := lfs.NewDownloadQueue(lfs.Config.ConcurrentTransfers(), len(pointers))
for _, p := range pointers {
q.Add(lfs.NewDownloadable(p))
}
target, err := git.ResolveRef(ref)
if err != nil {
Panic(err, "Could not resolve git ref")
}
current, err := git.CurrentRef()
if err != nil {
Panic(err, "Could not fetch the current git ref")
}
if target == current {
// We just downloaded the files for the current ref, we can copy them into
// the working directory and update the git index. We're doing this in a
// goroutine so they can be copied as they come in, for efficiency.
watch := q.Watch()
go func() {
files := make(map[string]*lfs.WrappedPointer, len(pointers))
for _, pointer := range pointers {
files[pointer.Oid] = pointer
}
// Fire up the update-index command
cmd := exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
stdin, err := cmd.StdinPipe()
if err != nil {
Panic(err, "Could not update the index")
}
if err := cmd.Start(); err != nil {
Panic(err, "Could not update the index")
}
// As files come in, write them to the wd and update the index
for oid := range watch {
pointer, ok := files[oid]
if !ok {
continue
}
file, err := os.Create(pointer.Name)
if err != nil {
Panic(err, "Could not create working directory file")
}
if err := lfs.PointerSmudge(file, pointer.Pointer, pointer.Name, nil); err != nil {
Panic(err, "Could not write working directory file")
}
file.Close()
stdin.Write([]byte(pointer.Name + "\n"))
}
stdin.Close()
if err := cmd.Wait(); err != nil {
Panic(err, "Error updating the git index")
}
}()
processQueue := time.Now()
q.Process()
tracerx.PerformanceSince("process queue", processQueue)
}
}