func uploadsWithObjectIDs(oids []string) *lfs.TransferQueue {
uploads := []*lfs.Uploadable{}
totalSize := int64(0)
for i, oid := range oids {
if pushDryRun {
Print("push object ID %s", oid)
continue
}
tracerx.Printf("prepare upload: %s %d/%d", oid, i+1, len(oids))
u, err := lfs.NewUploadable(oid, "")
if err != nil {
if Debugging || lfs.IsFatalError(err) {
Panic(err, err.Error())
} else {
Exit(err.Error())
}
}
uploads = append(uploads, u)
}
uploadQueue := lfs.NewUploadQueue(len(oids), totalSize, pushDryRun)
for _, u := range uploads {
uploadQueue.Add(u)
}
return uploadQueue
}
func uploadPointers(pointers []*lfs.WrappedPointer) *lfs.TransferQueue {
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
skipObjects := prePushCheckForMissingObjects(pointers)
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, pushDryRun)
for i, pointer := range pointers {
if pushDryRun {
Print("push %s => %s", pointer.Oid, pointer.Name)
continue
}
if _, skip := skipObjects[pointer.Oid]; skip {
// object missing locally but on server, don't bother
continue
}
tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
if err != nil {
if Debugging || lfs.IsFatalError(err) {
Panic(err, err.Error())
} else {
Exit(err.Error())
}
}
uploadQueue.Add(u)
}
return uploadQueue
}
func uploadPointers(pointers []*lfs.WrappedPointer) *lfs.TransferQueue {
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, pushDryRun)
for i, pointer := range pointers {
if pushDryRun {
Print("push %s [%s]", pointer.Name, pointer.Oid)
continue
}
tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name)
if wErr != nil {
if Debugging || wErr.Panic {
Panic(wErr.Err, wErr.Error())
} else {
Exit(wErr.Error())
}
}
uploadQueue.Add(u)
}
return uploadQueue
}
func uploadsBetweenRefs(left string, right string) *lfs.TransferQueue {
// Just use scanner here
pointers, err := lfs.ScanRefs(left, right, nil)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, pushDryRun)
for i, pointer := range pointers {
if pushDryRun {
Print("push %s", pointer.Name)
continue
}
tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name)
if wErr != nil {
if Debugging || wErr.Panic {
Panic(wErr.Err, wErr.Error())
} else {
Exit(wErr.Error())
}
}
uploadQueue.Add(u)
}
return uploadQueue
}
func buildTestData() (oidsExist, oidsMissing []TestObject, err error) {
const oidCount = 50
oidsExist = make([]TestObject, 0, oidCount)
oidsMissing = make([]TestObject, 0, oidCount)
// Build test data for existing files & upload
// Use test repo for this to simplify the process of making sure data matches oid
// We're not performing a real test at this point (although an upload fail will break it)
var callback testDataCallback
repo := test.NewRepo(&callback)
repo.Pushd()
defer repo.Cleanup()
// just one commit
commit := test.CommitInput{CommitterName: "A N Other", CommitterEmail: "*****@*****.**"}
var totalSize int64
for i := 0; i < oidCount; i++ {
filename := fmt.Sprintf("file%d.dat", i)
sz := int64(rand.Intn(200)) + 50
commit.Files = append(commit.Files, &test.FileInput{Filename: filename, Size: sz})
totalSize += sz
}
outputs := repo.AddCommits([]*test.CommitInput{&commit})
// now upload
uploadQueue := lfs.NewUploadQueue(len(oidsExist), totalSize, false)
for _, f := range outputs[0].Files {
oidsExist = append(oidsExist, TestObject{Oid: f.Oid, Size: f.Size})
u, err := lfs.NewUploadable(f.Oid, "Test file")
if err != nil {
return nil, nil, err
}
uploadQueue.Add(u)
}
uploadQueue.Wait()
for _, err := range uploadQueue.Errors() {
if errors.IsFatalError(err) {
exit("Fatal error setting up test data: %s", err)
}
}
// Generate SHAs for missing files, random but repeatable
// No actual file content needed for these
rand.Seed(int64(oidCount))
runningSha := sha256.New()
for i := 0; i < oidCount; i++ {
runningSha.Write([]byte{byte(rand.Intn(256))})
oid := hex.EncodeToString(runningSha.Sum(nil))
sz := int64(rand.Intn(200)) + 50
oidsMissing = append(oidsMissing, TestObject{Oid: oid, Size: sz})
}
return oidsExist, oidsMissing, nil
}
func (c *uploadContext) prepareUpload(unfiltered []*lfs.WrappedPointer) (*lfs.TransferQueue, []*lfs.WrappedPointer) {
numUnfiltered := len(unfiltered)
uploadables := make([]*lfs.WrappedPointer, 0, numUnfiltered)
missingLocalObjects := make([]*lfs.WrappedPointer, 0, numUnfiltered)
numObjects := 0
totalSize := int64(0)
missingSize := int64(0)
// separate out objects that _should_ be uploaded, but don't exist in
// .git/lfs/objects. Those will skipped if the server already has them.
for _, p := range unfiltered {
// object already uploaded in this process, skip!
if c.HasUploaded(p.Oid) {
continue
}
numObjects += 1
totalSize += p.Size
if lfs.ObjectExistsOfSize(p.Oid, p.Size) {
uploadables = append(uploadables, p)
} else {
// We think we need to push this but we don't have it
// Store for server checking later
missingLocalObjects = append(missingLocalObjects, p)
missingSize += p.Size
}
}
// check to see if the server has the missing objects.
c.checkMissing(missingLocalObjects, missingSize)
// build the TransferQueue, automatically skipping any missing objects that
// the server already has.
uploadQueue := lfs.NewUploadQueue(numObjects, totalSize, c.DryRun)
for _, p := range missingLocalObjects {
if c.HasUploaded(p.Oid) {
uploadQueue.Skip(p.Size)
} else {
uploadables = append(uploadables, p)
}
}
return uploadQueue, uploadables
}
func uploadsWithObjectIDs(oids []string) *lfs.TransferQueue {
uploadQueue := lfs.NewUploadQueue(lfs.Config.ConcurrentTransfers(), len(oids))
for i, oid := range oids {
if pushDryRun {
Print("push object ID %s", oid)
continue
}
tracerx.Printf("prepare upload: %s %d/%d", oid, i+1, len(oids))
u, wErr := lfs.NewUploadable(oid, "")
if wErr != nil {
if Debugging || wErr.Panic {
Panic(wErr.Err, wErr.Error())
} else {
Exit(wErr.Error())
}
}
uploadQueue.Add(u)
}
return uploadQueue
}
// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
// two arguments on the command line:
//
// 1. Name of the remote to which the push is being done
// 2. URL to which the push is being done
//
// The hook receives commit information on stdin in the form:
// <local ref> <local sha1> <remote ref> <remote sha1>
//
// In the typical case, prePushCommand will get a list of git objects being
// pushed by using the following:
//
// git rev-list --objects <local sha1> ^<remote sha1>
//
// If any of those git objects are associated with Git LFS objects, those
// objects will be pushed to the Git LFS API.
//
// In the case of pushing a new branch, the list of git objects will be all of
// the git objects in this branch.
//
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {
var left, right string
if len(args) == 0 {
Print("This should be run through Git's pre-push hook. Run `git lfs update` to install it.")
os.Exit(1)
}
lfs.Config.CurrentRemote = args[0]
refsData, err := ioutil.ReadAll(os.Stdin)
if err != nil {
Panic(err, "Error reading refs on stdin")
}
if len(refsData) == 0 {
return
}
left, right = decodeRefs(string(refsData))
if left == prePushDeleteBranch {
return
}
// Just use scanner here
pointers, err := lfs.ScanRefs(left, right, nil)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
uploadQueue := lfs.NewUploadQueue(lfs.Config.ConcurrentTransfers(), len(pointers))
for _, pointer := range pointers {
if prePushDryRun {
Print("push %s", pointer.Name)
continue
}
u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name)
if wErr != nil {
if cleanPointerErr, ok := wErr.Err.(*lfs.CleanedPointerError); ok {
Exit("%s is an LFS pointer to %s, which does not exist in .git/lfs/objects.\n\nRun 'git lfs fsck' to verify Git LFS objects.",
pointer.Name, cleanPointerErr.Pointer.Oid)
} else if Debugging || wErr.Panic {
Panic(wErr.Err, wErr.Error())
} else {
Exit(wErr.Error())
}
}
uploadQueue.Add(u)
}
if !prePushDryRun {
uploadQueue.Process()
for _, err := range uploadQueue.Errors() {
if Debugging || err.Panic {
LoggedError(err.Err, err.Error())
} else {
Error(err.Error())
}
}
if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
}
}
// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
// two arguments on the command line:
//
// 1. Name of the remote to which the push is being done
// 2. URL to which the push is being done
//
// The hook receives commit information on stdin in the form:
// <local ref> <local sha1> <remote ref> <remote sha1>
//
// In the typical case, prePushCommand will get a list of git objects being
// pushed by using the following:
//
// git rev-list --objects <local sha1> ^<remote sha1>
//
// If any of those git objects are associated with Git LFS objects, those
// objects will be pushed to the Git LFS API.
//
// In the case of pushing a new branch, the list of git objects will be all of
// the git objects in this branch.
//
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {
var left, right string
if len(args) == 0 {
Print("This should be run through Git's pre-push hook. Run `git lfs update` to install it.")
os.Exit(1)
}
lfs.Config.CurrentRemote = args[0]
refsData, err := ioutil.ReadAll(os.Stdin)
if err != nil {
Panic(err, "Error reading refs on stdin")
}
if len(refsData) == 0 {
return
}
left, right = decodeRefs(string(refsData))
if left == prePushDeleteBranch {
return
}
// Just use scanner here
scanOpt := &lfs.ScanRefsOptions{ScanMode: lfs.ScanLeftToRemoteMode, RemoteName: lfs.Config.CurrentRemote}
pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
// Objects to skip because they're missing locally but on server
var skipObjects map[string]struct{}
if !prePushDryRun {
// Do this as a pre-flight check since upload queue starts immediately
skipObjects = prePushCheckForMissingObjects(pointers)
}
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, prePushDryRun)
for _, pointer := range pointers {
if prePushDryRun {
Print("push %s [%s]", pointer.Name, pointer.Oid)
continue
}
if _, skip := skipObjects[pointer.Oid]; skip {
// object missing locally but on server, don't bother
continue
}
u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
if err != nil {
if lfs.IsCleanPointerError(err) {
Exit(prePushMissingErrMsg, pointer.Name, lfs.ErrorGetContext(err, "pointer").(*lfs.Pointer).Oid)
} else if Debugging || lfs.IsFatalError(err) {
Panic(err, err.Error())
} else {
Exit(err.Error())
}
}
uploadQueue.Add(u)
}
if !prePushDryRun {
uploadQueue.Wait()
for _, err := range uploadQueue.Errors() {
if Debugging || lfs.IsFatalError(err) {
LoggedError(err, err.Error())
} else {
Error(err.Error())
}
}
if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
}
}
func prePushRef(left, right string) {
// Just use scanner here
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = lfs.Config.CurrentRemote
pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
// Objects to skip because they're missing locally but on server
var skipObjects lfs.StringSet
if !prePushDryRun {
// Do this as a pre-flight check since upload queue starts immediately
skipObjects = prePushCheckForMissingObjects(pointers)
}
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, prePushDryRun)
for _, pointer := range pointers {
if prePushDryRun {
Print("push %s => %s", pointer.Oid, pointer.Name)
continue
}
if skipObjects.Contains(pointer.Oid) {
// object missing locally but on server, don't bother
continue
}
u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
if err != nil {
if lfs.IsCleanPointerError(err) {
Exit(prePushMissingErrMsg, pointer.Name, lfs.ErrorGetContext(err, "pointer").(*lfs.Pointer).Oid)
} else if Debugging || lfs.IsFatalError(err) {
Panic(err, err.Error())
} else {
Exit(err.Error())
}
}
uploadQueue.Add(u)
}
if !prePushDryRun {
uploadQueue.Wait()
for _, err := range uploadQueue.Errors() {
if Debugging || lfs.IsFatalError(err) {
LoggedError(err, err.Error())
} else {
Error(err.Error())
}
}
if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
}
}
// pushCommand pushes local objects to a Git LFS server. It takes two
// arguments:
//
// `<remote> <remote ref>`
//
// Both a remote name ("origin") or a remote URL are accepted.
//
// pushCommand calculates the git objects to send by looking comparing the range
// of commits between the local and remote git servers.
func pushCommand(cmd *cobra.Command, args []string) {
var left, right string
if len(args) == 0 {
Print("Specify a remote and a remote branch name (`git lfs push origin master`)")
os.Exit(1)
}
lfs.Config.CurrentRemote = args[0]
if useStdin {
requireStdin("Run this command from the Git pre-push hook, or leave the --stdin flag off.")
// called from a pre-push hook! Update the existing pre-push hook if it's
// one that git-lfs set.
lfs.InstallHooks(false)
refsData, err := ioutil.ReadAll(os.Stdin)
if err != nil {
Panic(err, "Error reading refs on stdin")
}
if len(refsData) == 0 {
return
}
left, right = decodeRefs(string(refsData))
if left == pushDeleteBranch {
return
}
} else {
var remoteArg, refArg string
if len(args) < 1 {
Print("Usage: git lfs push --dry-run <remote> [ref]")
return
}
remoteArg = args[0]
if len(args) == 2 {
refArg = args[1]
}
localRef, err := git.CurrentRef()
if err != nil {
Panic(err, "Error getting local ref")
}
left = localRef
remoteRef, err := git.LsRemote(remoteArg, refArg)
if err != nil {
Panic(err, "Error getting remote ref")
}
if remoteRef != "" {
right = "^" + strings.Split(remoteRef, "\t")[0]
}
}
// Just use scanner here
pointers, err := lfs.ScanRefs(left, right)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
uploadQueue := lfs.NewUploadQueue(lfs.Config.ConcurrentTransfers(), len(pointers))
for i, pointer := range pointers {
if pushDryRun {
Print("push %s", pointer.Name)
continue
}
tracerx.Printf("checking_asset: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name, i+1, len(pointers))
if wErr != nil {
if Debugging || wErr.Panic {
Panic(wErr.Err, wErr.Error())
} else {
Exit(wErr.Error())
}
}
uploadQueue.Add(u)
}
if !pushDryRun {
uploadQueue.Process()
for _, err := range uploadQueue.Errors() {
if Debugging || err.Panic {
LoggedError(err.Err, err.Error())
} else {
Error(err.Error())
}
}
if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
}
}