func prePushCheckForMissingObjects(pointers []*lfs.WrappedPointer) (objectsOnServer map[string]struct{}) {
var missingLocalObjects []*lfs.WrappedPointer
var missingSize int64
var skipObjects = make(map[string]struct{}, len(pointers))
for _, pointer := range pointers {
if !lfs.ObjectExistsOfSize(pointer.Oid, pointer.Size) {
// We think we need to push this but we don't have it
// Store for server checking later
missingLocalObjects = append(missingLocalObjects, pointer)
missingSize += pointer.Size
}
}
if len(missingLocalObjects) == 0 {
return nil
}
checkQueue := lfs.NewDownloadCheckQueue(len(missingLocalObjects), missingSize, false)
for _, p := range missingLocalObjects {
checkQueue.Add(lfs.NewDownloadCheckable(p))
}
// this channel is filled with oids for which Check() succeeded & Transfer() was called
transferc := checkQueue.Watch()
go func() {
for oid := range transferc {
skipObjects[oid] = struct{}{}
}
}()
checkQueue.Wait()
return skipObjects
}
func prePushCheckForMissingObjects(pointers []*lfs.WrappedPointer) (objectsOnServer lfs.StringSet) {
var missingLocalObjects []*lfs.WrappedPointer
var missingSize int64
var skipObjects = lfs.NewStringSetWithCapacity(len(pointers))
for _, pointer := range pointers {
if !lfs.ObjectExistsOfSize(pointer.Oid, pointer.Size) {
// We think we need to push this but we don't have it
// Store for server checking later
missingLocalObjects = append(missingLocalObjects, pointer)
missingSize += pointer.Size
}
}
if len(missingLocalObjects) == 0 {
return nil
}
checkQueue := lfs.NewDownloadCheckQueue(len(missingLocalObjects), missingSize, true)
for _, p := range missingLocalObjects {
checkQueue.Add(lfs.NewDownloadCheckable(p))
}
// this channel is filled with oids for which Check() succeeded & Transfer() was called
transferc := checkQueue.Watch()
done := make(chan int)
go func() {
for oid := range transferc {
skipObjects.Add(oid)
}
done <- 1
}()
// Currently this is needed to flush the batch but is not enough to sync transferc completely
checkQueue.Wait()
<-done
return skipObjects
}
func readyAndMissingPointers(allpointers []*lfs.WrappedPointer, include, exclude []string) ([]*lfs.WrappedPointer, []*lfs.WrappedPointer, int64) {
size := int64(0)
seen := make(map[string]bool, len(allpointers))
missing := make([]*lfs.WrappedPointer, 0, len(allpointers))
ready := make([]*lfs.WrappedPointer, 0, len(allpointers))
for _, p := range allpointers {
// Filtered out by --include or --exclude
if !lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude) {
continue
}
// no need to download the same object multiple times
if seen[p.Oid] {
continue
}
seen[p.Oid] = true
// no need to download objects that exist locally already
lfs.LinkOrCopyFromReference(p.Oid, p.Size)
if lfs.ObjectExistsOfSize(p.Oid, p.Size) {
ready = append(ready, p)
continue
}
missing = append(missing, p)
size += p.Size
}
return ready, missing, size
}
// Fetch and report completion of each OID to a channel (optional, pass nil to skip)
func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) {
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
q := lfs.NewDownloadQueue(len(pointers), totalSize, false)
for _, p := range pointers {
// Only add to download queue if local file is not the right size already
// This avoids previous case of over-reporting a requirement for files we already have
// which would only be skipped by PointerSmudgeObject later
passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude)
if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter {
q.Add(lfs.NewDownloadable(p))
} else {
// If we already have it, or it won't be fetched
// report it to chan immediately to support pull/checkout
if out != nil {
out <- p
}
}
}
if out != nil {
dlwatch := q.Watch()
go func() {
// fetch only reports single OID, but OID *might* be referenced by multiple
// WrappedPointers if same content is at multiple paths, so map oid->slice
oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers))
for _, pointer := range pointers {
plist := oidToPointers[pointer.Oid]
oidToPointers[pointer.Oid] = append(plist, pointer)
}
for oid := range dlwatch {
plist, ok := oidToPointers[oid]
if !ok {
continue
}
for _, p := range plist {
out <- p
}
}
close(out)
}()
}
processQueue := time.Now()
q.Wait()
tracerx.PerformanceSince("process queue", processQueue)
}
func (c *uploadContext) prepareUpload(unfiltered []*lfs.WrappedPointer) (*lfs.TransferQueue, []*lfs.WrappedPointer) {
numUnfiltered := len(unfiltered)
uploadables := make([]*lfs.WrappedPointer, 0, numUnfiltered)
missingLocalObjects := make([]*lfs.WrappedPointer, 0, numUnfiltered)
numObjects := 0
totalSize := int64(0)
missingSize := int64(0)
// separate out objects that _should_ be uploaded, but don't exist in
// .git/lfs/objects. Those will skipped if the server already has them.
for _, p := range unfiltered {
// object already uploaded in this process, skip!
if c.HasUploaded(p.Oid) {
continue
}
numObjects += 1
totalSize += p.Size
if lfs.ObjectExistsOfSize(p.Oid, p.Size) {
uploadables = append(uploadables, p)
} else {
// We think we need to push this but we don't have it
// Store for server checking later
missingLocalObjects = append(missingLocalObjects, p)
missingSize += p.Size
}
}
// check to see if the server has the missing objects.
c.checkMissing(missingLocalObjects, missingSize)
// build the TransferQueue, automatically skipping any missing objects that
// the server already has.
uploadQueue := lfs.NewUploadQueue(numObjects, totalSize, c.DryRun)
for _, p := range missingLocalObjects {
if c.HasUploaded(p.Oid) {
uploadQueue.Skip(p.Size)
} else {
uploadables = append(uploadables, p)
}
}
return uploadQueue, uploadables
}