func prePushCheckForMissingObjects(pointers []*lfs.WrappedPointer) (objectsOnServer map[string]struct{}) { var missingLocalObjects []*lfs.WrappedPointer var missingSize int64 var skipObjects = make(map[string]struct{}, len(pointers)) for _, pointer := range pointers { if !lfs.ObjectExistsOfSize(pointer.Oid, pointer.Size) { // We think we need to push this but we don't have it // Store for server checking later missingLocalObjects = append(missingLocalObjects, pointer) missingSize += pointer.Size } } if len(missingLocalObjects) == 0 { return nil } checkQueue := lfs.NewDownloadCheckQueue(len(missingLocalObjects), missingSize, false) for _, p := range missingLocalObjects { checkQueue.Add(lfs.NewDownloadCheckable(p)) } // this channel is filled with oids for which Check() succeeded & Transfer() was called transferc := checkQueue.Watch() go func() { for oid := range transferc { skipObjects[oid] = struct{}{} } }() checkQueue.Wait() return skipObjects }
func prePushCheckForMissingObjects(pointers []*lfs.WrappedPointer) (objectsOnServer lfs.StringSet) { var missingLocalObjects []*lfs.WrappedPointer var missingSize int64 var skipObjects = lfs.NewStringSetWithCapacity(len(pointers)) for _, pointer := range pointers { if !lfs.ObjectExistsOfSize(pointer.Oid, pointer.Size) { // We think we need to push this but we don't have it // Store for server checking later missingLocalObjects = append(missingLocalObjects, pointer) missingSize += pointer.Size } } if len(missingLocalObjects) == 0 { return nil } checkQueue := lfs.NewDownloadCheckQueue(len(missingLocalObjects), missingSize, true) for _, p := range missingLocalObjects { checkQueue.Add(lfs.NewDownloadCheckable(p)) } // this channel is filled with oids for which Check() succeeded & Transfer() was called transferc := checkQueue.Watch() done := make(chan int) go func() { for oid := range transferc { skipObjects.Add(oid) } done <- 1 }() // Currently this is needed to flush the batch but is not enough to sync transferc completely checkQueue.Wait() <-done return skipObjects }
func readyAndMissingPointers(allpointers []*lfs.WrappedPointer, include, exclude []string) ([]*lfs.WrappedPointer, []*lfs.WrappedPointer, int64) { size := int64(0) seen := make(map[string]bool, len(allpointers)) missing := make([]*lfs.WrappedPointer, 0, len(allpointers)) ready := make([]*lfs.WrappedPointer, 0, len(allpointers)) for _, p := range allpointers { // Filtered out by --include or --exclude if !lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude) { continue } // no need to download the same object multiple times if seen[p.Oid] { continue } seen[p.Oid] = true // no need to download objects that exist locally already lfs.LinkOrCopyFromReference(p.Oid, p.Size) if lfs.ObjectExistsOfSize(p.Oid, p.Size) { ready = append(ready, p) continue } missing = append(missing, p) size += p.Size } return ready, missing, size }
// Fetch and report completion of each OID to a channel (optional, pass nil to skip) func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) { totalSize := int64(0) for _, p := range pointers { totalSize += p.Size } q := lfs.NewDownloadQueue(len(pointers), totalSize, false) for _, p := range pointers { // Only add to download queue if local file is not the right size already // This avoids previous case of over-reporting a requirement for files we already have // which would only be skipped by PointerSmudgeObject later passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude) if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter { q.Add(lfs.NewDownloadable(p)) } else { // If we already have it, or it won't be fetched // report it to chan immediately to support pull/checkout if out != nil { out <- p } } } if out != nil { dlwatch := q.Watch() go func() { // fetch only reports single OID, but OID *might* be referenced by multiple // WrappedPointers if same content is at multiple paths, so map oid->slice oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers)) for _, pointer := range pointers { plist := oidToPointers[pointer.Oid] oidToPointers[pointer.Oid] = append(plist, pointer) } for oid := range dlwatch { plist, ok := oidToPointers[oid] if !ok { continue } for _, p := range plist { out <- p } } close(out) }() } processQueue := time.Now() q.Wait() tracerx.PerformanceSince("process queue", processQueue) }
func (c *uploadContext) prepareUpload(unfiltered []*lfs.WrappedPointer) (*lfs.TransferQueue, []*lfs.WrappedPointer) { numUnfiltered := len(unfiltered) uploadables := make([]*lfs.WrappedPointer, 0, numUnfiltered) missingLocalObjects := make([]*lfs.WrappedPointer, 0, numUnfiltered) numObjects := 0 totalSize := int64(0) missingSize := int64(0) // separate out objects that _should_ be uploaded, but don't exist in // .git/lfs/objects. Those will skipped if the server already has them. for _, p := range unfiltered { // object already uploaded in this process, skip! if c.HasUploaded(p.Oid) { continue } numObjects += 1 totalSize += p.Size if lfs.ObjectExistsOfSize(p.Oid, p.Size) { uploadables = append(uploadables, p) } else { // We think we need to push this but we don't have it // Store for server checking later missingLocalObjects = append(missingLocalObjects, p) missingSize += p.Size } } // check to see if the server has the missing objects. c.checkMissing(missingLocalObjects, missingSize) // build the TransferQueue, automatically skipping any missing objects that // the server already has. uploadQueue := lfs.NewUploadQueue(numObjects, totalSize, c.DryRun) for _, p := range missingLocalObjects { if c.HasUploaded(p.Oid) { uploadQueue.Skip(p.Size) } else { uploadables = append(uploadables, p) } } return uploadQueue, uploadables }