// Process starts the transfer queue and displays a progress bar. Process will // do individual or batch transfers depending on the Config.BatchTransfer() value. // Process will transfer files sequentially or concurrently depending on the // Concig.ConcurrentTransfers() value. func (q *TransferQueue) Process() { q.bar = pb.New64(q.size) q.bar.SetUnits(pb.U_BYTES) q.bar.ShowBar = false // This goroutine collects errors returned from transfers go func() { for err := range q.errorc { q.errors = append(q.errors, err) } }() // This goroutine watches for apiEvents. In order to prevent multiple // credential requests from happening, the queue is processed sequentially // until an API request succeeds (meaning authenication has happened successfully). // Once the an API request succeeds, all worker goroutines are woken up and allowed // to process transfers. Once a success happens, this goroutine exits. go func() { for { event := <-apiEvent switch event { case apiEventSuccess: atomic.StoreInt32(&q.clientAuthorized, 1) q.authCond.Broadcast() // Wake all remaining goroutines return case apiEventFail: q.authCond.Signal() // Wake the next goroutine } } }() for i := 0; i < q.workers; i++ { // These are the worker goroutines that process transfers go func(n int) { for transfer := range q.transferc { cb := func(total, read int64, current int) error { q.bar.Add(current) return nil } if err := transfer.Transfer(cb); err != nil { q.errorc <- err } else { oid := transfer.Oid() for _, c := range q.watchers { c <- oid } } f := atomic.AddInt64(&q.finished, 1) q.bar.Prefix(fmt.Sprintf("(%d of %d files) ", f, q.files)) q.wg.Done() } }(i) } if Config.BatchTransfer() { q.processBatch() } else { q.processIndividual() } q.wg.Wait() close(q.errorc) for _, watcher := range q.watchers { close(watcher) } q.bar.Finish() }
// Process starts the transfer queue and displays a progress bar. Process will // do individual or batch transfers depending on the Config.BatchTransfer() value. // Process will transfer files sequentially or concurrently depending on the // Concig.ConcurrentTransfers() value. func (q *TransferQueue) Process() { q.bar = pb.New64(q.size) q.bar.SetUnits(pb.U_BYTES) q.bar.ShowBar = false // This goroutine collects errors returned from transfers go func() { for err := range q.errorc { q.errors = append(q.errors, err) } }() // This goroutine watches for apiEvents. In order to prevent multiple // credential requests from happening, the queue is processed sequentially // until an API request succeeds (meaning authenication has happened successfully). // Once the an API request succeeds, all worker goroutines are woken up and allowed // to process transfers. Once a success happens, this goroutine exits. go func() { for { event := <-apiEvent switch event { case apiEventSuccess: atomic.StoreInt32(&q.clientAuthorized, 1) q.authCond.Broadcast() // Wake all remaining goroutines return case apiEventFail: q.authCond.Signal() // Wake the next goroutine } } }() // This goroutine will send progress output to GIT_LFS_PROGRESS if it has been set progressc := make(chan string, 100) go func() { output, err := newProgressLogger() if err != nil { q.errorc <- Error(err) } for l := range progressc { if err := output.Write([]byte(l)); err != nil { q.errorc <- Error(err) output.Shutdown() } } output.Close() }() var transferCount = int32(0) direction := "push" if q.transferKind == "download" { direction = "pull" } for i := 0; i < q.workers; i++ { // These are the worker goroutines that process transfers go func() { for transfer := range q.transferc { c := atomic.AddInt32(&transferCount, 1) cb := func(total, read int64, current int) error { progressc <- fmt.Sprintf("%s %d/%d %d/%d %s\n", direction, c, q.files, read, total, transfer.Name()) q.bar.Add(current) return nil } if err := transfer.Transfer(cb); err != nil { q.errorc <- err } else { oid := transfer.Oid() for _, c := range q.watchers { c <- oid } } f := atomic.AddInt32(&q.finished, 1) q.bar.Prefix(fmt.Sprintf("(%d of %d files) ", f, q.files)) q.wg.Done() } }() } if Config.BatchTransfer() { if err := q.processBatch(); err != nil { q.processIndividual() } } else { q.processIndividual() } q.wg.Wait() close(q.errorc) for _, watcher := range q.watchers { close(watcher) } close(progressc) q.bar.Finish() }