Exemple #1
0
// Process starts the transfer queue and displays a progress bar. Process will
// do individual or batch transfers depending on the Config.BatchTransfer() value.
// Process will transfer files sequentially or concurrently depending on the
// Concig.ConcurrentTransfers() value.
func (q *TransferQueue) Process() {
	q.bar = pb.New64(q.size)
	q.bar.SetUnits(pb.U_BYTES)
	q.bar.ShowBar = false

	// This goroutine collects errors returned from transfers
	go func() {
		for err := range q.errorc {
			q.errors = append(q.errors, err)
		}
	}()

	// This goroutine watches for apiEvents. In order to prevent multiple
	// credential requests from happening, the queue is processed sequentially
	// until an API request succeeds (meaning authenication has happened successfully).
	// Once the an API request succeeds, all worker goroutines are woken up and allowed
	// to process transfers. Once a success happens, this goroutine exits.
	go func() {
		for {
			event := <-apiEvent
			switch event {
			case apiEventSuccess:
				atomic.StoreInt32(&q.clientAuthorized, 1)
				q.authCond.Broadcast() // Wake all remaining goroutines
				return
			case apiEventFail:
				q.authCond.Signal() // Wake the next goroutine
			}
		}
	}()

	for i := 0; i < q.workers; i++ {
		// These are the worker goroutines that process transfers
		go func(n int) {

			for transfer := range q.transferc {
				cb := func(total, read int64, current int) error {
					q.bar.Add(current)
					return nil
				}

				if err := transfer.Transfer(cb); err != nil {
					q.errorc <- err
				} else {
					oid := transfer.Oid()
					for _, c := range q.watchers {
						c <- oid
					}
				}

				f := atomic.AddInt64(&q.finished, 1)
				q.bar.Prefix(fmt.Sprintf("(%d of %d files) ", f, q.files))
				q.wg.Done()
			}
		}(i)
	}

	if Config.BatchTransfer() {
		q.processBatch()
	} else {
		q.processIndividual()
	}

	q.wg.Wait()
	close(q.errorc)
	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.bar.Finish()
}
Exemple #2
0
// Process starts the transfer queue and displays a progress bar. Process will
// do individual or batch transfers depending on the Config.BatchTransfer() value.
// Process will transfer files sequentially or concurrently depending on the
// Concig.ConcurrentTransfers() value.
func (q *TransferQueue) Process() {
	q.bar = pb.New64(q.size)
	q.bar.SetUnits(pb.U_BYTES)
	q.bar.ShowBar = false

	// This goroutine collects errors returned from transfers
	go func() {
		for err := range q.errorc {
			q.errors = append(q.errors, err)
		}
	}()

	// This goroutine watches for apiEvents. In order to prevent multiple
	// credential requests from happening, the queue is processed sequentially
	// until an API request succeeds (meaning authenication has happened successfully).
	// Once the an API request succeeds, all worker goroutines are woken up and allowed
	// to process transfers. Once a success happens, this goroutine exits.
	go func() {
		for {
			event := <-apiEvent
			switch event {
			case apiEventSuccess:
				atomic.StoreInt32(&q.clientAuthorized, 1)
				q.authCond.Broadcast() // Wake all remaining goroutines
				return
			case apiEventFail:
				q.authCond.Signal() // Wake the next goroutine
			}
		}
	}()

	// This goroutine will send progress output to GIT_LFS_PROGRESS if it has been set
	progressc := make(chan string, 100)
	go func() {
		output, err := newProgressLogger()
		if err != nil {
			q.errorc <- Error(err)
		}

		for l := range progressc {
			if err := output.Write([]byte(l)); err != nil {
				q.errorc <- Error(err)
				output.Shutdown()
			}
		}

		output.Close()
	}()

	var transferCount = int32(0)
	direction := "push"
	if q.transferKind == "download" {
		direction = "pull"
	}

	for i := 0; i < q.workers; i++ {
		// These are the worker goroutines that process transfers
		go func() {
			for transfer := range q.transferc {
				c := atomic.AddInt32(&transferCount, 1)
				cb := func(total, read int64, current int) error {
					progressc <- fmt.Sprintf("%s %d/%d %d/%d %s\n", direction, c, q.files, read, total, transfer.Name())
					q.bar.Add(current)
					return nil
				}

				if err := transfer.Transfer(cb); err != nil {
					q.errorc <- err
				} else {
					oid := transfer.Oid()
					for _, c := range q.watchers {
						c <- oid
					}
				}

				f := atomic.AddInt32(&q.finished, 1)
				q.bar.Prefix(fmt.Sprintf("(%d of %d files) ", f, q.files))
				q.wg.Done()
			}
		}()
	}

	if Config.BatchTransfer() {
		if err := q.processBatch(); err != nil {
			q.processIndividual()
		}
	} else {
		q.processIndividual()
	}

	q.wg.Wait()
	close(q.errorc)
	for _, watcher := range q.watchers {
		close(watcher)
	}
	close(progressc)

	q.bar.Finish()
}