func (a *archiver) stage2HashLoop() { defer close(a.stage3LookupChan) pool := common.NewGoroutinePriorityPool(a.maxConcurrentContains, a.canceler) defer func() { _ = pool.Wait() }() for file := range a.stage2HashChan { // This loop will implicitly buffer when stage1 is too fast by creating a // lot of hung goroutines in pool. This permits reducing the contention on // a.closeLock. // TODO(tandrii): Implement backpressure in GoroutinePool, e.g. when it // exceeds 20k or something similar. item := file pool.Schedule(item.priority, func() { // calcDigest calls setErr() and update wgHashed even on failure. end := tracer.Span(a, "hash", tracer.Args{"name": item.DisplayName()}) if err := item.calcDigest(); err != nil { end(tracer.Args{"err": err}) a.Cancel(err) item.Close() return } end(tracer.Args{"size": float64(item.digestItem.Size)}) tracer.CounterAdd(a, "bytesHashed", float64(item.digestItem.Size)) a.progress.Update(groupHash, groupHashDone, 1) a.progress.Update(groupHash, groupHashDoneSize, item.digestItem.Size) a.progress.Update(groupLookup, groupLookupTodo, 1) a.stage3LookupChan <- item }, func() { item.setErr(a.CancelationReason()) item.wgHashed.Done() item.Close() }) } }
func (a *archiver) stage4UploadLoop() { pool := common.NewGoroutinePriorityPool(a.maxConcurrentUpload, a.canceler) defer func() { _ = pool.Wait() }() for state := range a.stage4UploadChan { item := state pool.Schedule(item.priority, func() { a.doUpload(item) }, nil) } }