func (a *archiver) stage2HashLoop() {
defer close(a.stage3LookupChan)
pool := common.NewGoroutinePriorityPool(a.maxConcurrentContains, a.canceler)
defer func() {
_ = pool.Wait()
}()
for file := range a.stage2HashChan {
// This loop will implicitly buffer when stage1 is too fast by creating a
// lot of hung goroutines in pool. This permits reducing the contention on
// a.closeLock.
// TODO(tandrii): Implement backpressure in GoroutinePool, e.g. when it
// exceeds 20k or something similar.
item := file
pool.Schedule(item.priority, func() {
// calcDigest calls setErr() and update wgHashed even on failure.
end := tracer.Span(a, "hash", tracer.Args{"name": item.DisplayName()})
if err := item.calcDigest(); err != nil {
end(tracer.Args{"err": err})
a.Cancel(err)
item.Close()
return
}
end(tracer.Args{"size": float64(item.digestItem.Size)})
tracer.CounterAdd(a, "bytesHashed", float64(item.digestItem.Size))
a.progress.Update(groupHash, groupHashDone, 1)
a.progress.Update(groupHash, groupHashDoneSize, item.digestItem.Size)
a.progress.Update(groupLookup, groupLookupTodo, 1)
a.stage3LookupChan <- item
}, func() {
item.setErr(a.CancelationReason())
item.wgHashed.Done()
item.Close()
})
}
}
func (a *archiver) stage4UploadLoop() {
pool := common.NewGoroutinePriorityPool(a.maxConcurrentUpload, a.canceler)
defer func() {
_ = pool.Wait()
}()
for state := range a.stage4UploadChan {
item := state
pool.Schedule(item.priority, func() {
a.doUpload(item)
}, nil)
}
}