// BatchOrLegacy calls the Batch API and falls back on the Legacy API
// This is for simplicity, legacy route is not most optimal (serial)
// TODO LEGACY API: remove when legacy API removed
func BatchOrLegacy(cfg *config.Configuration, objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
if !cfg.BatchTransfer() {
objs, err := Legacy(cfg, objects, operation)
return objs, "", err
}
objs, adapterName, err := Batch(cfg, objects, operation, transferAdapters)
if err != nil {
if errors.IsNotImplementedError(err) {
git.Config.SetLocal("", "lfs.batch", "false")
objs, err := Legacy(cfg, objects, operation)
return objs, "", err
}
return nil, "", err
}
return objs, adapterName, nil
}
// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
var startProgress sync.Once
transferAdapterNames := q.manifest.GetAdapterNames(q.direction)
for {
batch := q.batcher.Next()
if batch == nil {
break
}
tracerx.Printf("tq: sending batch of size %d", len(batch))
transfers := make([]*api.ObjectResource, 0, len(batch))
for _, i := range batch {
t := i.(Transferable)
transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()})
}
if len(transfers) == 0 {
continue
}
objs, adapterName, err := api.Batch(config.Config, transfers, q.transferKind(), transferAdapterNames)
if err != nil {
if errors.IsNotImplementedError(err) {
git.Config.SetLocal("", "lfs.batch", "false")
go q.legacyFallback(batch)
return
}
var errOnce sync.Once
for _, o := range batch {
t := o.(Transferable)
if q.canRetryObject(t.Oid(), err) {
q.retry(t)
} else {
q.wait.Done()
errOnce.Do(func() { q.errorc <- err })
}
}
continue
}
q.useAdapter(adapterName)
startProgress.Do(q.meter.Start)
for _, o := range objs {
if o.Error != nil {
q.errorc <- errors.Wrapf(o.Error, "[%v] %v", o.Oid, o.Error.Message)
q.Skip(o.Size)
q.wait.Done()
continue
}
if _, ok := o.Rel(q.transferKind()); ok {
// This object needs to be transferred
q.trMutex.Lock()
transfer, ok := q.transferables[o.Oid]
q.trMutex.Unlock()
if ok {
transfer.SetObject(o)
q.meter.Add(transfer.Name())
q.addToAdapter(transfer)
} else {
q.Skip(transfer.Size())
q.wait.Done()
}
} else {
q.Skip(o.Size)
q.wait.Done()
}
}
}
}