func callBatchApi(op string, objs []TestObject) ([]*api.ObjectResource, error) {
apiobjs := make([]*api.ObjectResource, 0, len(objs))
for _, o := range objs {
apiobjs = append(apiobjs, &api.ObjectResource{Oid: o.Oid, Size: o.Size})
}
o, _, err := api.Batch(config.Config, apiobjs, op, []string{"basic"})
if err != nil {
return nil, err
}
return o, nil
}
// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
var startProgress sync.Once
transferAdapterNames := q.manifest.GetAdapterNames(q.direction)
for {
batch := q.batcher.Next()
if batch == nil {
break
}
tracerx.Printf("tq: sending batch of size %d", len(batch))
transfers := make([]*api.ObjectResource, 0, len(batch))
for _, i := range batch {
t := i.(Transferable)
transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()})
}
if len(transfers) == 0 {
continue
}
objs, adapterName, err := api.Batch(config.Config, transfers, q.transferKind(), transferAdapterNames)
if err != nil {
if errors.IsNotImplementedError(err) {
git.Config.SetLocal("", "lfs.batch", "false")
go q.legacyFallback(batch)
return
}
var errOnce sync.Once
for _, o := range batch {
t := o.(Transferable)
if q.canRetryObject(t.Oid(), err) {
q.retry(t)
} else {
q.wait.Done()
errOnce.Do(func() { q.errorc <- err })
}
}
continue
}
q.useAdapter(adapterName)
startProgress.Do(q.meter.Start)
for _, o := range objs {
if o.Error != nil {
q.errorc <- errors.Wrapf(o.Error, "[%v] %v", o.Oid, o.Error.Message)
q.Skip(o.Size)
q.wait.Done()
continue
}
if _, ok := o.Rel(q.transferKind()); ok {
// This object needs to be transferred
q.trMutex.Lock()
transfer, ok := q.transferables[o.Oid]
q.trMutex.Unlock()
if ok {
transfer.SetObject(o)
q.meter.Add(transfer.Name())
q.addToAdapter(transfer)
} else {
q.Skip(transfer.Size())
q.wait.Done()
}
} else {
q.Skip(o.Size)
q.wait.Done()
}
}
}
}
