// worker function, many of these run per adapter
func (a *adapterBase) worker(workerNum int, ctx interface{}) {
tracerx.Printf("xfer: adapter %q worker %d starting", a.Name(), workerNum)
waitForAuth := workerNum > 0
signalAuthOnResponse := workerNum == 0
// First worker is the only one allowed to start immediately
// The rest wait until successful response from 1st worker to
// make sure only 1 login prompt is presented if necessary
// Deliberately outside jobChan processing so we know worker 0 will process 1st item
if waitForAuth {
tracerx.Printf("xfer: adapter %q worker %d waiting for Auth", a.Name(), workerNum)
a.authWait.Wait()
tracerx.Printf("xfer: adapter %q worker %d auth signal received", a.Name(), workerNum)
}
for t := range a.jobChan {
var authCallback func()
if signalAuthOnResponse {
authCallback = func() {
a.authWait.Done()
signalAuthOnResponse = false
}
}
tracerx.Printf("xfer: adapter %q worker %d processing job for %q", a.Name(), workerNum, t.Object.Oid)
// Actual transfer happens here
var err error
if t.Object.IsExpired(time.Now().Add(objectExpirationGracePeriod)) {
tracerx.Printf("xfer: adapter %q worker %d found job for %q expired, retrying...", a.Name(), workerNum, t.Object.Oid)
err = errors.NewRetriableError(errors.Errorf("lfs/transfer: object %q has expired", t.Object.Oid))
} else if t.Object.Size < 0 {
tracerx.Printf("xfer: adapter %q worker %d found invalid size for %q (got: %d), retrying...", a.Name(), workerNum, t.Object.Oid, t.Object.Size)
err = fmt.Errorf("Git LFS: object %q has invalid size (got: %d)", t.Object.Oid, t.Object.Size)
} else {
err = a.transferImpl.DoTransfer(ctx, t, a.cb, authCallback)
}
if a.outChan != nil {
res := TransferResult{t, err}
a.outChan <- res
}
tracerx.Printf("xfer: adapter %q worker %d finished job for %q", a.Name(), workerNum, t.Object.Oid)
}
// This will only happen if no jobs were submitted; just wake up all workers to finish
if signalAuthOnResponse {
a.authWait.Done()
}
tracerx.Printf("xfer: adapter %q worker %d stopping", a.Name(), workerNum)
a.transferImpl.WorkerEnding(workerNum, ctx)
a.workerWait.Done()
}
func defaultError(res *http.Response) error {
var msgFmt string
if f, ok := defaultErrors[res.StatusCode]; ok {
msgFmt = f
} else if res.StatusCode < 500 {
msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
} else {
msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode)
}
return errors.Errorf(msgFmt, res.Request.URL)
}
// Check the response from a HTTP request for problems
func handleResponse(cfg *config.Configuration, res *http.Response, creds auth.Creds) error {
auth.SaveCredentials(cfg, creds, res)
if res.StatusCode < 400 {
return nil
}
defer func() {
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
}()
cliErr := &ClientError{}
err := DecodeResponse(res, cliErr)
if err == nil {
if len(cliErr.Message) == 0 {
err = defaultError(res)
} else {
err = errors.Wrap(cliErr, "http")
}
}
if res.StatusCode == 401 {
if err == nil {
err = errors.New("api: received status 401")
}
return errors.NewAuthError(err)
}
if res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 507 && res.StatusCode != 509 {
if err == nil {
err = errors.Errorf("api: received status %d", res.StatusCode)
}
return errors.NewFatalError(err)
}
return err
}
// Batch calls the batch API and returns object results
func Batch(cfg *config.Configuration, objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
if len(objects) == 0 {
return nil, "", nil
}
// Compatibility; omit transfers list when only basic
// older schemas included `additionalproperties=false`
if len(transferAdapters) == 1 && transferAdapters[0] == "basic" {
transferAdapters = nil
}
o := &batchRequest{Operation: operation, Objects: objects, TransferAdapterNames: transferAdapters}
by, err := json.Marshal(o)
if err != nil {
return nil, "", errors.Wrap(err, "batch request")
}
req, err := NewBatchRequest(cfg, operation)
if err != nil {
return nil, "", errors.Wrap(err, "batch request")
}
req.Header.Set("Content-Type", MediaType)
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
req.ContentLength = int64(len(by))
req.Body = tools.NewReadSeekCloserWrapper(bytes.NewReader(by))
tracerx.Printf("api: batch %d files", len(objects))
res, bresp, err := DoBatchRequest(cfg, req)
if err != nil {
if res == nil {
return nil, "", errors.NewRetriableError(err)
}
if res.StatusCode == 0 {
return nil, "", errors.NewRetriableError(err)
}
if errors.IsAuthError(err) {
httputil.SetAuthType(cfg, req, res)
return Batch(cfg, objects, operation, transferAdapters)
}
switch res.StatusCode {
case 404, 410:
return nil, "", errors.NewNotImplementedError(errors.Errorf("api: batch not implemented: %d", res.StatusCode))
}
tracerx.Printf("api error: %s", err)
return nil, "", errors.Wrap(err, "batch response")
}
httputil.LogTransfer(cfg, "lfs.batch", res)
if res.StatusCode != 200 {
return nil, "", errors.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
}
return bresp.Objects, bresp.TransferAdapterName, nil
}