// Batch calls the batch API and returns object results
func Batch(objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
if len(objects) == 0 {
return nil, "", nil
}
o := &batchRequest{Operation: operation, Objects: objects, TransferAdapterNames: transferAdapters}
by, err := json.Marshal(o)
if err != nil {
return nil, "", errutil.Error(err)
}
req, err := NewBatchRequest(operation)
if err != nil {
return nil, "", errutil.Error(err)
}
req.Header.Set("Content-Type", MediaType)
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
req.ContentLength = int64(len(by))
req.Body = tools.NewReadSeekCloserWrapper(bytes.NewReader(by))
tracerx.Printf("api: batch %d files", len(objects))
res, bresp, err := DoBatchRequest(req)
if err != nil {
if res == nil {
return nil, "", errutil.NewRetriableError(err)
}
if res.StatusCode == 0 {
return nil, "", errutil.NewRetriableError(err)
}
if errutil.IsAuthError(err) {
httputil.SetAuthType(req, res)
return Batch(objects, operation, transferAdapters)
}
switch res.StatusCode {
case 404, 410:
tracerx.Printf("api: batch not implemented: %d", res.StatusCode)
return nil, "", errutil.NewNotImplementedError(nil)
}
tracerx.Printf("api error: %s", err)
return nil, "", errutil.Error(err)
}
httputil.LogTransfer("lfs.batch", res)
if res.StatusCode != 200 {
return nil, "", errutil.Error(fmt.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode))
}
return bresp.Objects, bresp.TransferAdapterName, nil
}
func performUpload(oid string, size int64, a *action, fromPath string, writer, errWriter *bufio.Writer) {
// We just use the URLs we're given, so we're just a proxy for the direct method
// but this is enough to test intermediate custom adapters
req, err := httputil.NewHttpRequest("PUT", a.Href, a.Header)
if err != nil {
sendTransferError(oid, 2, err.Error(), writer, errWriter)
return
}
if len(req.Header.Get("Content-Type")) == 0 {
req.Header.Set("Content-Type", "application/octet-stream")
}
if req.Header.Get("Transfer-Encoding") == "chunked" {
req.TransferEncoding = []string{"chunked"}
} else {
req.Header.Set("Content-Length", strconv.FormatInt(size, 10))
}
req.ContentLength = size
f, err := os.OpenFile(fromPath, os.O_RDONLY, 0644)
if err != nil {
sendTransferError(oid, 3, fmt.Sprintf("Cannot read data from %q: %v", fromPath, err), writer, errWriter)
return
}
defer f.Close()
// Turn callback into progress messages
cb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
sendProgress(oid, readSoFar, readSinceLast, writer, errWriter)
return nil
}
var reader io.Reader
reader = &progress.CallbackReader{
C: cb,
TotalSize: size,
Reader: f,
}
req.Body = ioutil.NopCloser(reader)
res, err := httputil.DoHttpRequest(cfg, req, true)
if err != nil {
sendTransferError(oid, res.StatusCode, fmt.Sprintf("Error uploading data for %s: %v", oid, err), writer, errWriter)
return
}
if res.StatusCode > 299 {
sendTransferError(oid, res.StatusCode, fmt.Sprintf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode), writer, errWriter)
return
}
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
// completed
complete := &transferResponse{"complete", oid, "", nil}
err = sendResponse(complete, writer, errWriter)
if err != nil {
writeToStderr(fmt.Sprintf("Unable to send completion message: %v\n", err), errWriter)
}
}
func (a *basicUploadAdapter) DoTransfer(t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
rel, ok := t.Object.Rel("upload")
if !ok {
return fmt.Errorf("No upload action for this object.")
}
req, err := httputil.NewHttpRequest("PUT", rel.Href, rel.Header)
if err != nil {
return err
}
if len(req.Header.Get("Content-Type")) == 0 {
req.Header.Set("Content-Type", "application/octet-stream")
}
if req.Header.Get("Transfer-Encoding") == "chunked" {
req.TransferEncoding = []string{"chunked"}
} else {
req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size, 10))
}
req.ContentLength = t.Object.Size
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
if err != nil {
return errutil.Error(err)
}
defer f.Close()
// Ensure progress callbacks made while uploading
// Wrap callback to give name context
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
if cb != nil {
return cb(t.Name, totalSize, readSoFar, readSinceLast)
}
return nil
}
var reader io.Reader
reader = &progress.CallbackReader{
C: ccb,
TotalSize: t.Object.Size,
Reader: f,
}
// Signal auth was ok on first read; this frees up other workers to start
if authOkFunc != nil {
reader = newStartCallbackReader(reader, func(*startCallbackReader) {
authOkFunc()
})
}
req.Body = ioutil.NopCloser(reader)
res, err := httputil.DoHttpRequest(req, true)
if err != nil {
return errutil.NewRetriableError(err)
}
httputil.LogTransfer("lfs.data.upload", res)
// A status code of 403 likely means that an authentication token for the
// upload has expired. This can be safely retried.
if res.StatusCode == 403 {
return errutil.NewRetriableError(err)
}
if res.StatusCode > 299 {
return errutil.Errorf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
}
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
return api.VerifyUpload(t.Object)
}
func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
rel, ok := t.Object.Rel("upload")
if !ok {
return fmt.Errorf("No upload action for this object.")
}
// Note not supporting the Creation extension since the batch API generates URLs
// Also not supporting Concatenation to support parallel uploads of chunks; forward only
// 1. Send HEAD request to determine upload start point
// Request must include Tus-Resumable header (version)
tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Object.Oid)
req, err := httputil.NewHttpRequest("HEAD", rel.Href, rel.Header)
if err != nil {
return err
}
req.Header.Set("Tus-Resumable", TusVersion)
res, err := httputil.DoHttpRequest(config.Config, req, false)
if err != nil {
return errors.NewRetriableError(err)
}
// Response will contain Upload-Offset if supported
offHdr := res.Header.Get("Upload-Offset")
if len(offHdr) == 0 {
return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href)
}
offset, err := strconv.ParseInt(offHdr, 10, 64)
if err != nil || offset < 0 {
return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href)
}
// Upload-Offset=size means already completed (skip)
// Batch API will probably already detect this, but handle just in case
if offset >= t.Object.Size {
tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Object.Oid)
advanceCallbackProgress(cb, t, t.Object.Size)
return nil
}
// Open file for uploading
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
if err != nil {
return errors.Wrap(err, "tus upload")
}
defer f.Close()
// Upload-Offset=0 means start from scratch, but still send PATCH
if offset == 0 {
tracerx.Printf("xfer: tus.io uploading %q from start", t.Object.Oid)
} else {
tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Object.Oid, offset)
advanceCallbackProgress(cb, t, offset)
_, err := f.Seek(offset, os.SEEK_CUR)
if err != nil {
return errors.Wrap(err, "tus upload")
}
}
// 2. Send PATCH request with byte start point (even if 0) in Upload-Offset
// Response status must be 204
// Response Upload-Offset must be request Upload-Offset plus sent bytes
// Response may include Upload-Expires header in which case check not passed
tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Object.Oid)
req, err = httputil.NewHttpRequest("PATCH", rel.Href, rel.Header)
if err != nil {
return err
}
req.Header.Set("Tus-Resumable", TusVersion)
req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10))
req.Header.Set("Content-Type", "application/offset+octet-stream")
req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size-offset, 10))
req.ContentLength = t.Object.Size - offset
// Ensure progress callbacks made while uploading
// Wrap callback to give name context
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
if cb != nil {
return cb(t.Name, totalSize, readSoFar, readSinceLast)
}
return nil
}
var reader io.Reader
reader = &progress.CallbackReader{
C: ccb,
TotalSize: t.Object.Size,
Reader: f,
}
// Signal auth was ok on first read; this frees up other workers to start
if authOkFunc != nil {
reader = newStartCallbackReader(reader, func(*startCallbackReader) {
authOkFunc()
})
}
req.Body = ioutil.NopCloser(reader)
res, err = httputil.DoHttpRequest(config.Config, req, false)
if err != nil {
return errors.NewRetriableError(err)
}
httputil.LogTransfer(config.Config, "lfs.data.upload", res)
// A status code of 403 likely means that an authentication token for the
// upload has expired. This can be safely retried.
if res.StatusCode == 403 {
err = errors.New("http: received status 403")
return errors.NewRetriableError(err)
}
if res.StatusCode > 299 {
return errors.Wrapf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
}
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
return api.VerifyUpload(config.Config, t.Object)
}
// Batch calls the batch API and returns object results
func Batch(cfg *config.Configuration, objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
if len(objects) == 0 {
return nil, "", nil
}
// Compatibility; omit transfers list when only basic
// older schemas included `additionalproperties=false`
if len(transferAdapters) == 1 && transferAdapters[0] == "basic" {
transferAdapters = nil
}
o := &batchRequest{Operation: operation, Objects: objects, TransferAdapterNames: transferAdapters}
by, err := json.Marshal(o)
if err != nil {
return nil, "", errors.Wrap(err, "batch request")
}
req, err := NewBatchRequest(cfg, operation)
if err != nil {
return nil, "", errors.Wrap(err, "batch request")
}
req.Header.Set("Content-Type", MediaType)
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
req.ContentLength = int64(len(by))
req.Body = tools.NewReadSeekCloserWrapper(bytes.NewReader(by))
tracerx.Printf("api: batch %d files", len(objects))
res, bresp, err := DoBatchRequest(cfg, req)
if err != nil {
if res == nil {
return nil, "", errors.NewRetriableError(err)
}
if res.StatusCode == 0 {
return nil, "", errors.NewRetriableError(err)
}
if errors.IsAuthError(err) {
httputil.SetAuthType(cfg, req, res)
return Batch(cfg, objects, operation, transferAdapters)
}
switch res.StatusCode {
case 404, 410:
return nil, "", errors.NewNotImplementedError(errors.Errorf("api: batch not implemented: %d", res.StatusCode))
}
tracerx.Printf("api error: %s", err)
return nil, "", errors.Wrap(err, "batch response")
}
httputil.LogTransfer(cfg, "lfs.batch", res)
if res.StatusCode != 200 {
return nil, "", errors.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
}
return bresp.Objects, bresp.TransferAdapterName, nil
}