// Batch calls the batch API and returns object results func Batch(objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) { if len(objects) == 0 { return nil, "", nil } o := &batchRequest{Operation: operation, Objects: objects, TransferAdapterNames: transferAdapters} by, err := json.Marshal(o) if err != nil { return nil, "", errutil.Error(err) } req, err := NewBatchRequest(operation) if err != nil { return nil, "", errutil.Error(err) } req.Header.Set("Content-Type", MediaType) req.Header.Set("Content-Length", strconv.Itoa(len(by))) req.ContentLength = int64(len(by)) req.Body = tools.NewReadSeekCloserWrapper(bytes.NewReader(by)) tracerx.Printf("api: batch %d files", len(objects)) res, bresp, err := DoBatchRequest(req) if err != nil { if res == nil { return nil, "", errutil.NewRetriableError(err) } if res.StatusCode == 0 { return nil, "", errutil.NewRetriableError(err) } if errutil.IsAuthError(err) { httputil.SetAuthType(req, res) return Batch(objects, operation, transferAdapters) } switch res.StatusCode { case 404, 410: tracerx.Printf("api: batch not implemented: %d", res.StatusCode) return nil, "", errutil.NewNotImplementedError(nil) } tracerx.Printf("api error: %s", err) return nil, "", errutil.Error(err) } httputil.LogTransfer("lfs.batch", res) if res.StatusCode != 200 { return nil, "", errutil.Error(fmt.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)) } return bresp.Objects, bresp.TransferAdapterName, nil }
func performUpload(oid string, size int64, a *action, fromPath string, writer, errWriter *bufio.Writer) { // We just use the URLs we're given, so we're just a proxy for the direct method // but this is enough to test intermediate custom adapters req, err := httputil.NewHttpRequest("PUT", a.Href, a.Header) if err != nil { sendTransferError(oid, 2, err.Error(), writer, errWriter) return } if len(req.Header.Get("Content-Type")) == 0 { req.Header.Set("Content-Type", "application/octet-stream") } if req.Header.Get("Transfer-Encoding") == "chunked" { req.TransferEncoding = []string{"chunked"} } else { req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) } req.ContentLength = size f, err := os.OpenFile(fromPath, os.O_RDONLY, 0644) if err != nil { sendTransferError(oid, 3, fmt.Sprintf("Cannot read data from %q: %v", fromPath, err), writer, errWriter) return } defer f.Close() // Turn callback into progress messages cb := func(totalSize int64, readSoFar int64, readSinceLast int) error { sendProgress(oid, readSoFar, readSinceLast, writer, errWriter) return nil } var reader io.Reader reader = &progress.CallbackReader{ C: cb, TotalSize: size, Reader: f, } req.Body = ioutil.NopCloser(reader) res, err := httputil.DoHttpRequest(cfg, req, true) if err != nil { sendTransferError(oid, res.StatusCode, fmt.Sprintf("Error uploading data for %s: %v", oid, err), writer, errWriter) return } if res.StatusCode > 299 { sendTransferError(oid, res.StatusCode, fmt.Sprintf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode), writer, errWriter) return } io.Copy(ioutil.Discard, res.Body) res.Body.Close() // completed complete := &transferResponse{"complete", oid, "", nil} err = sendResponse(complete, writer, errWriter) if err != nil { writeToStderr(fmt.Sprintf("Unable to send completion message: %v\n", err), errWriter) } }
func (a *basicUploadAdapter) DoTransfer(t *Transfer, cb TransferProgressCallback, authOkFunc func()) error { rel, ok := t.Object.Rel("upload") if !ok { return fmt.Errorf("No upload action for this object.") } req, err := httputil.NewHttpRequest("PUT", rel.Href, rel.Header) if err != nil { return err } if len(req.Header.Get("Content-Type")) == 0 { req.Header.Set("Content-Type", "application/octet-stream") } if req.Header.Get("Transfer-Encoding") == "chunked" { req.TransferEncoding = []string{"chunked"} } else { req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size, 10)) } req.ContentLength = t.Object.Size f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644) if err != nil { return errutil.Error(err) } defer f.Close() // Ensure progress callbacks made while uploading // Wrap callback to give name context ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error { if cb != nil { return cb(t.Name, totalSize, readSoFar, readSinceLast) } return nil } var reader io.Reader reader = &progress.CallbackReader{ C: ccb, TotalSize: t.Object.Size, Reader: f, } // Signal auth was ok on first read; this frees up other workers to start if authOkFunc != nil { reader = newStartCallbackReader(reader, func(*startCallbackReader) { authOkFunc() }) } req.Body = ioutil.NopCloser(reader) res, err := httputil.DoHttpRequest(req, true) if err != nil { return errutil.NewRetriableError(err) } httputil.LogTransfer("lfs.data.upload", res) // A status code of 403 likely means that an authentication token for the // upload has expired. This can be safely retried. if res.StatusCode == 403 { return errutil.NewRetriableError(err) } if res.StatusCode > 299 { return errutil.Errorf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode) } io.Copy(ioutil.Discard, res.Body) res.Body.Close() return api.VerifyUpload(t.Object) }
func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error { rel, ok := t.Object.Rel("upload") if !ok { return fmt.Errorf("No upload action for this object.") } // Note not supporting the Creation extension since the batch API generates URLs // Also not supporting Concatenation to support parallel uploads of chunks; forward only // 1. Send HEAD request to determine upload start point // Request must include Tus-Resumable header (version) tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Object.Oid) req, err := httputil.NewHttpRequest("HEAD", rel.Href, rel.Header) if err != nil { return err } req.Header.Set("Tus-Resumable", TusVersion) res, err := httputil.DoHttpRequest(config.Config, req, false) if err != nil { return errors.NewRetriableError(err) } // Response will contain Upload-Offset if supported offHdr := res.Header.Get("Upload-Offset") if len(offHdr) == 0 { return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href) } offset, err := strconv.ParseInt(offHdr, 10, 64) if err != nil || offset < 0 { return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href) } // Upload-Offset=size means already completed (skip) // Batch API will probably already detect this, but handle just in case if offset >= t.Object.Size { tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Object.Oid) advanceCallbackProgress(cb, t, t.Object.Size) return nil } // Open file for uploading f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644) if err != nil { return errors.Wrap(err, "tus upload") } defer f.Close() // Upload-Offset=0 means start from scratch, but still send PATCH if offset == 0 { tracerx.Printf("xfer: tus.io uploading %q from start", t.Object.Oid) } else { tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Object.Oid, offset) advanceCallbackProgress(cb, t, offset) _, err := f.Seek(offset, os.SEEK_CUR) if err != nil { return errors.Wrap(err, "tus upload") } } // 2. Send PATCH request with byte start point (even if 0) in Upload-Offset // Response status must be 204 // Response Upload-Offset must be request Upload-Offset plus sent bytes // Response may include Upload-Expires header in which case check not passed tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Object.Oid) req, err = httputil.NewHttpRequest("PATCH", rel.Href, rel.Header) if err != nil { return err } req.Header.Set("Tus-Resumable", TusVersion) req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10)) req.Header.Set("Content-Type", "application/offset+octet-stream") req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size-offset, 10)) req.ContentLength = t.Object.Size - offset // Ensure progress callbacks made while uploading // Wrap callback to give name context ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error { if cb != nil { return cb(t.Name, totalSize, readSoFar, readSinceLast) } return nil } var reader io.Reader reader = &progress.CallbackReader{ C: ccb, TotalSize: t.Object.Size, Reader: f, } // Signal auth was ok on first read; this frees up other workers to start if authOkFunc != nil { reader = newStartCallbackReader(reader, func(*startCallbackReader) { authOkFunc() }) } req.Body = ioutil.NopCloser(reader) res, err = httputil.DoHttpRequest(config.Config, req, false) if err != nil { return errors.NewRetriableError(err) } httputil.LogTransfer(config.Config, "lfs.data.upload", res) // A status code of 403 likely means that an authentication token for the // upload has expired. This can be safely retried. if res.StatusCode == 403 { err = errors.New("http: received status 403") return errors.NewRetriableError(err) } if res.StatusCode > 299 { return errors.Wrapf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode) } io.Copy(ioutil.Discard, res.Body) res.Body.Close() return api.VerifyUpload(config.Config, t.Object) }
// Batch calls the batch API and returns object results func Batch(cfg *config.Configuration, objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) { if len(objects) == 0 { return nil, "", nil } // Compatibility; omit transfers list when only basic // older schemas included `additionalproperties=false` if len(transferAdapters) == 1 && transferAdapters[0] == "basic" { transferAdapters = nil } o := &batchRequest{Operation: operation, Objects: objects, TransferAdapterNames: transferAdapters} by, err := json.Marshal(o) if err != nil { return nil, "", errors.Wrap(err, "batch request") } req, err := NewBatchRequest(cfg, operation) if err != nil { return nil, "", errors.Wrap(err, "batch request") } req.Header.Set("Content-Type", MediaType) req.Header.Set("Content-Length", strconv.Itoa(len(by))) req.ContentLength = int64(len(by)) req.Body = tools.NewReadSeekCloserWrapper(bytes.NewReader(by)) tracerx.Printf("api: batch %d files", len(objects)) res, bresp, err := DoBatchRequest(cfg, req) if err != nil { if res == nil { return nil, "", errors.NewRetriableError(err) } if res.StatusCode == 0 { return nil, "", errors.NewRetriableError(err) } if errors.IsAuthError(err) { httputil.SetAuthType(cfg, req, res) return Batch(cfg, objects, operation, transferAdapters) } switch res.StatusCode { case 404, 410: return nil, "", errors.NewNotImplementedError(errors.Errorf("api: batch not implemented: %d", res.StatusCode)) } tracerx.Printf("api error: %s", err) return nil, "", errors.Wrap(err, "batch response") } httputil.LogTransfer(cfg, "lfs.batch", res) if res.StatusCode != 200 { return nil, "", errors.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode) } return bresp.Objects, bresp.TransferAdapterName, nil }