// VerifyUpload calls the "verify" API link relation on obj if it exists
func VerifyUpload(cfg *config.Configuration, obj *ObjectResource) error {
// Do we need to do verify?
if _, ok := obj.Rel("verify"); !ok {
return nil
}
req, err := obj.NewRequest("verify", "POST")
if err != nil {
return errors.Wrap(err, "verify")
}
by, err := json.Marshal(obj)
if err != nil {
return errors.Wrap(err, "verify")
}
req.Header.Set("Content-Type", MediaType)
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
req.ContentLength = int64(len(by))
req.Body = ioutil.NopCloser(bytes.NewReader(by))
res, err := DoRequest(req, true)
if err != nil {
return err
}
httputil.LogTransfer(cfg, "lfs.data.verify", res)
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
return err
}
// TODO LEGACY API: remove when legacy API removed
func UploadCheck(cfg *config.Configuration, oid string, size int64) (*ObjectResource, error) {
reqObj := &ObjectResource{
Oid: oid,
Size: size,
}
by, err := json.Marshal(reqObj)
if err != nil {
return nil, errors.Wrap(err, "upload check")
}
req, err := NewRequest(cfg, "POST", oid)
if err != nil {
return nil, errors.Wrap(err, "upload check")
}
req.Header.Set("Content-Type", MediaType)
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
req.ContentLength = int64(len(by))
req.Body = tools.NewReadSeekCloserWrapper(bytes.NewReader(by))
tracerx.Printf("api: uploading (%s)", oid)
res, obj, err := DoLegacyRequest(cfg, req)
if err != nil {
if errors.IsAuthError(err) {
httputil.SetAuthType(cfg, req, res)
return UploadCheck(cfg, oid, size)
}
return nil, errors.NewRetriableError(err)
}
httputil.LogTransfer(cfg, "lfs.upload", res)
if res.StatusCode == 200 {
return nil, nil
}
if obj.Oid == "" {
obj.Oid = oid
}
if obj.Size == 0 {
obj.Size = reqObj.Size
}
return obj, nil
}
// DoHttpRequestWithRedirects runs a HTTP request and responds to redirects
func DoHttpRequestWithRedirects(cfg *config.Configuration, req *http.Request, via []*http.Request, useCreds bool) (*http.Response, error) {
var creds auth.Creds
if useCreds {
c, err := auth.GetCreds(cfg, req)
if err != nil {
return nil, err
}
creds = c
}
res, err := doHttpRequest(cfg, req, creds)
if err != nil {
return res, err
}
if res.StatusCode == 307 {
redirectTo := res.Header.Get("Location")
locurl, err := url.Parse(redirectTo)
if err == nil && !locurl.IsAbs() {
locurl = req.URL.ResolveReference(locurl)
redirectTo = locurl.String()
}
redirectedReq, err := NewHttpRequest(req.Method, redirectTo, nil)
if err != nil {
return res, errors.Wrapf(err, err.Error())
}
via = append(via, req)
// Avoid seeking and re-wrapping the CountingReadCloser, just get the "real" body
realBody := req.Body
if wrappedBody, ok := req.Body.(*CountingReadCloser); ok {
realBody = wrappedBody.ReadCloser
}
seeker, ok := realBody.(io.Seeker)
if !ok {
return res, errors.Wrapf(nil, "Request body needs to be an io.Seeker to handle redirects.")
}
if _, err := seeker.Seek(0, 0); err != nil {
return res, errors.Wrap(err, "request retry")
}
redirectedReq.Body = realBody
redirectedReq.ContentLength = req.ContentLength
if err = CheckRedirect(redirectedReq, via); err != nil {
return res, errors.Wrapf(err, err.Error())
}
return DoHttpRequestWithRedirects(cfg, redirectedReq, via, useCreds)
}
return res, nil
}
// TODO LEGACY API: remove when legacy API removed
func DownloadCheck(cfg *config.Configuration, oid string) (*ObjectResource, error) {
req, err := NewRequest(cfg, "GET", oid)
if err != nil {
return nil, errors.Wrap(err, "download check")
}
res, obj, err := DoLegacyRequest(cfg, req)
if err != nil {
return nil, err
}
httputil.LogTransfer(cfg, "lfs.download", res)
_, err = obj.NewRequest("download", "GET")
if err != nil {
return nil, errors.Wrap(err, "download check")
}
return obj, nil
}
// Internal http request management
func doHttpRequest(cfg *config.Configuration, req *http.Request, creds auth.Creds) (*http.Response, error) {
var (
res *http.Response
cause string
err error
)
if cfg.NtlmAccess(auth.GetOperationForRequest(req)) {
cause = "ntlm"
res, err = doNTLMRequest(cfg, req, true)
} else {
cause = "http"
res, err = NewHttpClient(cfg, req.Host).Do(req)
}
if res == nil {
res = &http.Response{
StatusCode: 0,
Header: make(http.Header),
Request: req,
Body: ioutil.NopCloser(bytes.NewBufferString("")),
}
}
if err != nil {
if errors.IsAuthError(err) {
SetAuthType(cfg, req, res)
doHttpRequest(cfg, req, creds)
} else {
err = errors.Wrap(err, cause)
}
} else {
err = handleResponse(cfg, res, creds)
}
if err != nil {
if res != nil {
SetErrorResponseContext(cfg, err, res)
} else {
setErrorRequestContext(cfg, err, req)
}
}
return res, err
}
// getCreds gets the credentials for a HTTP request and sets the given // request's Authorization header with them using Basic Authentication. // 1. Check the URL for authentication. Ex: http://user:[email protected] // 2. Check netrc for authentication. // 3. Check the Git remote URL for authentication IF it's the same scheme and // host of the URL. // 4. Ask 'git credential' to fill in the password from one of the above URLs. // // This prefers the Git remote URL for checking credentials so that users only // have to enter their passwords once for Git and Git LFS. It uses the same // URL path that Git does, in case 'useHttpPath' is enabled in the Git config. func GetCreds(cfg *config.Configuration, req *http.Request) (Creds, error) { if skipCredsCheck(cfg, req) { return nil, nil } credsUrl, err := getCredURLForAPI(cfg, req) if err != nil { return nil, errors.Wrap(err, "creds") } if credsUrl == nil { return nil, nil } if setCredURLFromNetrc(cfg, req) { return nil, nil } return fillCredentials(cfg, req, credsUrl) }
// Check the response from a HTTP request for problems
func handleResponse(cfg *config.Configuration, res *http.Response, creds auth.Creds) error {
auth.SaveCredentials(cfg, creds, res)
if res.StatusCode < 400 {
return nil
}
defer func() {
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
}()
cliErr := &ClientError{}
err := DecodeResponse(res, cliErr)
if err == nil {
if len(cliErr.Message) == 0 {
err = defaultError(res)
} else {
err = errors.Wrap(cliErr, "http")
}
}
if res.StatusCode == 401 {
if err == nil {
err = errors.New("api: received status 401")
}
return errors.NewAuthError(err)
}
if res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 507 && res.StatusCode != 509 {
if err == nil {
err = errors.Errorf("api: received status %d", res.StatusCode)
}
return errors.NewFatalError(err)
}
return err
}
func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile string, cb progress.CopyCallback) error {
reader, err := os.Open(mediafile)
if err != nil {
return errors.Wrapf(err, "Error opening media file.")
}
defer reader.Close()
if ptr.Size == 0 {
if stat, _ := os.Stat(mediafile); stat != nil {
ptr.Size = stat.Size()
}
}
if len(ptr.Extensions) > 0 {
registeredExts := config.Config.Extensions()
extensions := make(map[string]config.Extension)
for _, ptrExt := range ptr.Extensions {
ext, ok := registeredExts[ptrExt.Name]
if !ok {
err := fmt.Errorf("Extension '%s' is not configured.", ptrExt.Name)
return errors.Wrap(err, "smudge")
}
ext.Priority = ptrExt.Priority
extensions[ext.Name] = ext
}
exts, err := config.SortExtensions(extensions)
if err != nil {
return errors.Wrap(err, "smudge")
}
// pipe extensions in reverse order
var extsR []config.Extension
for i := range exts {
ext := exts[len(exts)-1-i]
extsR = append(extsR, ext)
}
request := &pipeRequest{"smudge", reader, workingfile, extsR}
response, err := pipeExtensions(request)
if err != nil {
return errors.Wrap(err, "smudge")
}
actualExts := make(map[string]*pipeExtResult)
for _, result := range response.results {
actualExts[result.name] = result
}
// verify name, order, and oids
oid := response.results[0].oidIn
if ptr.Oid != oid {
err = fmt.Errorf("Actual oid %s during smudge does not match expected %s", oid, ptr.Oid)
return errors.Wrap(err, "smudge")
}
for _, expected := range ptr.Extensions {
actual := actualExts[expected.Name]
if actual.name != expected.Name {
err = fmt.Errorf("Actual extension name '%s' does not match expected '%s'", actual.name, expected.Name)
return errors.Wrap(err, "smudge")
}
if actual.oidOut != expected.Oid {
err = fmt.Errorf("Actual oid %s for extension '%s' does not match expected %s", actual.oidOut, expected.Name, expected.Oid)
return errors.Wrap(err, "smudge")
}
}
// setup reader
reader, err = os.Open(response.file.Name())
if err != nil {
return errors.Wrapf(err, "Error opening smudged file: %s", err)
}
defer reader.Close()
}
_, err = tools.CopyWithCallback(writer, reader, ptr.Size, cb)
if err != nil {
return errors.Wrapf(err, "Error reading from media file: %s", err)
}
return nil
}
func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
rel, ok := t.Object.Rel("upload")
if !ok {
return fmt.Errorf("No upload action for this object.")
}
// Note not supporting the Creation extension since the batch API generates URLs
// Also not supporting Concatenation to support parallel uploads of chunks; forward only
// 1. Send HEAD request to determine upload start point
// Request must include Tus-Resumable header (version)
tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Object.Oid)
req, err := httputil.NewHttpRequest("HEAD", rel.Href, rel.Header)
if err != nil {
return err
}
req.Header.Set("Tus-Resumable", TusVersion)
res, err := httputil.DoHttpRequest(config.Config, req, false)
if err != nil {
return errors.NewRetriableError(err)
}
// Response will contain Upload-Offset if supported
offHdr := res.Header.Get("Upload-Offset")
if len(offHdr) == 0 {
return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href)
}
offset, err := strconv.ParseInt(offHdr, 10, 64)
if err != nil || offset < 0 {
return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href)
}
// Upload-Offset=size means already completed (skip)
// Batch API will probably already detect this, but handle just in case
if offset >= t.Object.Size {
tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Object.Oid)
advanceCallbackProgress(cb, t, t.Object.Size)
return nil
}
// Open file for uploading
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
if err != nil {
return errors.Wrap(err, "tus upload")
}
defer f.Close()
// Upload-Offset=0 means start from scratch, but still send PATCH
if offset == 0 {
tracerx.Printf("xfer: tus.io uploading %q from start", t.Object.Oid)
} else {
tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Object.Oid, offset)
advanceCallbackProgress(cb, t, offset)
_, err := f.Seek(offset, os.SEEK_CUR)
if err != nil {
return errors.Wrap(err, "tus upload")
}
}
// 2. Send PATCH request with byte start point (even if 0) in Upload-Offset
// Response status must be 204
// Response Upload-Offset must be request Upload-Offset plus sent bytes
// Response may include Upload-Expires header in which case check not passed
tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Object.Oid)
req, err = httputil.NewHttpRequest("PATCH", rel.Href, rel.Header)
if err != nil {
return err
}
req.Header.Set("Tus-Resumable", TusVersion)
req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10))
req.Header.Set("Content-Type", "application/offset+octet-stream")
req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size-offset, 10))
req.ContentLength = t.Object.Size - offset
// Ensure progress callbacks made while uploading
// Wrap callback to give name context
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
if cb != nil {
return cb(t.Name, totalSize, readSoFar, readSinceLast)
}
return nil
}
var reader io.Reader
reader = &progress.CallbackReader{
C: ccb,
TotalSize: t.Object.Size,
Reader: f,
}
// Signal auth was ok on first read; this frees up other workers to start
if authOkFunc != nil {
reader = newStartCallbackReader(reader, func(*startCallbackReader) {
authOkFunc()
})
}
req.Body = ioutil.NopCloser(reader)
res, err = httputil.DoHttpRequest(config.Config, req, false)
if err != nil {
return errors.NewRetriableError(err)
}
httputil.LogTransfer(config.Config, "lfs.data.upload", res)
// A status code of 403 likely means that an authentication token for the
// upload has expired. This can be safely retried.
if res.StatusCode == 403 {
err = errors.New("http: received status 403")
return errors.NewRetriableError(err)
}
if res.StatusCode > 299 {
return errors.Wrapf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
}
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
return api.VerifyUpload(config.Config, t.Object)
}
func (a *basicUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
rel, ok := t.Object.Rel("upload")
if !ok {
return fmt.Errorf("No upload action for this object.")
}
req, err := httputil.NewHttpRequest("PUT", rel.Href, rel.Header)
if err != nil {
return err
}
if len(req.Header.Get("Content-Type")) == 0 {
req.Header.Set("Content-Type", "application/octet-stream")
}
if req.Header.Get("Transfer-Encoding") == "chunked" {
req.TransferEncoding = []string{"chunked"}
} else {
req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size, 10))
}
req.ContentLength = t.Object.Size
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
if err != nil {
return errors.Wrap(err, "basic upload")
}
defer f.Close()
// Ensure progress callbacks made while uploading
// Wrap callback to give name context
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
if cb != nil {
return cb(t.Name, totalSize, readSoFar, readSinceLast)
}
return nil
}
var reader io.Reader
reader = &progress.CallbackReader{
C: ccb,
TotalSize: t.Object.Size,
Reader: f,
}
// Signal auth was ok on first read; this frees up other workers to start
if authOkFunc != nil {
reader = newStartCallbackReader(reader, func(*startCallbackReader) {
authOkFunc()
})
}
req.Body = ioutil.NopCloser(reader)
res, err := httputil.DoHttpRequest(config.Config, req, t.Object.NeedsAuth())
if err != nil {
return errors.NewRetriableError(err)
}
httputil.LogTransfer(config.Config, "lfs.data.upload", res)
// A status code of 403 likely means that an authentication token for the
// upload has expired. This can be safely retried.
if res.StatusCode == 403 {
err = errors.New("http: received status 403")
return errors.NewRetriableError(err)
}
if res.StatusCode > 299 {
return errors.Wrapf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
}
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
return api.VerifyUpload(config.Config, t.Object)
}
// Batch calls the batch API and returns object results
func Batch(cfg *config.Configuration, objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
if len(objects) == 0 {
return nil, "", nil
}
// Compatibility; omit transfers list when only basic
// older schemas included `additionalproperties=false`
if len(transferAdapters) == 1 && transferAdapters[0] == "basic" {
transferAdapters = nil
}
o := &batchRequest{Operation: operation, Objects: objects, TransferAdapterNames: transferAdapters}
by, err := json.Marshal(o)
if err != nil {
return nil, "", errors.Wrap(err, "batch request")
}
req, err := NewBatchRequest(cfg, operation)
if err != nil {
return nil, "", errors.Wrap(err, "batch request")
}
req.Header.Set("Content-Type", MediaType)
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
req.ContentLength = int64(len(by))
req.Body = tools.NewReadSeekCloserWrapper(bytes.NewReader(by))
tracerx.Printf("api: batch %d files", len(objects))
res, bresp, err := DoBatchRequest(cfg, req)
if err != nil {
if res == nil {
return nil, "", errors.NewRetriableError(err)
}
if res.StatusCode == 0 {
return nil, "", errors.NewRetriableError(err)
}
if errors.IsAuthError(err) {
httputil.SetAuthType(cfg, req, res)
return Batch(cfg, objects, operation, transferAdapters)
}
switch res.StatusCode {
case 404, 410:
return nil, "", errors.NewNotImplementedError(errors.Errorf("api: batch not implemented: %d", res.StatusCode))
}
tracerx.Printf("api error: %s", err)
return nil, "", errors.Wrap(err, "batch response")
}
httputil.LogTransfer(cfg, "lfs.batch", res)
if res.StatusCode != 200 {
return nil, "", errors.Errorf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
}
return bresp.Objects, bresp.TransferAdapterName, nil
}