func copyToTemp(reader io.Reader, fileSize int64, cb progress.CopyCallback) (oid string, size int64, tmp *os.File, err error) {
tmp, err = TempFile("")
if err != nil {
return
}
defer tmp.Close()
oidHash := sha256.New()
writer := io.MultiWriter(oidHash, tmp)
if fileSize == 0 {
cb = nil
}
by, ptr, err := DecodeFrom(reader)
if err == nil && len(by) < 512 {
err = errutil.NewCleanPointerError(err, ptr, by)
return
}
multi := io.MultiReader(bytes.NewReader(by), reader)
size, err = tools.CopyWithCallback(writer, multi, fileSize, cb)
if err != nil {
return
}
oid = hex.EncodeToString(oidHash.Sum(nil))
return
}
func performDownload(oid string, size int64, a *action, writer, errWriter *bufio.Writer) {
// We just use the URLs we're given, so we're just a proxy for the direct method
// but this is enough to test intermediate custom adapters
req, err := httputil.NewHttpRequest("GET", a.Href, a.Header)
if err != nil {
sendTransferError(oid, 2, err.Error(), writer, errWriter)
return
}
res, err := httputil.DoHttpRequest(cfg, req, true)
if err != nil {
sendTransferError(oid, res.StatusCode, err.Error(), writer, errWriter)
return
}
defer res.Body.Close()
dlFile, err := ioutil.TempFile("", "lfscustomdl")
if err != nil {
sendTransferError(oid, 3, err.Error(), writer, errWriter)
return
}
defer dlFile.Close()
dlfilename := dlFile.Name()
// Turn callback into progress messages
cb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
sendProgress(oid, readSoFar, readSinceLast, writer, errWriter)
return nil
}
_, err = tools.CopyWithCallback(dlFile, res.Body, res.ContentLength, cb)
if err != nil {
sendTransferError(oid, 4, fmt.Sprintf("cannot write data to tempfile %q: %v", dlfilename, err), writer, errWriter)
os.Remove(dlfilename)
return
}
if err := dlFile.Close(); err != nil {
sendTransferError(oid, 5, fmt.Sprintf("can't close tempfile %q: %v", dlfilename, err), writer, errWriter)
os.Remove(dlfilename)
return
}
// completed
complete := &transferResponse{"complete", oid, dlfilename, nil}
err = sendResponse(complete, writer, errWriter)
if err != nil {
writeToStderr(fmt.Sprintf("Unable to send completion message: %v\n", err), errWriter)
}
}
// download starts or resumes and download. Always closes dlFile if non-nil
func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback, authOkFunc func(), dlFile *os.File, fromByte int64, hash hash.Hash) error {
if dlFile != nil {
// ensure we always close dlFile. Note that this does not conflict with the
// early close below, as close is idempotent.
defer dlFile.Close()
}
rel, ok := t.Object.Rel("download")
if !ok {
return errors.New("Object not found on the server.")
}
req, err := httputil.NewHttpRequest("GET", rel.Href, rel.Header)
if err != nil {
return err
}
if fromByte > 0 {
if dlFile == nil || hash == nil {
return fmt.Errorf("Cannot restart %v from %d without a file & hash", t.Object.Oid, fromByte)
}
// We could just use a start byte, but since we know the length be specific
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", fromByte, t.Object.Size-1))
}
res, err := httputil.DoHttpRequest(req, true)
if err != nil {
// Special-case status code 416 () - fall back
if fromByte > 0 && dlFile != nil && res.StatusCode == 416 {
tracerx.Printf("xfer: server rejected resume download request for %q from byte %d; re-downloading from start", t.Object.Oid, fromByte)
dlFile.Close()
os.Remove(dlFile.Name())
return a.download(t, cb, authOkFunc, nil, 0, nil)
}
return errutil.NewRetriableError(err)
}
httputil.LogTransfer("lfs.data.download", res)
defer res.Body.Close()
// Range request must return 206 & content range to confirm
if fromByte > 0 {
rangeRequestOk := false
var failReason string
// check 206 and Content-Range, fall back if either not as expected
if res.StatusCode == 206 {
// Probably a successful range request, check Content-Range
if rangeHdr := res.Header.Get("Content-Range"); rangeHdr != "" {
regex := regexp.MustCompile(`bytes (\d+)\-.*`)
match := regex.FindStringSubmatch(rangeHdr)
if match != nil && len(match) > 1 {
contentStart, _ := strconv.ParseInt(match[1], 10, 64)
if contentStart == fromByte {
rangeRequestOk = true
} else {
failReason = fmt.Sprintf("Content-Range start byte incorrect: %s expected %d", match[1], fromByte)
}
} else {
failReason = fmt.Sprintf("badly formatted Content-Range header: %q", rangeHdr)
}
} else {
failReason = "missing Content-Range header in response"
}
} else {
failReason = fmt.Sprintf("expected status code 206, received %d", res.StatusCode)
}
if rangeRequestOk {
tracerx.Printf("xfer: server accepted resume download request: %q from byte %d", t.Object.Oid, fromByte)
// Advance progress callback; must split into max int sizes though
if cb != nil {
const maxInt = int(^uint(0) >> 1)
for read := int64(0); read < fromByte; {
remainder := fromByte - read
if remainder > int64(maxInt) {
read += int64(maxInt)
cb(t.Name, t.Object.Size, read, maxInt)
} else {
read += remainder
cb(t.Name, t.Object.Size, read, int(remainder))
}
}
}
} else {
// Abort resume, perform regular download
tracerx.Printf("xfer: failed to resume download for %q from byte %d: %s. Re-downloading from start", t.Object.Oid, fromByte, failReason)
dlFile.Close()
os.Remove(dlFile.Name())
if res.StatusCode == 200 {
// If status code was 200 then server just ignored Range header and
// sent everything. Don't re-request, use this one from byte 0
dlFile = nil
fromByte = 0
hash = nil
} else {
// re-request needed
return a.download(t, cb, authOkFunc, nil, 0, nil)
}
}
}
// Signal auth OK on success response, before starting download to free up
// other workers immediately
if authOkFunc != nil {
authOkFunc()
}
var hasher *tools.HashingReader
if fromByte > 0 && hash != nil {
// pre-load hashing reader with previous content
hasher = tools.NewHashingReaderPreloadHash(res.Body, hash)
} else {
hasher = tools.NewHashingReader(res.Body)
}
if dlFile == nil {
// New file start
dlFile, err = os.OpenFile(a.downloadFilename(t), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer dlFile.Close()
}
dlfilename := dlFile.Name()
// Wrap callback to give name context
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
if cb != nil {
return cb(t.Name, totalSize, readSoFar+fromByte, readSinceLast)
}
return nil
}
written, err := tools.CopyWithCallback(dlFile, hasher, res.ContentLength, ccb)
if err != nil {
return fmt.Errorf("cannot write data to tempfile %q: %v", dlfilename, err)
}
if err := dlFile.Close(); err != nil {
return fmt.Errorf("can't close tempfile %q: %v", dlfilename, err)
}
if actual := hasher.Hash(); actual != t.Object.Oid {
return fmt.Errorf("Expected OID %s, got %s after %d bytes written", t.Object.Oid, actual, written)
}
return tools.RenameFileCopyPermissions(dlfilename, t.Path)
}
func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile string, cb progress.CopyCallback) error {
reader, err := os.Open(mediafile)
if err != nil {
return errors.Wrapf(err, "Error opening media file.")
}
defer reader.Close()
if ptr.Size == 0 {
if stat, _ := os.Stat(mediafile); stat != nil {
ptr.Size = stat.Size()
}
}
if len(ptr.Extensions) > 0 {
registeredExts := config.Config.Extensions()
extensions := make(map[string]config.Extension)
for _, ptrExt := range ptr.Extensions {
ext, ok := registeredExts[ptrExt.Name]
if !ok {
err := fmt.Errorf("Extension '%s' is not configured.", ptrExt.Name)
return errors.Wrap(err, "smudge")
}
ext.Priority = ptrExt.Priority
extensions[ext.Name] = ext
}
exts, err := config.SortExtensions(extensions)
if err != nil {
return errors.Wrap(err, "smudge")
}
// pipe extensions in reverse order
var extsR []config.Extension
for i := range exts {
ext := exts[len(exts)-1-i]
extsR = append(extsR, ext)
}
request := &pipeRequest{"smudge", reader, workingfile, extsR}
response, err := pipeExtensions(request)
if err != nil {
return errors.Wrap(err, "smudge")
}
actualExts := make(map[string]*pipeExtResult)
for _, result := range response.results {
actualExts[result.name] = result
}
// verify name, order, and oids
oid := response.results[0].oidIn
if ptr.Oid != oid {
err = fmt.Errorf("Actual oid %s during smudge does not match expected %s", oid, ptr.Oid)
return errors.Wrap(err, "smudge")
}
for _, expected := range ptr.Extensions {
actual := actualExts[expected.Name]
if actual.name != expected.Name {
err = fmt.Errorf("Actual extension name '%s' does not match expected '%s'", actual.name, expected.Name)
return errors.Wrap(err, "smudge")
}
if actual.oidOut != expected.Oid {
err = fmt.Errorf("Actual oid %s for extension '%s' does not match expected %s", actual.oidOut, expected.Name, expected.Oid)
return errors.Wrap(err, "smudge")
}
}
// setup reader
reader, err = os.Open(response.file.Name())
if err != nil {
return errors.Wrapf(err, "Error opening smudged file: %s", err)
}
defer reader.Close()
}
_, err = tools.CopyWithCallback(writer, reader, ptr.Size, cb)
if err != nil {
return errors.Wrapf(err, "Error reading from media file: %s", err)
}
return nil
}