func TestRetriableReaderReturnsEOFs(t *testing.T) { r := tools.NewRetriableReader(bytes.NewBuffer([]byte{ /* empty */ })) var buf [1]byte n, err := r.Read(buf[:]) assert.Equal(t, io.EOF, err) assert.Equal(t, 0, n) }
func TestRetriableReaderReturnsSuccessfulReads(t *testing.T) { r := tools.NewRetriableReader(bytes.NewBuffer([]byte{0x1, 0x2, 0x3, 0x4})) var buf [4]byte n, err := r.Read(buf[:]) assert.Nil(t, err) assert.Equal(t, 4, n) assert.Equal(t, []byte{0x1, 0x2, 0x3, 0x4}, buf[:]) }
func TestRetriableReaderMakesErrorsRetriable(t *testing.T) { expected := errors.New("example error") r := tools.NewRetriableReader(&ErrReader{expected}) var buf [1]byte n, err := r.Read(buf[:]) assert.Equal(t, 0, n) assert.EqualError(t, err, "LFS: "+expected.Error()) assert.True(t, errors.IsRetriableError(err)) }
func TestRetriableReaderDoesNotRewrap(t *testing.T) { // expected is already "retriable", as would be the case if the // underlying reader was a *RetriableReader itself. expected := errors.NewRetriableError(errors.New("example error")) r := tools.NewRetriableReader(&ErrReader{expected}) var buf [1]byte n, err := r.Read(buf[:]) assert.Equal(t, 0, n) // errors.NewRetriableError wraps the given error with the prefix // message "LFS", so these two errors should be equal, indicating that // the RetriableReader did not re-wrap the error it received. assert.EqualError(t, err, expected.Error()) assert.True(t, errors.IsRetriableError(err)) }
// download starts or resumes and download. Always closes dlFile if non-nil func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback, authOkFunc func(), dlFile *os.File, fromByte int64, hash hash.Hash) error { if dlFile != nil { // ensure we always close dlFile. Note that this does not conflict with the // early close below, as close is idempotent. defer dlFile.Close() } rel, ok := t.Object.Rel("download") if !ok { return errors.New("Object not found on the server.") } req, err := httputil.NewHttpRequest("GET", rel.Href, rel.Header) if err != nil { return err } if fromByte > 0 { if dlFile == nil || hash == nil { return fmt.Errorf("Cannot restart %v from %d without a file & hash", t.Object.Oid, fromByte) } // We could just use a start byte, but since we know the length be specific req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", fromByte, t.Object.Size-1)) } res, err := httputil.DoHttpRequest(config.Config, req, t.Object.NeedsAuth()) if err != nil { // Special-case status code 416 () - fall back if fromByte > 0 && dlFile != nil && res.StatusCode == 416 { tracerx.Printf("xfer: server rejected resume download request for %q from byte %d; re-downloading from start", t.Object.Oid, fromByte) dlFile.Close() os.Remove(dlFile.Name()) return a.download(t, cb, authOkFunc, nil, 0, nil) } return errors.NewRetriableError(err) } httputil.LogTransfer(config.Config, "lfs.data.download", res) defer res.Body.Close() // Range request must return 206 & content range to confirm if fromByte > 0 { rangeRequestOk := false var failReason string // check 206 and Content-Range, fall back if either not as expected if res.StatusCode == 206 { // Probably a successful range request, check Content-Range if rangeHdr := res.Header.Get("Content-Range"); rangeHdr != "" { regex := regexp.MustCompile(`bytes (\d+)\-.*`) match := regex.FindStringSubmatch(rangeHdr) if match != nil && len(match) > 1 { contentStart, _ := strconv.ParseInt(match[1], 10, 64) if contentStart == fromByte { rangeRequestOk = true } else { failReason = fmt.Sprintf("Content-Range start byte incorrect: %s expected %d", match[1], fromByte) } } else { failReason = fmt.Sprintf("badly formatted Content-Range header: %q", rangeHdr) } } else { failReason = "missing Content-Range header in response" } } else { failReason = fmt.Sprintf("expected status code 206, received %d", res.StatusCode) } if rangeRequestOk { tracerx.Printf("xfer: server accepted resume download request: %q from byte %d", t.Object.Oid, fromByte) advanceCallbackProgress(cb, t, fromByte) } else { // Abort resume, perform regular download tracerx.Printf("xfer: failed to resume download for %q from byte %d: %s. Re-downloading from start", t.Object.Oid, fromByte, failReason) dlFile.Close() os.Remove(dlFile.Name()) if res.StatusCode == 200 { // If status code was 200 then server just ignored Range header and // sent everything. Don't re-request, use this one from byte 0 dlFile = nil fromByte = 0 hash = nil } else { // re-request needed return a.download(t, cb, authOkFunc, nil, 0, nil) } } } // Signal auth OK on success response, before starting download to free up // other workers immediately if authOkFunc != nil { authOkFunc() } var hasher *tools.HashingReader httpReader := tools.NewRetriableReader(res.Body) if fromByte > 0 && hash != nil { // pre-load hashing reader with previous content hasher = tools.NewHashingReaderPreloadHash(httpReader, hash) } else { hasher = tools.NewHashingReader(httpReader) } if dlFile == nil { // New file start dlFile, err = os.OpenFile(a.downloadFilename(t), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err } defer dlFile.Close() } dlfilename := dlFile.Name() // Wrap callback to give name context ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error { if cb != nil { return cb(t.Name, totalSize, readSoFar+fromByte, readSinceLast) } return nil } written, err := tools.CopyWithCallback(dlFile, hasher, res.ContentLength, ccb) if err != nil { return errors.Wrapf(err, "cannot write data to tempfile %q", dlfilename) } if err := dlFile.Close(); err != nil { return fmt.Errorf("can't close tempfile %q: %v", dlfilename, err) } if actual := hasher.Hash(); actual != t.Object.Oid { return fmt.Errorf("Expected OID %s, got %s after %d bytes written", t.Object.Oid, actual, written) } return tools.RenameFileCopyPermissions(dlfilename, t.Path) }