func verifyVersion(version string) error {
if len(version) == 0 {
return errors.NewNotAPointerError(errors.New("Missing version"))
}
for _, v := range v1Aliases {
if v == version {
return nil
}
}
return errors.New("Invalid version: " + version)
}
func parseOid(value string) (string, error) {
parts := strings.SplitN(value, ":", 2)
if len(parts) != 2 {
return "", errors.New("Invalid Oid value: " + value)
}
if parts[0] != oidType {
return "", errors.New("Invalid Oid type: " + parts[0])
}
oid := parts[1]
if !oidRE.Match([]byte(oid)) {
return "", errors.New("Invalid Oid: " + oid)
}
return oid, nil
}
func fillCredentials(cfg *config.Configuration, req *http.Request, u *url.URL) (Creds, error) {
path := strings.TrimPrefix(u.Path, "/")
input := Creds{"protocol": u.Scheme, "host": u.Host, "path": path}
if u.User != nil && u.User.Username() != "" {
input["username"] = u.User.Username()
}
creds, err := execCreds(cfg, input, "fill")
if creds == nil || len(creds) < 1 {
errmsg := fmt.Sprintf("Git credentials for %s not found", u)
if err != nil {
errmsg = errmsg + ":\n" + err.Error()
} else {
errmsg = errmsg + "."
}
err = errors.New(errmsg)
}
if err != nil {
return nil, err
}
tracerx.Printf("Filled credentials for %s", u)
setRequestAuth(cfg, req, creds["username"], creds["password"])
return creds, err
}
func parsePointerExtension(key string, value string) (*PointerExtension, error) {
keyParts := strings.SplitN(key, "-", 3)
if len(keyParts) != 3 || keyParts[0] != "ext" {
return nil, errors.New("Invalid extension value: " + value)
}
p, err := strconv.Atoi(keyParts[1])
if err != nil || p < 0 {
return nil, errors.New("Invalid priority: " + keyParts[1])
}
name := keyParts[2]
oid, err := parseOid(value)
if err != nil {
return nil, err
}
return NewPointerExtension(name, p, oid), nil
}
func TestRetriableReaderMakesErrorsRetriable(t *testing.T) {
expected := errors.New("example error")
r := tools.NewRetriableReader(&ErrReader{expected})
var buf [1]byte
n, err := r.Read(buf[:])
assert.Equal(t, 0, n)
assert.EqualError(t, err, "LFS: "+expected.Error())
assert.True(t, errors.IsRetriableError(err))
}
func decodeKVData(data []byte) (kvps map[string]string, exts map[string]string, err error) {
kvps = make(map[string]string)
if !matcherRE.Match(data) {
err = errors.NewNotAPointerError(errors.New("invalid header"))
return
}
scanner := bufio.NewScanner(bytes.NewBuffer(data))
line := 0
numKeys := len(pointerKeys)
for scanner.Scan() {
text := scanner.Text()
if len(text) == 0 {
continue
}
parts := strings.SplitN(text, " ", 2)
if len(parts) < 2 {
err = fmt.Errorf("Error reading line %d: %s", line, text)
return
}
key := parts[0]
value := parts[1]
if numKeys <= line {
err = fmt.Errorf("Extra line: %s", text)
return
}
if expected := pointerKeys[line]; key != expected {
if !extRE.Match([]byte(key)) {
err = errors.NewBadPointerKeyError(expected, key)
return
}
if exts == nil {
exts = make(map[string]string)
}
exts[key] = value
continue
}
line += 1
kvps[key] = value
}
err = scanner.Err()
return
}
// Uninstall removes the hook on disk so long as it matches the current version,
// or any of the past versions of this hook.
func (h *Hook) Uninstall() error {
if !InRepo() {
return errors.New("Not in a git repository")
}
match, err := h.matchesCurrent()
if err != nil {
return err
}
if !match {
return nil
}
return os.RemoveAll(h.Path())
}
func DecodePointerFromFile(file string) (*Pointer, error) {
// Check size before reading
stat, err := os.Stat(file)
if err != nil {
return nil, err
}
if stat.Size() > blobSizeCutoff {
return nil, errors.NewNotAPointerError(errors.New("file size exceeds lfs pointer size cutoff"))
}
f, err := os.OpenFile(file, os.O_RDONLY, 0644)
if err != nil {
return nil, err
}
defer f.Close()
return DecodePointer(f)
}
func decodeKV(data []byte) (*Pointer, error) {
kvps, exts, err := decodeKVData(data)
if err != nil {
if errors.IsBadPointerKeyError(err) {
return nil, errors.StandardizeBadPointerError(err)
}
return nil, err
}
if err := verifyVersion(kvps["version"]); err != nil {
return nil, err
}
value, ok := kvps["oid"]
if !ok {
return nil, errors.New("Invalid Oid")
}
oid, err := parseOid(value)
if err != nil {
return nil, err
}
value, ok = kvps["size"]
size, err := strconv.ParseInt(value, 10, 0)
if err != nil || size < 0 {
return nil, fmt.Errorf("Invalid size: %q", value)
}
var extensions []*PointerExtension
if exts != nil {
for key, value := range exts {
ext, err := parsePointerExtension(key, value)
if err != nil {
return nil, err
}
extensions = append(extensions, ext)
}
if err = validatePointerExtensions(extensions); err != nil {
return nil, err
}
sort.Sort(ByPriority(extensions))
}
return NewPointer(oid, size, extensions), nil
}
func TestRetriableReaderDoesNotRewrap(t *testing.T) {
// expected is already "retriable", as would be the case if the
// underlying reader was a *RetriableReader itself.
expected := errors.NewRetriableError(errors.New("example error"))
r := tools.NewRetriableReader(&ErrReader{expected})
var buf [1]byte
n, err := r.Read(buf[:])
assert.Equal(t, 0, n)
// errors.NewRetriableError wraps the given error with the prefix
// message "LFS", so these two errors should be equal, indicating that
// the RetriableReader did not re-wrap the error it received.
assert.EqualError(t, err, expected.Error())
assert.True(t, errors.IsRetriableError(err))
}
// Check the response from a HTTP request for problems
func handleResponse(cfg *config.Configuration, res *http.Response, creds auth.Creds) error {
auth.SaveCredentials(cfg, creds, res)
if res.StatusCode < 400 {
return nil
}
defer func() {
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
}()
cliErr := &ClientError{}
err := DecodeResponse(res, cliErr)
if err == nil {
if len(cliErr.Message) == 0 {
err = defaultError(res)
} else {
err = errors.Wrap(cliErr, "http")
}
}
if res.StatusCode == 401 {
if err == nil {
err = errors.New("api: received status 401")
}
return errors.NewAuthError(err)
}
if res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 507 && res.StatusCode != 509 {
if err == nil {
err = errors.Errorf("api: received status %d", res.StatusCode)
}
return errors.NewFatalError(err)
}
return err
}
func logsBoomtownCommand(cmd *cobra.Command, args []string) {
Debug("Debug message")
err := errors.Wrapf(errors.New("Inner error message!"), "Error")
Panic(err, "Welcome to Boomtown")
Debug("Never seen")
}
func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
rel, ok := t.Object.Rel("upload")
if !ok {
return fmt.Errorf("No upload action for this object.")
}
// Note not supporting the Creation extension since the batch API generates URLs
// Also not supporting Concatenation to support parallel uploads of chunks; forward only
// 1. Send HEAD request to determine upload start point
// Request must include Tus-Resumable header (version)
tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Object.Oid)
req, err := httputil.NewHttpRequest("HEAD", rel.Href, rel.Header)
if err != nil {
return err
}
req.Header.Set("Tus-Resumable", TusVersion)
res, err := httputil.DoHttpRequest(config.Config, req, false)
if err != nil {
return errors.NewRetriableError(err)
}
// Response will contain Upload-Offset if supported
offHdr := res.Header.Get("Upload-Offset")
if len(offHdr) == 0 {
return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href)
}
offset, err := strconv.ParseInt(offHdr, 10, 64)
if err != nil || offset < 0 {
return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href)
}
// Upload-Offset=size means already completed (skip)
// Batch API will probably already detect this, but handle just in case
if offset >= t.Object.Size {
tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Object.Oid)
advanceCallbackProgress(cb, t, t.Object.Size)
return nil
}
// Open file for uploading
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
if err != nil {
return errors.Wrap(err, "tus upload")
}
defer f.Close()
// Upload-Offset=0 means start from scratch, but still send PATCH
if offset == 0 {
tracerx.Printf("xfer: tus.io uploading %q from start", t.Object.Oid)
} else {
tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Object.Oid, offset)
advanceCallbackProgress(cb, t, offset)
_, err := f.Seek(offset, os.SEEK_CUR)
if err != nil {
return errors.Wrap(err, "tus upload")
}
}
// 2. Send PATCH request with byte start point (even if 0) in Upload-Offset
// Response status must be 204
// Response Upload-Offset must be request Upload-Offset plus sent bytes
// Response may include Upload-Expires header in which case check not passed
tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Object.Oid)
req, err = httputil.NewHttpRequest("PATCH", rel.Href, rel.Header)
if err != nil {
return err
}
req.Header.Set("Tus-Resumable", TusVersion)
req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10))
req.Header.Set("Content-Type", "application/offset+octet-stream")
req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size-offset, 10))
req.ContentLength = t.Object.Size - offset
// Ensure progress callbacks made while uploading
// Wrap callback to give name context
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
if cb != nil {
return cb(t.Name, totalSize, readSoFar, readSinceLast)
}
return nil
}
var reader io.Reader
reader = &progress.CallbackReader{
C: ccb,
TotalSize: t.Object.Size,
Reader: f,
}
// Signal auth was ok on first read; this frees up other workers to start
if authOkFunc != nil {
reader = newStartCallbackReader(reader, func(*startCallbackReader) {
authOkFunc()
})
}
req.Body = ioutil.NopCloser(reader)
res, err = httputil.DoHttpRequest(config.Config, req, false)
if err != nil {
return errors.NewRetriableError(err)
}
httputil.LogTransfer(config.Config, "lfs.data.upload", res)
// A status code of 403 likely means that an authentication token for the
// upload has expired. This can be safely retried.
if res.StatusCode == 403 {
err = errors.New("http: received status 403")
return errors.NewRetriableError(err)
}
if res.StatusCode > 299 {
return errors.Wrapf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
}
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
return api.VerifyUpload(config.Config, t.Object)
}
func (a *basicUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
rel, ok := t.Object.Rel("upload")
if !ok {
return fmt.Errorf("No upload action for this object.")
}
req, err := httputil.NewHttpRequest("PUT", rel.Href, rel.Header)
if err != nil {
return err
}
if len(req.Header.Get("Content-Type")) == 0 {
req.Header.Set("Content-Type", "application/octet-stream")
}
if req.Header.Get("Transfer-Encoding") == "chunked" {
req.TransferEncoding = []string{"chunked"}
} else {
req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size, 10))
}
req.ContentLength = t.Object.Size
f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
if err != nil {
return errors.Wrap(err, "basic upload")
}
defer f.Close()
// Ensure progress callbacks made while uploading
// Wrap callback to give name context
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
if cb != nil {
return cb(t.Name, totalSize, readSoFar, readSinceLast)
}
return nil
}
var reader io.Reader
reader = &progress.CallbackReader{
C: ccb,
TotalSize: t.Object.Size,
Reader: f,
}
// Signal auth was ok on first read; this frees up other workers to start
if authOkFunc != nil {
reader = newStartCallbackReader(reader, func(*startCallbackReader) {
authOkFunc()
})
}
req.Body = ioutil.NopCloser(reader)
res, err := httputil.DoHttpRequest(config.Config, req, t.Object.NeedsAuth())
if err != nil {
return errors.NewRetriableError(err)
}
httputil.LogTransfer(config.Config, "lfs.data.upload", res)
// A status code of 403 likely means that an authentication token for the
// upload has expired. This can be safely retried.
if res.StatusCode == 403 {
err = errors.New("http: received status 403")
return errors.NewRetriableError(err)
}
if res.StatusCode > 299 {
return errors.Wrapf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
}
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
return api.VerifyUpload(config.Config, t.Object)
}
// download starts or resumes and download. Always closes dlFile if non-nil
func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback, authOkFunc func(), dlFile *os.File, fromByte int64, hash hash.Hash) error {
if dlFile != nil {
// ensure we always close dlFile. Note that this does not conflict with the
// early close below, as close is idempotent.
defer dlFile.Close()
}
rel, ok := t.Object.Rel("download")
if !ok {
return errors.New("Object not found on the server.")
}
req, err := httputil.NewHttpRequest("GET", rel.Href, rel.Header)
if err != nil {
return err
}
if fromByte > 0 {
if dlFile == nil || hash == nil {
return fmt.Errorf("Cannot restart %v from %d without a file & hash", t.Object.Oid, fromByte)
}
// We could just use a start byte, but since we know the length be specific
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", fromByte, t.Object.Size-1))
}
res, err := httputil.DoHttpRequest(config.Config, req, t.Object.NeedsAuth())
if err != nil {
// Special-case status code 416 () - fall back
if fromByte > 0 && dlFile != nil && res.StatusCode == 416 {
tracerx.Printf("xfer: server rejected resume download request for %q from byte %d; re-downloading from start", t.Object.Oid, fromByte)
dlFile.Close()
os.Remove(dlFile.Name())
return a.download(t, cb, authOkFunc, nil, 0, nil)
}
return errors.NewRetriableError(err)
}
httputil.LogTransfer(config.Config, "lfs.data.download", res)
defer res.Body.Close()
// Range request must return 206 & content range to confirm
if fromByte > 0 {
rangeRequestOk := false
var failReason string
// check 206 and Content-Range, fall back if either not as expected
if res.StatusCode == 206 {
// Probably a successful range request, check Content-Range
if rangeHdr := res.Header.Get("Content-Range"); rangeHdr != "" {
regex := regexp.MustCompile(`bytes (\d+)\-.*`)
match := regex.FindStringSubmatch(rangeHdr)
if match != nil && len(match) > 1 {
contentStart, _ := strconv.ParseInt(match[1], 10, 64)
if contentStart == fromByte {
rangeRequestOk = true
} else {
failReason = fmt.Sprintf("Content-Range start byte incorrect: %s expected %d", match[1], fromByte)
}
} else {
failReason = fmt.Sprintf("badly formatted Content-Range header: %q", rangeHdr)
}
} else {
failReason = "missing Content-Range header in response"
}
} else {
failReason = fmt.Sprintf("expected status code 206, received %d", res.StatusCode)
}
if rangeRequestOk {
tracerx.Printf("xfer: server accepted resume download request: %q from byte %d", t.Object.Oid, fromByte)
advanceCallbackProgress(cb, t, fromByte)
} else {
// Abort resume, perform regular download
tracerx.Printf("xfer: failed to resume download for %q from byte %d: %s. Re-downloading from start", t.Object.Oid, fromByte, failReason)
dlFile.Close()
os.Remove(dlFile.Name())
if res.StatusCode == 200 {
// If status code was 200 then server just ignored Range header and
// sent everything. Don't re-request, use this one from byte 0
dlFile = nil
fromByte = 0
hash = nil
} else {
// re-request needed
return a.download(t, cb, authOkFunc, nil, 0, nil)
}
}
}
// Signal auth OK on success response, before starting download to free up
// other workers immediately
if authOkFunc != nil {
authOkFunc()
}
var hasher *tools.HashingReader
httpReader := tools.NewRetriableReader(res.Body)
if fromByte > 0 && hash != nil {
// pre-load hashing reader with previous content
hasher = tools.NewHashingReaderPreloadHash(httpReader, hash)
} else {
hasher = tools.NewHashingReader(httpReader)
}
if dlFile == nil {
// New file start
dlFile, err = os.OpenFile(a.downloadFilename(t), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer dlFile.Close()
}
dlfilename := dlFile.Name()
// Wrap callback to give name context
ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
if cb != nil {
return cb(t.Name, totalSize, readSoFar+fromByte, readSinceLast)
}
return nil
}
written, err := tools.CopyWithCallback(dlFile, hasher, res.ContentLength, ccb)
if err != nil {
return errors.Wrapf(err, "cannot write data to tempfile %q", dlfilename)
}
if err := dlFile.Close(); err != nil {
return fmt.Errorf("can't close tempfile %q: %v", dlfilename, err)
}
if actual := hasher.Hash(); actual != t.Object.Oid {
return fmt.Errorf("Expected OID %s, got %s after %d bytes written", t.Object.Oid, actual, written)
}
return tools.RenameFileCopyPermissions(dlfilename, t.Path)
}