func verifyVersion(version string) error { if len(version) == 0 { return errors.NewNotAPointerError(errors.New("Missing version")) } for _, v := range v1Aliases { if v == version { return nil } } return errors.New("Invalid version: " + version) }
func parseOid(value string) (string, error) { parts := strings.SplitN(value, ":", 2) if len(parts) != 2 { return "", errors.New("Invalid Oid value: " + value) } if parts[0] != oidType { return "", errors.New("Invalid Oid type: " + parts[0]) } oid := parts[1] if !oidRE.Match([]byte(oid)) { return "", errors.New("Invalid Oid: " + oid) } return oid, nil }
func fillCredentials(cfg *config.Configuration, req *http.Request, u *url.URL) (Creds, error) { path := strings.TrimPrefix(u.Path, "/") input := Creds{"protocol": u.Scheme, "host": u.Host, "path": path} if u.User != nil && u.User.Username() != "" { input["username"] = u.User.Username() } creds, err := execCreds(cfg, input, "fill") if creds == nil || len(creds) < 1 { errmsg := fmt.Sprintf("Git credentials for %s not found", u) if err != nil { errmsg = errmsg + ":\n" + err.Error() } else { errmsg = errmsg + "." } err = errors.New(errmsg) } if err != nil { return nil, err } tracerx.Printf("Filled credentials for %s", u) setRequestAuth(cfg, req, creds["username"], creds["password"]) return creds, err }
func parsePointerExtension(key string, value string) (*PointerExtension, error) { keyParts := strings.SplitN(key, "-", 3) if len(keyParts) != 3 || keyParts[0] != "ext" { return nil, errors.New("Invalid extension value: " + value) } p, err := strconv.Atoi(keyParts[1]) if err != nil || p < 0 { return nil, errors.New("Invalid priority: " + keyParts[1]) } name := keyParts[2] oid, err := parseOid(value) if err != nil { return nil, err } return NewPointerExtension(name, p, oid), nil }
func TestRetriableReaderMakesErrorsRetriable(t *testing.T) { expected := errors.New("example error") r := tools.NewRetriableReader(&ErrReader{expected}) var buf [1]byte n, err := r.Read(buf[:]) assert.Equal(t, 0, n) assert.EqualError(t, err, "LFS: "+expected.Error()) assert.True(t, errors.IsRetriableError(err)) }
func decodeKVData(data []byte) (kvps map[string]string, exts map[string]string, err error) { kvps = make(map[string]string) if !matcherRE.Match(data) { err = errors.NewNotAPointerError(errors.New("invalid header")) return } scanner := bufio.NewScanner(bytes.NewBuffer(data)) line := 0 numKeys := len(pointerKeys) for scanner.Scan() { text := scanner.Text() if len(text) == 0 { continue } parts := strings.SplitN(text, " ", 2) if len(parts) < 2 { err = fmt.Errorf("Error reading line %d: %s", line, text) return } key := parts[0] value := parts[1] if numKeys <= line { err = fmt.Errorf("Extra line: %s", text) return } if expected := pointerKeys[line]; key != expected { if !extRE.Match([]byte(key)) { err = errors.NewBadPointerKeyError(expected, key) return } if exts == nil { exts = make(map[string]string) } exts[key] = value continue } line += 1 kvps[key] = value } err = scanner.Err() return }
// Uninstall removes the hook on disk so long as it matches the current version, // or any of the past versions of this hook. func (h *Hook) Uninstall() error { if !InRepo() { return errors.New("Not in a git repository") } match, err := h.matchesCurrent() if err != nil { return err } if !match { return nil } return os.RemoveAll(h.Path()) }
func DecodePointerFromFile(file string) (*Pointer, error) { // Check size before reading stat, err := os.Stat(file) if err != nil { return nil, err } if stat.Size() > blobSizeCutoff { return nil, errors.NewNotAPointerError(errors.New("file size exceeds lfs pointer size cutoff")) } f, err := os.OpenFile(file, os.O_RDONLY, 0644) if err != nil { return nil, err } defer f.Close() return DecodePointer(f) }
func decodeKV(data []byte) (*Pointer, error) { kvps, exts, err := decodeKVData(data) if err != nil { if errors.IsBadPointerKeyError(err) { return nil, errors.StandardizeBadPointerError(err) } return nil, err } if err := verifyVersion(kvps["version"]); err != nil { return nil, err } value, ok := kvps["oid"] if !ok { return nil, errors.New("Invalid Oid") } oid, err := parseOid(value) if err != nil { return nil, err } value, ok = kvps["size"] size, err := strconv.ParseInt(value, 10, 0) if err != nil || size < 0 { return nil, fmt.Errorf("Invalid size: %q", value) } var extensions []*PointerExtension if exts != nil { for key, value := range exts { ext, err := parsePointerExtension(key, value) if err != nil { return nil, err } extensions = append(extensions, ext) } if err = validatePointerExtensions(extensions); err != nil { return nil, err } sort.Sort(ByPriority(extensions)) } return NewPointer(oid, size, extensions), nil }
func TestRetriableReaderDoesNotRewrap(t *testing.T) { // expected is already "retriable", as would be the case if the // underlying reader was a *RetriableReader itself. expected := errors.NewRetriableError(errors.New("example error")) r := tools.NewRetriableReader(&ErrReader{expected}) var buf [1]byte n, err := r.Read(buf[:]) assert.Equal(t, 0, n) // errors.NewRetriableError wraps the given error with the prefix // message "LFS", so these two errors should be equal, indicating that // the RetriableReader did not re-wrap the error it received. assert.EqualError(t, err, expected.Error()) assert.True(t, errors.IsRetriableError(err)) }
// Check the response from a HTTP request for problems func handleResponse(cfg *config.Configuration, res *http.Response, creds auth.Creds) error { auth.SaveCredentials(cfg, creds, res) if res.StatusCode < 400 { return nil } defer func() { io.Copy(ioutil.Discard, res.Body) res.Body.Close() }() cliErr := &ClientError{} err := DecodeResponse(res, cliErr) if err == nil { if len(cliErr.Message) == 0 { err = defaultError(res) } else { err = errors.Wrap(cliErr, "http") } } if res.StatusCode == 401 { if err == nil { err = errors.New("api: received status 401") } return errors.NewAuthError(err) } if res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 507 && res.StatusCode != 509 { if err == nil { err = errors.Errorf("api: received status %d", res.StatusCode) } return errors.NewFatalError(err) } return err }
func logsBoomtownCommand(cmd *cobra.Command, args []string) { Debug("Debug message") err := errors.Wrapf(errors.New("Inner error message!"), "Error") Panic(err, "Welcome to Boomtown") Debug("Never seen") }
func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error { rel, ok := t.Object.Rel("upload") if !ok { return fmt.Errorf("No upload action for this object.") } // Note not supporting the Creation extension since the batch API generates URLs // Also not supporting Concatenation to support parallel uploads of chunks; forward only // 1. Send HEAD request to determine upload start point // Request must include Tus-Resumable header (version) tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Object.Oid) req, err := httputil.NewHttpRequest("HEAD", rel.Href, rel.Header) if err != nil { return err } req.Header.Set("Tus-Resumable", TusVersion) res, err := httputil.DoHttpRequest(config.Config, req, false) if err != nil { return errors.NewRetriableError(err) } // Response will contain Upload-Offset if supported offHdr := res.Header.Get("Upload-Offset") if len(offHdr) == 0 { return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href) } offset, err := strconv.ParseInt(offHdr, 10, 64) if err != nil || offset < 0 { return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href) } // Upload-Offset=size means already completed (skip) // Batch API will probably already detect this, but handle just in case if offset >= t.Object.Size { tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Object.Oid) advanceCallbackProgress(cb, t, t.Object.Size) return nil } // Open file for uploading f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644) if err != nil { return errors.Wrap(err, "tus upload") } defer f.Close() // Upload-Offset=0 means start from scratch, but still send PATCH if offset == 0 { tracerx.Printf("xfer: tus.io uploading %q from start", t.Object.Oid) } else { tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Object.Oid, offset) advanceCallbackProgress(cb, t, offset) _, err := f.Seek(offset, os.SEEK_CUR) if err != nil { return errors.Wrap(err, "tus upload") } } // 2. Send PATCH request with byte start point (even if 0) in Upload-Offset // Response status must be 204 // Response Upload-Offset must be request Upload-Offset plus sent bytes // Response may include Upload-Expires header in which case check not passed tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Object.Oid) req, err = httputil.NewHttpRequest("PATCH", rel.Href, rel.Header) if err != nil { return err } req.Header.Set("Tus-Resumable", TusVersion) req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10)) req.Header.Set("Content-Type", "application/offset+octet-stream") req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size-offset, 10)) req.ContentLength = t.Object.Size - offset // Ensure progress callbacks made while uploading // Wrap callback to give name context ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error { if cb != nil { return cb(t.Name, totalSize, readSoFar, readSinceLast) } return nil } var reader io.Reader reader = &progress.CallbackReader{ C: ccb, TotalSize: t.Object.Size, Reader: f, } // Signal auth was ok on first read; this frees up other workers to start if authOkFunc != nil { reader = newStartCallbackReader(reader, func(*startCallbackReader) { authOkFunc() }) } req.Body = ioutil.NopCloser(reader) res, err = httputil.DoHttpRequest(config.Config, req, false) if err != nil { return errors.NewRetriableError(err) } httputil.LogTransfer(config.Config, "lfs.data.upload", res) // A status code of 403 likely means that an authentication token for the // upload has expired. This can be safely retried. if res.StatusCode == 403 { err = errors.New("http: received status 403") return errors.NewRetriableError(err) } if res.StatusCode > 299 { return errors.Wrapf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode) } io.Copy(ioutil.Discard, res.Body) res.Body.Close() return api.VerifyUpload(config.Config, t.Object) }
func (a *basicUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error { rel, ok := t.Object.Rel("upload") if !ok { return fmt.Errorf("No upload action for this object.") } req, err := httputil.NewHttpRequest("PUT", rel.Href, rel.Header) if err != nil { return err } if len(req.Header.Get("Content-Type")) == 0 { req.Header.Set("Content-Type", "application/octet-stream") } if req.Header.Get("Transfer-Encoding") == "chunked" { req.TransferEncoding = []string{"chunked"} } else { req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size, 10)) } req.ContentLength = t.Object.Size f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644) if err != nil { return errors.Wrap(err, "basic upload") } defer f.Close() // Ensure progress callbacks made while uploading // Wrap callback to give name context ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error { if cb != nil { return cb(t.Name, totalSize, readSoFar, readSinceLast) } return nil } var reader io.Reader reader = &progress.CallbackReader{ C: ccb, TotalSize: t.Object.Size, Reader: f, } // Signal auth was ok on first read; this frees up other workers to start if authOkFunc != nil { reader = newStartCallbackReader(reader, func(*startCallbackReader) { authOkFunc() }) } req.Body = ioutil.NopCloser(reader) res, err := httputil.DoHttpRequest(config.Config, req, t.Object.NeedsAuth()) if err != nil { return errors.NewRetriableError(err) } httputil.LogTransfer(config.Config, "lfs.data.upload", res) // A status code of 403 likely means that an authentication token for the // upload has expired. This can be safely retried. if res.StatusCode == 403 { err = errors.New("http: received status 403") return errors.NewRetriableError(err) } if res.StatusCode > 299 { return errors.Wrapf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode) } io.Copy(ioutil.Discard, res.Body) res.Body.Close() return api.VerifyUpload(config.Config, t.Object) }
// download starts or resumes and download. Always closes dlFile if non-nil func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback, authOkFunc func(), dlFile *os.File, fromByte int64, hash hash.Hash) error { if dlFile != nil { // ensure we always close dlFile. Note that this does not conflict with the // early close below, as close is idempotent. defer dlFile.Close() } rel, ok := t.Object.Rel("download") if !ok { return errors.New("Object not found on the server.") } req, err := httputil.NewHttpRequest("GET", rel.Href, rel.Header) if err != nil { return err } if fromByte > 0 { if dlFile == nil || hash == nil { return fmt.Errorf("Cannot restart %v from %d without a file & hash", t.Object.Oid, fromByte) } // We could just use a start byte, but since we know the length be specific req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", fromByte, t.Object.Size-1)) } res, err := httputil.DoHttpRequest(config.Config, req, t.Object.NeedsAuth()) if err != nil { // Special-case status code 416 () - fall back if fromByte > 0 && dlFile != nil && res.StatusCode == 416 { tracerx.Printf("xfer: server rejected resume download request for %q from byte %d; re-downloading from start", t.Object.Oid, fromByte) dlFile.Close() os.Remove(dlFile.Name()) return a.download(t, cb, authOkFunc, nil, 0, nil) } return errors.NewRetriableError(err) } httputil.LogTransfer(config.Config, "lfs.data.download", res) defer res.Body.Close() // Range request must return 206 & content range to confirm if fromByte > 0 { rangeRequestOk := false var failReason string // check 206 and Content-Range, fall back if either not as expected if res.StatusCode == 206 { // Probably a successful range request, check Content-Range if rangeHdr := res.Header.Get("Content-Range"); rangeHdr != "" { regex := regexp.MustCompile(`bytes (\d+)\-.*`) match := regex.FindStringSubmatch(rangeHdr) if match != nil && len(match) > 1 { contentStart, _ := strconv.ParseInt(match[1], 10, 64) if contentStart == fromByte { rangeRequestOk = true } else { failReason = fmt.Sprintf("Content-Range start byte incorrect: %s expected %d", match[1], fromByte) } } else { failReason = fmt.Sprintf("badly formatted Content-Range header: %q", rangeHdr) } } else { failReason = "missing Content-Range header in response" } } else { failReason = fmt.Sprintf("expected status code 206, received %d", res.StatusCode) } if rangeRequestOk { tracerx.Printf("xfer: server accepted resume download request: %q from byte %d", t.Object.Oid, fromByte) advanceCallbackProgress(cb, t, fromByte) } else { // Abort resume, perform regular download tracerx.Printf("xfer: failed to resume download for %q from byte %d: %s. Re-downloading from start", t.Object.Oid, fromByte, failReason) dlFile.Close() os.Remove(dlFile.Name()) if res.StatusCode == 200 { // If status code was 200 then server just ignored Range header and // sent everything. Don't re-request, use this one from byte 0 dlFile = nil fromByte = 0 hash = nil } else { // re-request needed return a.download(t, cb, authOkFunc, nil, 0, nil) } } } // Signal auth OK on success response, before starting download to free up // other workers immediately if authOkFunc != nil { authOkFunc() } var hasher *tools.HashingReader httpReader := tools.NewRetriableReader(res.Body) if fromByte > 0 && hash != nil { // pre-load hashing reader with previous content hasher = tools.NewHashingReaderPreloadHash(httpReader, hash) } else { hasher = tools.NewHashingReader(httpReader) } if dlFile == nil { // New file start dlFile, err = os.OpenFile(a.downloadFilename(t), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err } defer dlFile.Close() } dlfilename := dlFile.Name() // Wrap callback to give name context ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error { if cb != nil { return cb(t.Name, totalSize, readSoFar+fromByte, readSinceLast) } return nil } written, err := tools.CopyWithCallback(dlFile, hasher, res.ContentLength, ccb) if err != nil { return errors.Wrapf(err, "cannot write data to tempfile %q", dlfilename) } if err := dlFile.Close(); err != nil { return fmt.Errorf("can't close tempfile %q: %v", dlfilename, err) } if actual := hasher.Hash(); actual != t.Object.Oid { return fmt.Errorf("Expected OID %s, got %s after %d bytes written", t.Object.Oid, actual, written) } return tools.RenameFileCopyPermissions(dlfilename, t.Path) }