func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, cb progress.CopyCallback) error { fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, pb.FormatBytes(ptr.Size)) xfers := transfer.GetDownloadAdapterNames() obj, adapterName, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: ptr.Oid, Size: ptr.Size}, "download", xfers) if err != nil { return errutil.Errorf(err, "Error downloading %s: %s", filepath.Base(mediafile), err) } if ptr.Size == 0 { ptr.Size = obj.Size } adapter := transfer.NewDownloadAdapter(adapterName) var tcb transfer.TransferProgressCallback if cb != nil { tcb = func(name string, totalSize, readSoFar int64, readSinceLast int) error { return cb(totalSize, readSoFar, readSinceLast) } } // Single download adapterResultChan := make(chan transfer.TransferResult, 1) adapter.Begin(1, tcb, adapterResultChan) adapter.Add(transfer.NewTransfer(filepath.Base(workingfile), obj, mediafile)) adapter.End() res := <-adapterResultChan if res.Error != nil { return errutil.Errorf(err, "Error buffering media file: %s", res.Error) } return readLocalFile(writer, ptr, mediafile, workingfile, nil) }
// DoHttpRequestWithRedirects runs a HTTP request and responds to redirects func DoHttpRequestWithRedirects(req *http.Request, via []*http.Request, useCreds bool) (*http.Response, error) { var creds auth.Creds if useCreds { c, err := auth.GetCreds(req) if err != nil { return nil, err } creds = c } res, err := doHttpRequest(req, creds) if err != nil { return res, err } if res.StatusCode == 307 { redirectTo := res.Header.Get("Location") locurl, err := url.Parse(redirectTo) if err == nil && !locurl.IsAbs() { locurl = req.URL.ResolveReference(locurl) redirectTo = locurl.String() } redirectedReq, err := NewHttpRequest(req.Method, redirectTo, nil) if err != nil { return res, errutil.Errorf(err, err.Error()) } via = append(via, req) // Avoid seeking and re-wrapping the CountingReadCloser, just get the "real" body realBody := req.Body if wrappedBody, ok := req.Body.(*CountingReadCloser); ok { realBody = wrappedBody.ReadCloser } seeker, ok := realBody.(io.Seeker) if !ok { return res, errutil.Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.") } if _, err := seeker.Seek(0, 0); err != nil { return res, errutil.Error(err) } redirectedReq.Body = realBody redirectedReq.ContentLength = req.ContentLength if err = CheckRedirect(redirectedReq, via); err != nil { return res, errutil.Errorf(err, err.Error()) } return DoHttpRequestWithRedirects(redirectedReq, via, useCreds) } return res, nil }
// NewUploadable builds the Uploadable from the given information. // "filename" can be empty if a raw object is pushed (see "object-id" flag in push command)/ func NewUploadable(oid, filename string) (*Uploadable, error) { localMediaPath, err := LocalMediaPath(oid) if err != nil { return nil, errutil.Errorf(err, "Error uploading file %s (%s)", filename, oid) } if len(filename) > 0 { if err := ensureFile(filename, localMediaPath); err != nil { return nil, err } } fi, err := os.Stat(localMediaPath) if err != nil { return nil, errutil.Errorf(err, "Error uploading file %s (%s)", filename, oid) } return &Uploadable{oid: oid, OidPath: localMediaPath, Filename: filename, size: fi.Size()}, nil }
// DecodeResponse attempts to decode the contents of the response as a JSON object func DecodeResponse(res *http.Response, obj interface{}) error { ctype := res.Header.Get("Content-Type") if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) { return nil } err := json.NewDecoder(res.Body).Decode(obj) io.Copy(ioutil.Discard, res.Body) res.Body.Close() if err != nil { return errutil.Errorf(err, "Unable to parse HTTP response for %s", TraceHttpReq(res.Request)) } return nil }
func logsBoomtownCommand(cmd *cobra.Command, args []string) { Debug("Debug message") err := errutil.Errorf(errors.New("Inner error message!"), "Error!") Panic(err, "Welcome to Boomtown") Debug("Never seen") }
func (a *basicUploadAdapter) DoTransfer(t *Transfer, cb TransferProgressCallback, authOkFunc func()) error { rel, ok := t.Object.Rel("upload") if !ok { return fmt.Errorf("No upload action for this object.") } req, err := httputil.NewHttpRequest("PUT", rel.Href, rel.Header) if err != nil { return err } if len(req.Header.Get("Content-Type")) == 0 { req.Header.Set("Content-Type", "application/octet-stream") } if req.Header.Get("Transfer-Encoding") == "chunked" { req.TransferEncoding = []string{"chunked"} } else { req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size, 10)) } req.ContentLength = t.Object.Size f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644) if err != nil { return errutil.Error(err) } defer f.Close() // Ensure progress callbacks made while uploading // Wrap callback to give name context ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error { if cb != nil { return cb(t.Name, totalSize, readSoFar, readSinceLast) } return nil } var reader io.Reader reader = &progress.CallbackReader{ C: ccb, TotalSize: t.Object.Size, Reader: f, } // Signal auth was ok on first read; this frees up other workers to start if authOkFunc != nil { reader = newStartCallbackReader(reader, func(*startCallbackReader) { authOkFunc() }) } req.Body = ioutil.NopCloser(reader) res, err := httputil.DoHttpRequest(req, true) if err != nil { return errutil.NewRetriableError(err) } httputil.LogTransfer("lfs.data.upload", res) // A status code of 403 likely means that an authentication token for the // upload has expired. This can be safely retried. if res.StatusCode == 403 { return errutil.NewRetriableError(err) } if res.StatusCode > 299 { return errutil.Errorf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode) } io.Copy(ioutil.Discard, res.Body) res.Body.Close() return api.VerifyUpload(t.Object) }
func readLocalFile(writer io.Writer, ptr *Pointer, mediafile string, workingfile string, cb progress.CopyCallback) error { reader, err := os.Open(mediafile) if err != nil { return errutil.Errorf(err, "Error opening media file.") } defer reader.Close() if ptr.Size == 0 { if stat, _ := os.Stat(mediafile); stat != nil { ptr.Size = stat.Size() } } if len(ptr.Extensions) > 0 { registeredExts := config.Config.Extensions() extensions := make(map[string]config.Extension) for _, ptrExt := range ptr.Extensions { ext, ok := registeredExts[ptrExt.Name] if !ok { err := fmt.Errorf("Extension '%s' is not configured.", ptrExt.Name) return errutil.Error(err) } ext.Priority = ptrExt.Priority extensions[ext.Name] = ext } exts, err := config.SortExtensions(extensions) if err != nil { return errutil.Error(err) } // pipe extensions in reverse order var extsR []config.Extension for i := range exts { ext := exts[len(exts)-1-i] extsR = append(extsR, ext) } request := &pipeRequest{"smudge", reader, workingfile, extsR} response, err := pipeExtensions(request) if err != nil { return errutil.Error(err) } actualExts := make(map[string]*pipeExtResult) for _, result := range response.results { actualExts[result.name] = result } // verify name, order, and oids oid := response.results[0].oidIn if ptr.Oid != oid { err = fmt.Errorf("Actual oid %s during smudge does not match expected %s", oid, ptr.Oid) return errutil.Error(err) } for _, expected := range ptr.Extensions { actual := actualExts[expected.Name] if actual.name != expected.Name { err = fmt.Errorf("Actual extension name '%s' does not match expected '%s'", actual.name, expected.Name) return errutil.Error(err) } if actual.oidOut != expected.Oid { err = fmt.Errorf("Actual oid %s for extension '%s' does not match expected %s", actual.oidOut, expected.Name, expected.Oid) return errutil.Error(err) } } // setup reader reader, err = os.Open(response.file.Name()) if err != nil { return errutil.Errorf(err, "Error opening smudged file: %s", err) } defer reader.Close() } _, err = tools.CopyWithCallback(writer, reader, ptr.Size, cb) if err != nil { return errutil.Errorf(err, "Error reading from media file: %s", err) } return nil }
// batchApiRoutine processes the queue of transfers using the batch endpoint, // making only one POST call for all objects. The results are then handed // off to the transfer workers. func (q *TransferQueue) batchApiRoutine() { var startProgress sync.Once transferAdapterNames := transfer.GetAdapterNames(q.direction) for { batch := q.batcher.Next() if batch == nil { break } tracerx.Printf("tq: sending batch of size %d", len(batch)) transfers := make([]*api.ObjectResource, 0, len(batch)) for _, i := range batch { t := i.(Transferable) transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()}) } if len(transfers) == 0 { continue } objs, adapterName, err := api.Batch(transfers, q.transferKind(), transferAdapterNames) if err != nil { if errutil.IsNotImplementedError(err) { git.Config.SetLocal("", "lfs.batch", "false") go q.legacyFallback(batch) return } if q.canRetry(err) { for _, t := range batch { q.retry(t.(Transferable)) } } else { q.errorc <- err } q.wait.Add(-len(transfers)) continue } q.useAdapter(adapterName) startProgress.Do(q.meter.Start) for _, o := range objs { if o.Error != nil { q.errorc <- errutil.Errorf(o.Error, "[%v] %v", o.Oid, o.Error.Message) q.Skip(o.Size) q.wait.Done() continue } if _, ok := o.Rel(q.transferKind()); ok { // This object needs to be transferred q.trMutex.Lock() transfer, ok := q.transferables[o.Oid] q.trMutex.Unlock() if ok { transfer.SetObject(o) q.meter.Add(transfer.Name()) q.addToAdapter(transfer) } else { q.Skip(transfer.Size()) q.wait.Done() } } else { q.Skip(o.Size) q.wait.Done() } } } }