Example #1
0
File: io.go Project: matomesc/rkt
// getIoProgressReader returns a reader that wraps the HTTP response
// body, so it prints a pretty progress bar when reading data from it.
func getIoProgressReader(label string, res *http.Response) io.Reader {
	prefix := "Downloading " + label
	fmtBytesSize := 18
	barSize := int64(80 - len(prefix) - fmtBytesSize)
	bar := ioprogress.DrawTextFormatBarForW(barSize, os.Stderr)
	fmtfunc := func(progress, total int64) string {
		// Content-Length is set to -1 when unknown.
		if total == -1 {
			return fmt.Sprintf(
				"%s: %v of an unknown total size",
				prefix,
				ioprogress.ByteUnitStr(progress),
			)
		}
		return fmt.Sprintf(
			"%s: %s %s",
			prefix,
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total),
		)
	}
	return &ioprogress.Reader{
		Reader:       res.Body,
		Size:         res.ContentLength,
		DrawFunc:     ioprogress.DrawTerminalf(os.Stderr, fmtfunc),
		DrawInterval: time.Second,
	}
}
Example #2
0
func (rb *RepositoryBackend) getLayerV1(imgID, registry string, repoData *RepoData, imgSize int64, tmpDir string) (*os.File, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", rb.protocol()+path.Join(registry, "images", imgID, "layer"), nil)
	if err != nil {
		return nil, err
	}

	setAuthTokenV1(req, repoData.Tokens)
	setCookieV1(req, repoData.Cookie)

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		res.Body.Close()
		return nil, fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL)
	}

	// if we didn't receive the size via X-Docker-Size when we retrieved the
	// layer's json, try Content-Length
	if imgSize == -1 {
		if hdr := res.Header.Get("Content-Length"); hdr != "" {
			imgSize, err = strconv.ParseInt(hdr, 10, 64)
			if err != nil {
				return nil, err
			}
		}
	}

	prefix := "Downloading " + imgID[:12]
	fmtBytesSize := 18
	barSize := int64(80 - len(prefix) - fmtBytesSize)
	bar := ioprogress.DrawTextFormatBarForW(barSize, os.Stderr)
	fmtfunc := func(progress, total int64) string {
		return fmt.Sprintf(
			"%s: %s %s",
			prefix,
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total),
		)
	}

	progressReader := &ioprogress.Reader{
		Reader:       res.Body,
		Size:         imgSize,
		DrawFunc:     ioprogress.DrawTerminalf(os.Stderr, fmtfunc),
		DrawInterval: 500 * time.Millisecond,
	}

	layerFile, err := ioutil.TempFile(tmpDir, "dockerlayer-")
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(layerFile, progressReader)
	if err != nil {
		return nil, err
	}

	if err := layerFile.Sync(); err != nil {
		return nil, err
	}

	return layerFile, nil
}
Example #3
0
// downloadHTTP retrieves url, creating a temp file using getTempFile
// http:// and https:// urls supported
func (f *fetcher) downloadHTTP(url, label string, out writeSyncer, etag string) (*cacheData, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	options := make(http.Header)
	// Send credentials only over secure channel
	if req.URL.Scheme == "https" {
		if hostOpts, ok := f.headers[req.URL.Host]; ok {
			options = hostOpts.Header()
		}
	}
	for k, v := range options {
		for _, e := range v {
			req.Header.Add(k, e)
		}
	}
	transport := http.DefaultTransport
	if f.insecureSkipVerify {
		transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}
	if etag != "" {
		req.Header.Add("If-None-Match", etag)
	}

	client := &http.Client{Transport: transport}
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	cd := &cacheData{}
	// TODO(jonboulle): handle http more robustly (redirects?)
	switch res.StatusCode {
	case http.StatusAccepted:
		// If the server returns Status Accepted (HTTP 202), we should retry
		// downloading the signature later.
		return nil, errStatusAccepted
	case http.StatusOK:
		fallthrough
	case http.StatusNotModified:
		cd.etag = res.Header.Get("ETag")
		cd.maxAge = getMaxAge(res.Header.Get("Cache-Control"))
		cd.useCached = (res.StatusCode == http.StatusNotModified)
		if cd.useCached {
			return cd, nil
		}
	default:
		return nil, fmt.Errorf("bad HTTP status code: %d", res.StatusCode)
	}

	prefix := "Downloading " + label
	fmtBytesSize := 18
	barSize := int64(80 - len(prefix) - fmtBytesSize)
	bar := ioprogress.DrawTextFormatBar(barSize)
	fmtfunc := func(progress, total int64) string {
		// Content-Length is set to -1 when unknown.
		if total == -1 {
			return fmt.Sprintf(
				"%s: %v of an unknown total size",
				prefix,
				ioprogress.ByteUnitStr(progress),
			)
		}
		return fmt.Sprintf(
			"%s: %s %s",
			prefix,
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total),
		)
	}

	reader := &ioprogress.Reader{
		Reader:       res.Body,
		Size:         res.ContentLength,
		DrawFunc:     ioprogress.DrawTerminalf(os.Stdout, fmtfunc),
		DrawInterval: time.Second,
	}

	if _, err := io.Copy(out, reader); err != nil {
		return nil, fmt.Errorf("error copying %s: %v", label, err)
	}

	if err := out.Sync(); err != nil {
		return nil, fmt.Errorf("error writing %s: %v", label, err)
	}

	return cd, nil
}
Example #4
0
// downloadHTTP retrieves url, creating a temp file using getTempFile
// http:// and https:// urls supported
func (f *fetcher) downloadHTTP(url, label string, out writeSyncer, etag string) (*cacheData, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	transport := http.DefaultTransport
	if f.insecureFlags.SkipTlsCheck() {
		transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}

	var etagFilePath string
	var amountAlreadyHere int64
	byteRangeSupported := false

	client := &http.Client{Transport: transport}
	f.setHTTPHeaders(req, etag)

	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) >= 10 {
			return fmt.Errorf("too many redirects")
		}
		f.setHTTPHeaders(req, etag)
		if amountAlreadyHere > 0 && byteRangeSupported {
			req.Header.Add("Range", fmt.Sprintf("bytes=%d-", amountAlreadyHere))
		}
		return nil
	}

	if f, ok := out.(*os.File); ok {
		finfo, err := f.Stat()
		if err != nil {
			return nil, err
		}

		etagFilePath = f.Name() + ".etag"

		amountAlreadyHere = finfo.Size()
		if amountAlreadyHere > 0 {
			// There's already some data in the file we're going to write the
			// aci to. We're going to send an HTTP HEAD request to see if the
			// server accepts Range requests. If so, just request the
			// remaining data. If not, seek to the beginning of the file.
			resp, err := client.Head(url)
			if err != nil {
				return nil, err
			}
			if resp.StatusCode != http.StatusOK {
				stderr("bad HTTP status code from HEAD request: %d",
					resp.StatusCode)
			} else {
				var modOK bool
				var etagOK bool
				lastModified := resp.Header.Get("Last-Modified")
				etag := resp.Header.Get("ETag")
				acceptRanges, rangeOK := resp.Header["Accept-Ranges"]
				if lastModified != "" {
					t, err := time.Parse("Mon, 02 Jan 2006 15:04:05 MST",
						lastModified)
					if err == nil && t.Before(finfo.ModTime()) {
						modOK = true
					}
				}
				if etag != "" {
					savedEtag, err := ioutil.ReadFile(etagFilePath)
					if err == nil && string(savedEtag) == etag {
						etagOK = true
					}
				}
				if rangeOK && (modOK || etagOK) {
					for _, rng := range acceptRanges {
						if rng == "bytes" {
							byteRangeSupported = true
							req.Header.Add("Range",
								fmt.Sprintf("bytes=%d-", amountAlreadyHere))
						}
					}
				}
				if !byteRangeSupported {
					if rangeOK {
						stderr("Can't use cached partial download, resource updated.")
					} else {
						stderr("Can't use cached partial download, range request unsupported.")
					}
					amountAlreadyHere = 0
					_, err := f.Seek(0, os.SEEK_SET)
					if err != nil {
						return nil, err
					}
					err = f.Truncate(0)
					if err != nil {
						return nil, err
					}
				}
			}
		}
	}

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	cd := &cacheData{}
	// TODO(jonboulle): handle http more robustly (redirects?)
	switch res.StatusCode {
	case http.StatusAccepted:
		// If the server returns Status Accepted (HTTP 202), we should retry
		// downloading the signature later.
		return nil, errStatusAccepted
	case http.StatusOK, http.StatusPartialContent:
		fallthrough
	case http.StatusNotModified:
		cd.etag = res.Header.Get("ETag")
		cd.maxAge = getMaxAge(res.Header.Get("Cache-Control"))
		cd.useCached = (res.StatusCode == http.StatusNotModified)
		if cd.useCached {
			return cd, nil
		}
	case http.StatusRequestedRangeNotSatisfiable:
		if file, ok := out.(*os.File); ok {
			finfo, err := file.Stat()
			if err != nil {
				return nil, err
			}
			if finfo.Size() != 0 {
				_, err = file.Seek(0, os.SEEK_SET)
				if err != nil {
					return nil, err
				}
				err = file.Truncate(0)
				if err != nil {
					return nil, err
				}
				os.Remove(etagFilePath)
				return f.downloadHTTP(url, label, out, etag)
			}
		}
		fallthrough
	default:
		return nil, fmt.Errorf("bad HTTP status code: %d", res.StatusCode)
	}

	_ = ioutil.WriteFile(etagFilePath, []byte(res.Header.Get("ETag")), 0644)

	prefix := "Downloading " + label
	fmtBytesSize := 18
	barSize := int64(80 - len(prefix) - fmtBytesSize)
	bar := ioprogress.DrawTextFormatBarForW(barSize, os.Stderr)
	fmtfunc := func(progress, total int64) string {
		// Content-Length is set to -1 when unknown.
		if total == -1 {
			return fmt.Sprintf(
				"%s: %v of an unknown total size",
				prefix,
				ioprogress.ByteUnitStr(progress),
			)
		}
		return fmt.Sprintf(
			"%s: %s %s",
			prefix,
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total),
		)
	}

	reader := &ioprogress.Reader{
		Reader:       res.Body,
		Size:         res.ContentLength,
		DrawFunc:     ioprogress.DrawTerminalf(os.Stderr, fmtfunc),
		DrawInterval: time.Second,
	}

	if _, err := io.Copy(out, reader); err != nil {
		return nil, fmt.Errorf("error copying %s: %v", label, err)
	}

	if err := out.Sync(); err != nil {
		return nil, fmt.Errorf("error writing %s: %v", label, err)
	}

	os.Remove(etagFilePath)

	return cd, nil
}
Example #5
0
// downloadHTTP retrieves url, creating a temp file using getTempFile
// http:// and https:// urls supported
func (f *fetcher) downloadHTTP(url, label string, out writeSyncer, etag string) (*cacheData, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	transport := http.DefaultTransport
	if f.insecureSkipVerify {
		transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}

	client := &http.Client{Transport: transport}
	f.setHTTPHeaders(req, etag)

	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) >= 10 {
			return fmt.Errorf("too many redirects")
		}
		f.setHTTPHeaders(req, etag)
		return nil
	}

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	cd := &cacheData{}
	// TODO(jonboulle): handle http more robustly (redirects?)
	switch res.StatusCode {
	case http.StatusAccepted:
		// If the server returns Status Accepted (HTTP 202), we should retry
		// downloading the signature later.
		return nil, errStatusAccepted
	case http.StatusOK:
		fallthrough
	case http.StatusNotModified:
		cd.etag = res.Header.Get("ETag")
		cd.maxAge = getMaxAge(res.Header.Get("Cache-Control"))
		cd.useCached = (res.StatusCode == http.StatusNotModified)
		if cd.useCached {
			return cd, nil
		}
	default:
		return nil, fmt.Errorf("bad HTTP status code: %d", res.StatusCode)
	}

	prefix := "Downloading " + label
	fmtBytesSize := 18
	barSize := int64(80 - len(prefix) - fmtBytesSize)
	bar := ioprogress.DrawTextFormatBarForW(barSize, os.Stderr)
	fmtfunc := func(progress, total int64) string {
		// Content-Length is set to -1 when unknown.
		if total == -1 {
			return fmt.Sprintf(
				"%s: %v of an unknown total size",
				prefix,
				ioprogress.ByteUnitStr(progress),
			)
		}
		return fmt.Sprintf(
			"%s: %s %s",
			prefix,
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total),
		)
	}

	reader := &ioprogress.Reader{
		Reader:       res.Body,
		Size:         res.ContentLength,
		DrawFunc:     ioprogress.DrawTerminalf(os.Stderr, fmtfunc),
		DrawInterval: time.Second,
	}

	if _, err := io.Copy(out, reader); err != nil {
		return nil, fmt.Errorf("error copying %s: %v", label, err)
	}

	if err := out.Sync(); err != nil {
		return nil, fmt.Errorf("error writing %s: %v", label, err)
	}

	return cd, nil
}
Example #6
0
func (rb *RepositoryBackend) getLayerV2(layerID string, dockerURL *types.ParsedDockerURL, tmpDir string) (*os.File, error) {
	url := rb.protocol() + path.Join(dockerURL.IndexURL, "v2", dockerURL.ImageName, "blobs", layerID)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	res, err := rb.makeRequest(req, dockerURL.ImageName)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		return nil, fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL)
	}

	var in io.Reader
	in = res.Body

	if hdr := res.Header.Get("Content-Length"); hdr != "" {
		imgSize, err := strconv.ParseInt(hdr, 10, 64)
		if err != nil {
			return nil, err
		}

		prefix := "Downloading " + layerID[:18]
		fmtBytesSize := 18
		barSize := int64(80 - len(prefix) - fmtBytesSize)
		bar := ioprogress.DrawTextFormatBarForW(barSize, os.Stderr)
		fmtfunc := func(progress, total int64) string {
			return fmt.Sprintf(
				"%s: %s %s",
				prefix,
				bar(progress, total),
				ioprogress.DrawTextFormatBytes(progress, total),
			)
		}
		in = &ioprogress.Reader{
			Reader:       res.Body,
			Size:         imgSize,
			DrawFunc:     ioprogress.DrawTerminalf(os.Stderr, fmtfunc),
			DrawInterval: 500 * time.Millisecond,
		}
	}

	layerFile, err := ioutil.TempFile(tmpDir, "dockerlayer-")
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(layerFile, in)
	if err != nil {
		return nil, err
	}

	if err := layerFile.Sync(); err != nil {
		return nil, err
	}

	return layerFile, nil
}