Exemple #1
0
// downloadHTTP retrieves url, creating a temp file using getTempFile
// http:// and https:// urls supported
func (f *fetcher) downloadHTTP(url, label string, out writeSyncer, etag string) (*cacheData, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	options := make(http.Header)
	// Send credentials only over secure channel
	if req.URL.Scheme == "https" {
		if hostOpts, ok := f.headers[req.URL.Host]; ok {
			options = hostOpts.Header()
		}
	}
	for k, v := range options {
		for _, e := range v {
			req.Header.Add(k, e)
		}
	}
	transport := http.DefaultTransport
	if f.insecureSkipVerify {
		transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}
	if etag != "" {
		req.Header.Add("If-None-Match", etag)
	}

	client := &http.Client{Transport: transport}
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	cd := &cacheData{}
	// TODO(jonboulle): handle http more robustly (redirects?)
	switch res.StatusCode {
	case http.StatusAccepted:
		// If the server returns Status Accepted (HTTP 202), we should retry
		// downloading the signature later.
		return nil, errStatusAccepted
	case http.StatusOK:
		fallthrough
	case http.StatusNotModified:
		cd.etag = res.Header.Get("ETag")
		cd.maxAge = getMaxAge(res.Header.Get("Cache-Control"))
		cd.useCached = (res.StatusCode == http.StatusNotModified)
		if cd.useCached {
			return cd, nil
		}
	default:
		return nil, fmt.Errorf("bad HTTP status code: %d", res.StatusCode)
	}

	prefix := "Downloading " + label
	fmtBytesSize := 18
	barSize := int64(80 - len(prefix) - fmtBytesSize)
	bar := ioprogress.DrawTextFormatBar(barSize)
	fmtfunc := func(progress, total int64) string {
		// Content-Length is set to -1 when unknown.
		if total == -1 {
			return fmt.Sprintf(
				"%s: %v of an unknown total size",
				prefix,
				ioprogress.ByteUnitStr(progress),
			)
		}
		return fmt.Sprintf(
			"%s: %s %s",
			prefix,
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total),
		)
	}

	reader := &ioprogress.Reader{
		Reader:       res.Body,
		Size:         res.ContentLength,
		DrawFunc:     ioprogress.DrawTerminalf(os.Stdout, fmtfunc),
		DrawInterval: time.Second,
	}

	if _, err := io.Copy(out, reader); err != nil {
		return nil, fmt.Errorf("error copying %s: %v", label, err)
	}

	if err := out.Sync(); err != nil {
		return nil, fmt.Errorf("error writing %s: %v", label, err)
	}

	return cd, nil
}
Exemple #2
0
// downloadHTTP retrieves url, creating a temp file using getTempFile
// http:// and https:// urls supported
func (f *fetcher) downloadHTTP(url, label string, out writeSyncer, etag string) (*cacheData, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	transport := http.DefaultTransport
	if f.insecureSkipVerify {
		transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}

	client := &http.Client{Transport: transport}
	f.setHTTPHeaders(req, etag)

	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) >= 10 {
			return fmt.Errorf("too many redirects")
		}
		f.setHTTPHeaders(req, etag)
		return nil
	}

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	cd := &cacheData{}
	// TODO(jonboulle): handle http more robustly (redirects?)
	switch res.StatusCode {
	case http.StatusAccepted:
		// If the server returns Status Accepted (HTTP 202), we should retry
		// downloading the signature later.
		return nil, errStatusAccepted
	case http.StatusOK:
		fallthrough
	case http.StatusNotModified:
		cd.etag = res.Header.Get("ETag")
		cd.maxAge = getMaxAge(res.Header.Get("Cache-Control"))
		cd.useCached = (res.StatusCode == http.StatusNotModified)
		if cd.useCached {
			return cd, nil
		}
	default:
		return nil, fmt.Errorf("bad HTTP status code: %d", res.StatusCode)
	}

	prefix := "Downloading " + label
	fmtBytesSize := 18
	barSize := int64(80 - len(prefix) - fmtBytesSize)
	bar := ioprogress.DrawTextFormatBar(barSize)
	fmtfunc := func(progress, total int64) string {
		// Content-Length is set to -1 when unknown.
		if total == -1 {
			return fmt.Sprintf(
				"%s: %v of an unknown total size",
				prefix,
				ioprogress.ByteUnitStr(progress),
			)
		}
		return fmt.Sprintf(
			"%s: %s %s",
			prefix,
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total),
		)
	}

	reader := &ioprogress.Reader{
		Reader:       res.Body,
		Size:         res.ContentLength,
		DrawFunc:     ioprogress.DrawTerminalf(os.Stderr, fmtfunc),
		DrawInterval: time.Second,
	}

	if _, err := io.Copy(out, reader); err != nil {
		return nil, fmt.Errorf("error copying %s: %v", label, err)
	}

	if err := out.Sync(); err != nil {
		return nil, fmt.Errorf("error writing %s: %v", label, err)
	}

	return cd, nil
}