// getIoProgressReader returns a reader that wraps the HTTP response
// body, so it prints a pretty progress bar when reading data from it.
func getIoProgressReader(label string, res *http.Response) io.Reader {
prefix := "Downloading " + label
fmtBytesSize := 18
barSize := int64(80 - len(prefix) - fmtBytesSize)
bar := ioprogress.DrawTextFormatBarForW(barSize, os.Stderr)
fmtfunc := func(progress, total int64) string {
// Content-Length is set to -1 when unknown.
if total == -1 {
return fmt.Sprintf(
"%s: %v of an unknown total size",
prefix,
ioprogress.ByteUnitStr(progress),
)
}
return fmt.Sprintf(
"%s: %s %s",
prefix,
bar(progress, total),
ioprogress.DrawTextFormatBytes(progress, total),
)
}
return &ioprogress.Reader{
Reader: res.Body,
Size: res.ContentLength,
DrawFunc: ioprogress.DrawTerminalf(os.Stderr, fmtfunc),
DrawInterval: time.Second,
}
}
// downloadHTTP retrieves url, creating a temp file using getTempFile
// http:// and https:// urls supported
func (f *fetcher) downloadHTTP(url, label string, out writeSyncer, etag string) (*cacheData, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
options := make(http.Header)
// Send credentials only over secure channel
if req.URL.Scheme == "https" {
if hostOpts, ok := f.headers[req.URL.Host]; ok {
options = hostOpts.Header()
}
}
for k, v := range options {
for _, e := range v {
req.Header.Add(k, e)
}
}
transport := http.DefaultTransport
if f.insecureSkipVerify {
transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
if etag != "" {
req.Header.Add("If-None-Match", etag)
}
client := &http.Client{Transport: transport}
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
cd := &cacheData{}
// TODO(jonboulle): handle http more robustly (redirects?)
switch res.StatusCode {
case http.StatusAccepted:
// If the server returns Status Accepted (HTTP 202), we should retry
// downloading the signature later.
return nil, errStatusAccepted
case http.StatusOK:
fallthrough
case http.StatusNotModified:
cd.etag = res.Header.Get("ETag")
cd.maxAge = getMaxAge(res.Header.Get("Cache-Control"))
cd.useCached = (res.StatusCode == http.StatusNotModified)
if cd.useCached {
return cd, nil
}
default:
return nil, fmt.Errorf("bad HTTP status code: %d", res.StatusCode)
}
prefix := "Downloading " + label
fmtBytesSize := 18
barSize := int64(80 - len(prefix) - fmtBytesSize)
bar := ioprogress.DrawTextFormatBar(barSize)
fmtfunc := func(progress, total int64) string {
// Content-Length is set to -1 when unknown.
if total == -1 {
return fmt.Sprintf(
"%s: %v of an unknown total size",
prefix,
ioprogress.ByteUnitStr(progress),
)
}
return fmt.Sprintf(
"%s: %s %s",
prefix,
bar(progress, total),
ioprogress.DrawTextFormatBytes(progress, total),
)
}
reader := &ioprogress.Reader{
Reader: res.Body,
Size: res.ContentLength,
DrawFunc: ioprogress.DrawTerminalf(os.Stdout, fmtfunc),
DrawInterval: time.Second,
}
if _, err := io.Copy(out, reader); err != nil {
return nil, fmt.Errorf("error copying %s: %v", label, err)
}
if err := out.Sync(); err != nil {
return nil, fmt.Errorf("error writing %s: %v", label, err)
}
return cd, nil
}
// downloadHTTP retrieves url, creating a temp file using getTempFile
// http:// and https:// urls supported
func (f *fetcher) downloadHTTP(url, label string, out writeSyncer, etag string) (*cacheData, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
transport := http.DefaultTransport
if f.insecureFlags.SkipTlsCheck() {
transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
var etagFilePath string
var amountAlreadyHere int64
byteRangeSupported := false
client := &http.Client{Transport: transport}
f.setHTTPHeaders(req, etag)
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return fmt.Errorf("too many redirects")
}
f.setHTTPHeaders(req, etag)
if amountAlreadyHere > 0 && byteRangeSupported {
req.Header.Add("Range", fmt.Sprintf("bytes=%d-", amountAlreadyHere))
}
return nil
}
if f, ok := out.(*os.File); ok {
finfo, err := f.Stat()
if err != nil {
return nil, err
}
etagFilePath = f.Name() + ".etag"
amountAlreadyHere = finfo.Size()
if amountAlreadyHere > 0 {
// There's already some data in the file we're going to write the
// aci to. We're going to send an HTTP HEAD request to see if the
// server accepts Range requests. If so, just request the
// remaining data. If not, seek to the beginning of the file.
resp, err := client.Head(url)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
stderr("bad HTTP status code from HEAD request: %d",
resp.StatusCode)
} else {
var modOK bool
var etagOK bool
lastModified := resp.Header.Get("Last-Modified")
etag := resp.Header.Get("ETag")
acceptRanges, rangeOK := resp.Header["Accept-Ranges"]
if lastModified != "" {
t, err := time.Parse("Mon, 02 Jan 2006 15:04:05 MST",
lastModified)
if err == nil && t.Before(finfo.ModTime()) {
modOK = true
}
}
if etag != "" {
savedEtag, err := ioutil.ReadFile(etagFilePath)
if err == nil && string(savedEtag) == etag {
etagOK = true
}
}
if rangeOK && (modOK || etagOK) {
for _, rng := range acceptRanges {
if rng == "bytes" {
byteRangeSupported = true
req.Header.Add("Range",
fmt.Sprintf("bytes=%d-", amountAlreadyHere))
}
}
}
if !byteRangeSupported {
if rangeOK {
stderr("Can't use cached partial download, resource updated.")
} else {
stderr("Can't use cached partial download, range request unsupported.")
}
amountAlreadyHere = 0
_, err := f.Seek(0, os.SEEK_SET)
if err != nil {
return nil, err
}
err = f.Truncate(0)
if err != nil {
return nil, err
}
}
}
}
}
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
cd := &cacheData{}
// TODO(jonboulle): handle http more robustly (redirects?)
switch res.StatusCode {
case http.StatusAccepted:
// If the server returns Status Accepted (HTTP 202), we should retry
// downloading the signature later.
return nil, errStatusAccepted
case http.StatusOK, http.StatusPartialContent:
fallthrough
case http.StatusNotModified:
cd.etag = res.Header.Get("ETag")
cd.maxAge = getMaxAge(res.Header.Get("Cache-Control"))
cd.useCached = (res.StatusCode == http.StatusNotModified)
if cd.useCached {
return cd, nil
}
case http.StatusRequestedRangeNotSatisfiable:
if file, ok := out.(*os.File); ok {
finfo, err := file.Stat()
if err != nil {
return nil, err
}
if finfo.Size() != 0 {
_, err = file.Seek(0, os.SEEK_SET)
if err != nil {
return nil, err
}
err = file.Truncate(0)
if err != nil {
return nil, err
}
os.Remove(etagFilePath)
return f.downloadHTTP(url, label, out, etag)
}
}
fallthrough
default:
return nil, fmt.Errorf("bad HTTP status code: %d", res.StatusCode)
}
_ = ioutil.WriteFile(etagFilePath, []byte(res.Header.Get("ETag")), 0644)
prefix := "Downloading " + label
fmtBytesSize := 18
barSize := int64(80 - len(prefix) - fmtBytesSize)
bar := ioprogress.DrawTextFormatBarForW(barSize, os.Stderr)
fmtfunc := func(progress, total int64) string {
// Content-Length is set to -1 when unknown.
if total == -1 {
return fmt.Sprintf(
"%s: %v of an unknown total size",
prefix,
ioprogress.ByteUnitStr(progress),
)
}
return fmt.Sprintf(
"%s: %s %s",
prefix,
bar(progress, total),
ioprogress.DrawTextFormatBytes(progress, total),
)
}
reader := &ioprogress.Reader{
Reader: res.Body,
Size: res.ContentLength,
DrawFunc: ioprogress.DrawTerminalf(os.Stderr, fmtfunc),
DrawInterval: time.Second,
}
if _, err := io.Copy(out, reader); err != nil {
return nil, fmt.Errorf("error copying %s: %v", label, err)
}
if err := out.Sync(); err != nil {
return nil, fmt.Errorf("error writing %s: %v", label, err)
}
os.Remove(etagFilePath)
return cd, nil
}
// downloadHTTP retrieves url, creating a temp file using getTempFile
// http:// and https:// urls supported
func (f *fetcher) downloadHTTP(url, label string, out writeSyncer, etag string) (*cacheData, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
transport := http.DefaultTransport
if f.insecureSkipVerify {
transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
client := &http.Client{Transport: transport}
f.setHTTPHeaders(req, etag)
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return fmt.Errorf("too many redirects")
}
f.setHTTPHeaders(req, etag)
return nil
}
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
cd := &cacheData{}
// TODO(jonboulle): handle http more robustly (redirects?)
switch res.StatusCode {
case http.StatusAccepted:
// If the server returns Status Accepted (HTTP 202), we should retry
// downloading the signature later.
return nil, errStatusAccepted
case http.StatusOK:
fallthrough
case http.StatusNotModified:
cd.etag = res.Header.Get("ETag")
cd.maxAge = getMaxAge(res.Header.Get("Cache-Control"))
cd.useCached = (res.StatusCode == http.StatusNotModified)
if cd.useCached {
return cd, nil
}
default:
return nil, fmt.Errorf("bad HTTP status code: %d", res.StatusCode)
}
prefix := "Downloading " + label
fmtBytesSize := 18
barSize := int64(80 - len(prefix) - fmtBytesSize)
bar := ioprogress.DrawTextFormatBarForW(barSize, os.Stderr)
fmtfunc := func(progress, total int64) string {
// Content-Length is set to -1 when unknown.
if total == -1 {
return fmt.Sprintf(
"%s: %v of an unknown total size",
prefix,
ioprogress.ByteUnitStr(progress),
)
}
return fmt.Sprintf(
"%s: %s %s",
prefix,
bar(progress, total),
ioprogress.DrawTextFormatBytes(progress, total),
)
}
reader := &ioprogress.Reader{
Reader: res.Body,
Size: res.ContentLength,
DrawFunc: ioprogress.DrawTerminalf(os.Stderr, fmtfunc),
DrawInterval: time.Second,
}
if _, err := io.Copy(out, reader); err != nil {
return nil, fmt.Errorf("error copying %s: %v", label, err)
}
if err := out.Sync(); err != nil {
return nil, fmt.Errorf("error writing %s: %v", label, err)
}
return cd, nil
}