func (f *dockerFetcher) fetchImageFrom(u *url.URL, latest bool) (string, error) { if !f.InsecureFlags.SkipImageCheck() { return "", fmt.Errorf("signature verification for docker images is not supported (try --insecure-options=image)") } if f.Debug { log.Printf("fetching image from %s", u.String()) } aciFile, err := f.fetch(u) if err != nil { return "", err } // At this point, the ACI file is removed, but it is kept // alive, because we have an fd to it opened. defer aciFile.Close() key, err := f.S.WriteACI(aciFile, latest) if err != nil { return "", err } // TODO(krnowak): Consider dropping the signature URL part // from store.Remote. It is not used anywhere and the data // stored here is useless. newRem := store.NewRemote(u.String(), ascURLFromImgURL(u).String()) newRem.BlobKey = key newRem.DownloadTime = time.Now() err = f.S.WriteRemote(newRem) if err != nil { return "", err } return key, nil }
func (f *nameFetcher) fetchImageFromSingleEndpoint(app *discovery.App, aciURL string, a *asc, latest bool) (string, error) { if f.Debug { log.Printf("fetching image from %s", aciURL) } aciFile, cd, err := f.fetch(app, aciURL, a) if err != nil { return "", err } defer aciFile.Close() key, err := f.S.WriteACI(aciFile, latest) if err != nil { return "", err } rem := store.NewRemote(aciURL, a.Location) rem.BlobKey = key rem.DownloadTime = time.Now() rem.ETag = cd.ETag rem.CacheMaxAge = cd.MaxAge err = f.S.WriteRemote(rem) if err != nil { return "", err } return key, nil }
func (f *fetcher) fetchImageFrom(aciURL, ascURL, scheme string, ascFile *os.File, latest bool) (string, error) { if f.insecureSkipVerify { if f.ks != nil { stderr("rkt: warning: TLS verification and signature verification has been disabled") } } else if scheme == "docker" { return "", fmt.Errorf("signature verification for docker images is not supported (try --insecure-skip-verify)") } var key string rem, ok, err := f.s.GetRemote(aciURL) if err == nil { key = rem.BlobKey } else { return "", err } if ok { stderr("rkt: found image in local store, skipping fetching from %s", aciURL) return key, nil } if scheme != "file" || f.debug { stderr("rkt: fetching image from %s", aciURL) } if f.local && scheme != "file" { return "", fmt.Errorf("url %s not available in local store", aciURL) } entity, aciFile, err := f.fetch(aciURL, ascURL, ascFile) if err != nil { return "", err } if scheme != "file" { defer os.Remove(aciFile.Name()) } if entity != nil && !f.insecureSkipVerify { stderr("rkt: signature verified: ") for _, v := range entity.Identities { stderr(" %s", v.Name) } } key, err = f.s.WriteACI(aciFile, latest) if err != nil { return "", err } if scheme != "file" { rem = store.NewRemote(aciURL, ascURL) rem.BlobKey = key err = f.s.WriteRemote(rem) if err != nil { return "", err } } return key, nil }
// GetHash fetches the URL, optionally verifies it against passed asc, // stores it in the store and returns the hash. func (f *httpFetcher) GetHash(u *url.URL, a *asc) (string, error) { ensureLogger(f.Debug) urlStr := u.String() if f.Debug { log.Printf("fetching image from %s", urlStr) } aciFile, cd, err := f.fetchURL(u, a, f.getETag()) if err != nil { return "", err } defer func() { maybeClose(aciFile) }() if key := f.maybeUseCached(cd); key != "" { // TODO(krnowak): that does not update the store with // the new CacheMaxAge and Download Time, so it will // query the server every time after initial // CacheMaxAge is exceeded return key, nil } key, err := f.S.WriteACI(aciFile, false) if err != nil { return "", err } // TODO(krnowak): What's the point of the second parameter? // The SigURL field in store.Remote seems to be completely // unused. newRem := store.NewRemote(urlStr, a.Location) newRem.BlobKey = key newRem.DownloadTime = time.Now() if cd != nil { newRem.ETag = cd.ETag newRem.CacheMaxAge = cd.MaxAge } err = f.S.WriteRemote(newRem) if err != nil { return "", err } return key, nil }
func TestDownloading(t *testing.T) { dir, err := ioutil.TempDir("", "download-image") if err != nil { t.Fatalf("error creating tempdir: %v", err) } defer os.RemoveAll(dir) imj := `{ "acKind": "ImageManifest", "acVersion": "0.5.5", "name": "example.com/test01" }` entries := []*aci.ACIEntry{ // An empty file { Contents: "hello", Header: &tar.Header{ Name: "rootfs/file01.txt", Size: 5, }, }, } aci, err := aci.NewACI(dir, imj, entries) if err != nil { t.Fatalf("error creating test tar: %v", err) } // Rewind the ACI if _, err := aci.Seek(0, 0); err != nil { t.Fatalf("unexpected error: %v", err) } body, err := ioutil.ReadAll(aci) if err != nil { t.Fatalf("unexpected error: %v", err) } noauthServer := &serverHandler{ body: body, t: t, auth: "none", } basicServer := &serverHandler{ body: body, t: t, auth: "basic", } oauthServer := &serverHandler{ body: body, t: t, auth: "bearer", } denyServer := &serverHandler{ body: body, t: t, auth: "deny", } noAuthTS := httptest.NewTLSServer(noauthServer) defer noAuthTS.Close() basicTS := httptest.NewTLSServer(basicServer) defer basicTS.Close() oauthTS := httptest.NewTLSServer(oauthServer) defer oauthTS.Close() denyAuthTS := httptest.NewServer(denyServer) noAuth := http.Header{} // YmFyOmJheg== is base64(bar:baz) basicAuth := http.Header{"Authorization": {"Basic YmFyOmJheg=="}} bearerAuth := http.Header{"Authorization": {"Bearer sometoken"}} urlToName := map[string]string{ noAuthTS.URL: "no auth", basicTS.URL: "basic", oauthTS.URL: "oauth", denyAuthTS.URL: "deny auth", } tests := []struct { ACIURL string hit bool options http.Header authFail bool }{ {noAuthTS.URL, false, noAuth, false}, {noAuthTS.URL, true, noAuth, false}, {noAuthTS.URL, true, bearerAuth, false}, {noAuthTS.URL, true, basicAuth, false}, {basicTS.URL, false, noAuth, true}, {basicTS.URL, false, bearerAuth, true}, {basicTS.URL, false, basicAuth, false}, {oauthTS.URL, false, noAuth, true}, {oauthTS.URL, false, basicAuth, true}, {oauthTS.URL, false, bearerAuth, false}, {denyAuthTS.URL, false, basicAuth, false}, {denyAuthTS.URL, true, bearerAuth, false}, {denyAuthTS.URL, true, noAuth, false}, } s, err := store.NewStore(dir) if err != nil { t.Fatalf("unexpected error %v", err) } for _, tt := range tests { _, ok, err := s.GetRemote(tt.ACIURL) if err != nil { t.Fatalf("unexpected err: %v", err) } if tt.hit == false && ok { t.Fatalf("expected miss got a hit") } if tt.hit == true && !ok { t.Fatalf("expected a hit got a miss") } parsed, err := url.Parse(tt.ACIURL) if err != nil { panic(fmt.Sprintf("Invalid url from test server: %s", tt.ACIURL)) } headers := map[string]config.Headerer{ parsed.Host: &testHeaderer{tt.options}, } ft := &fetcher{ imageActionData: imageActionData{ s: s, headers: headers, insecureSkipVerify: true, }, } _, aciFile, err := ft.fetch(tt.ACIURL, "", nil) if err == nil { defer os.Remove(aciFile.Name()) } if err != nil && !tt.authFail { t.Fatalf("expected download to succeed, it failed: %v (server: %q, headers: `%v`)", err, urlToName[tt.ACIURL], tt.options) } if err == nil && tt.authFail { t.Fatalf("expected download to fail, it succeeded (server: %q, headers: `%v`)", urlToName[tt.ACIURL], tt.options) } if err != nil { continue } key, err := s.WriteACI(aciFile, false) if err != nil { t.Fatalf("unexpected err: %v", err) } rem := store.NewRemote(tt.ACIURL, "") rem.BlobKey = key err = s.WriteRemote(rem) if err != nil { t.Fatalf("unexpected err: %v", err) } } s.Dump(false) }
// fetchImageFrom fetches an image from the aciURL. // If the aciURL is a file path (scheme == 'file'), then we bypass the on-disk store. // If the `--local` flag is provided, then we will only fetch from the on-disk store (unless aciURL is a file path). // If the label is 'latest', then we will bypass the on-disk store (unless '--local' is specified). // Otherwise if '--local' is false, aciURL is not a file path, and the label is not 'latest' or empty, we will first // try to fetch from the on-disk store, if not found, then fetch from the internet. func (f *fetcher) fetchImageFrom(appName string, aciURL, ascURL, scheme string, ascFile *os.File, latest bool) (string, error) { var rem *store.Remote if f.insecureSkipVerify { if f.ks != nil { stderr("rkt: warning: TLS verification and signature verification has been disabled") } } else if scheme == "docker" { return "", fmt.Errorf("signature verification for docker images is not supported (try --insecure-skip-verify)") } if (f.local && scheme != "file") || (scheme != "file" && !latest) { var err error ok := false rem, ok, err = f.s.GetRemote(aciURL) if err != nil { return "", err } if ok { if f.local { stderr("rkt: using image in local store for app %s", appName) return rem.BlobKey, nil } if useCached(rem.DownloadTime, rem.CacheMaxAge) { stderr("rkt: found image in local store, skipping fetching from %s", aciURL) return rem.BlobKey, nil } } if f.local { return "", fmt.Errorf("url %s not available in local store", aciURL) } } if scheme != "file" && f.debug { stderr("rkt: fetching image from %s", aciURL) } var etag string if rem != nil { etag = rem.ETag } entity, aciFile, cd, err := f.fetch(appName, aciURL, ascURL, ascFile, etag) if err != nil { return "", err } if cd != nil && cd.useCached { if rem != nil { return rem.BlobKey, nil } else { // should never happen panic("asked to use cached image but remote is nil") } } if scheme != "file" { defer os.Remove(aciFile.Name()) } if entity != nil && !f.insecureSkipVerify { stderr("rkt: signature verified:") for _, v := range entity.Identities { stderr(" %s", v.Name) } } key, err := f.s.WriteACI(aciFile, latest) if err != nil { return "", err } if scheme != "file" { rem := store.NewRemote(aciURL, ascURL) rem.BlobKey = key rem.DownloadTime = time.Now() if cd != nil { rem.ETag = cd.etag rem.CacheMaxAge = cd.maxAge } err = f.s.WriteRemote(rem) if err != nil { return "", err } } return key, nil }