// LearnRegistryURL returns the registry URL after making sure that it responds to queries
func LearnRegistryURL(options Options) (string, error) {
defer trace.End(trace.Begin(options.Registry))
req := func(schema string) (string, error) {
registry := fmt.Sprintf("%s://%s/v2/", schema, options.Registry)
url, err := url.Parse(registry)
if err != nil {
return "", err
}
log.Debugf("URL: %s", url)
fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
Timeout: options.Timeout,
Username: options.Username,
Password: options.Password,
InsecureSkipVerify: options.InsecureSkipVerify,
})
headers, err := fetcher.Head(url)
if err != nil {
return "", err
}
// v2 API requires this check
if headers.Get("Docker-Distribution-API-Version") != "registry/2.0" {
return "", fmt.Errorf("Missing Docker-Distribution-API-Version header")
}
return registry, nil
}
// first try https
log.Debugf("Trying https scheme")
registry, err := req("https")
if err != nil && options.InsecureAllowHTTP {
// fallback to http if it's allowed
log.Debugf("Falling back to http scheme")
registry, err = req("http")
}
return registry, err
}
// FetchToken fetches the OAuth token from OAuth endpoint
func FetchToken(ctx context.Context, options Options, url *url.URL, progressOutput progress.Output) (*urlfetcher.Token, error) {
defer trace.End(trace.Begin(url.String()))
log.Debugf("URL: %s", url)
fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
Timeout: options.Timeout,
Username: options.Username,
Password: options.Password,
InsecureSkipVerify: options.InsecureSkipVerify,
})
token, err := fetcher.FetchAuthToken(url)
if err != nil {
err := fmt.Errorf("FetchToken (%s) failed: %s", url, err)
log.Error(err)
return nil, err
}
return token, nil
}
// LearnAuthURL returns the URL of the OAuth endpoint
func LearnAuthURL(options Options) (*url.URL, error) {
defer trace.End(trace.Begin(options.Image + "/" + options.Tag))
url, err := url.Parse(options.Registry)
if err != nil {
return nil, err
}
url.Path = path.Join(url.Path, options.Image, "manifests", options.Tag)
log.Debugf("URL: %s", url)
fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
Timeout: options.Timeout,
Username: options.Username,
Password: options.Password,
InsecureSkipVerify: options.InsecureSkipVerify,
})
// We expect docker registry to return a 401 to us - with a WWW-Authenticate header
// We parse that header and learn the OAuth endpoint to fetch OAuth token.
hdr, err := fetcher.Head(url)
if err == nil && fetcher.IsStatusUnauthorized() {
return fetcher.ExtractOAuthURL(hdr.Get("www-authenticate"), url)
}
// Private registry returned the manifest directly as auth option is optional.
// https://github.com/docker/distribution/blob/master/docs/configuration.md#auth
if err == nil && options.Registry != DefaultDockerURL && fetcher.IsStatusOK() {
log.Debugf("%s does not support OAuth", url)
return nil, nil
}
// Do we even have the image on that registry
if err != nil && fetcher.IsStatusNotFound() {
err = fmt.Errorf("image not found")
return nil, urlfetcher.ImageNotFoundError{Err: err}
}
return nil, fmt.Errorf("%s returned an unexpected response: %s", url, err)
}
// FetchImageManifest fetches the image manifest file
func FetchImageManifest(ctx context.Context, options Options, progressOutput progress.Output) (*Manifest, error) {
defer trace.End(trace.Begin(options.Image + "/" + options.Tag))
url, err := url.Parse(options.Registry)
if err != nil {
return nil, err
}
url.Path = path.Join(url.Path, options.Image, "manifests", options.Tag)
log.Debugf("URL: %s", url)
fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
Timeout: options.Timeout,
Username: options.Username,
Password: options.Password,
Token: options.Token,
InsecureSkipVerify: options.InsecureSkipVerify,
})
manifestFileName, err := fetcher.Fetch(ctx, url, true, progressOutput)
if err != nil {
return nil, err
}
// Cleanup function for the error case
defer func() {
if err != nil {
os.Remove(manifestFileName)
}
}()
// Read the entire file into []byte for json.Unmarshal
content, err := ioutil.ReadFile(manifestFileName)
if err != nil {
return nil, err
}
manifest := &Manifest{}
err = json.Unmarshal(content, manifest)
if err != nil {
return nil, err
}
if manifest.Name != options.Image {
return nil, fmt.Errorf("name doesn't match what was requested, expected: %s, downloaded: %s", options.Image, manifest.Name)
}
if manifest.Tag != options.Tag {
return nil, fmt.Errorf("tag doesn't match what was requested, expected: %s, downloaded: %s", options.Tag, manifest.Tag)
}
digest, err := getManifestDigest(content)
if err != nil {
return nil, err
}
manifest.Digest = digest
// Ensure the parent directory exists
destination := DestinationDirectory(options)
err = os.MkdirAll(destination, 0755) /* #nosec */
if err != nil {
return nil, err
}
// Move(rename) the temporary file to its final destination
err = os.Rename(string(manifestFileName), path.Join(destination, "manifest.json"))
if err != nil {
return nil, err
}
return manifest, nil
}
// FetchImageBlob fetches the image blob
func FetchImageBlob(ctx context.Context, options Options, image *ImageWithMeta, progressOutput progress.Output) (string, error) {
defer trace.End(trace.Begin(options.Image + "/" + image.Layer.BlobSum))
id := image.ID
layer := image.Layer.BlobSum
meta := image.Meta
diffID := ""
url, err := url.Parse(options.Registry)
if err != nil {
return diffID, err
}
url.Path = path.Join(url.Path, options.Image, "blobs", layer)
log.Debugf("URL: %s\n ", url)
fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
Timeout: options.Timeout,
Username: options.Username,
Password: options.Password,
Token: options.Token,
InsecureSkipVerify: options.InsecureSkipVerify,
})
// ctx
ctx, cancel := context.WithTimeout(ctx, options.Timeout)
defer cancel()
imageFileName, err := fetcher.Fetch(ctx, url, true, progressOutput, image.String())
if err != nil {
return diffID, err
}
// Cleanup function for the error case
defer func() {
if err != nil {
os.Remove(imageFileName)
}
}()
// Open the file so that we can use it as a io.Reader for sha256 calculation
imageFile, err := os.Open(string(imageFileName))
if err != nil {
return diffID, err
}
defer imageFile.Close()
// blobSum is the sha of the compressed layer
blobSum := sha256.New()
// diffIDSum is the sha of the uncompressed layer
diffIDSum := sha256.New()
// blobTr is an io.TeeReader that writes bytes to blobSum that it reads from imageFile
// see https://golang.org/pkg/io/#TeeReader
blobTr := io.TeeReader(imageFile, blobSum)
progress.Update(progressOutput, image.String(), "Verifying Checksum")
decompressedTar, err := archive.DecompressStream(blobTr)
if err != nil {
return diffID, err
}
// Copy bytes from decompressed layer into diffIDSum to calculate diffID
_, cerr := io.Copy(diffIDSum, decompressedTar)
if cerr != nil {
return diffID, cerr
}
bs := fmt.Sprintf("sha256:%x", blobSum.Sum(nil))
if bs != layer {
return diffID, fmt.Errorf("Failed to validate layer checksum. Expected %s got %s", layer, bs)
}
diffID = fmt.Sprintf("sha256:%x", diffIDSum.Sum(nil))
// this isn't an empty layer, so we need to calculate the size
if diffID != string(DigestSHA256EmptyTar) {
var layerSize int64
// seek to the beginning of the file
imageFile.Seek(0, 0)
// recreate the decompressed tar Reader
decompressedTar, err := archive.DecompressStream(imageFile)
if err != nil {
return "", err
}
// get a tar reader for access to the files in the archive
tr := tar.NewReader(decompressedTar)
// iterate through tar headers to get file sizes
for {
tarHeader, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return "", err
}
layerSize += tarHeader.Size
}
image.Size = layerSize
}
log.Infof("diffID for layer %s: %s", id, diffID)
// Ensure the parent directory exists
destination := path.Join(DestinationDirectory(options), id)
err = os.MkdirAll(destination, 0755) /* #nosec */
if err != nil {
return diffID, err
}
// Move(rename) the temporary file to its final destination
err = os.Rename(string(imageFileName), path.Join(destination, id+".tar"))
if err != nil {
return diffID, err
}
// Dump the history next to it
err = ioutil.WriteFile(path.Join(destination, id+".json"), []byte(meta), 0644)
if err != nil {
return diffID, err
}
progress.Update(progressOutput, image.String(), "Download complete")
return diffID, nil
}
func (s *System) AuthenticateToRegistry(ctx context.Context, authConfig *types.AuthConfig) (string, string, error) {
defer trace.End(trace.Begin(""))
fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
Timeout: loginTimeout,
Username: authConfig.Username,
Password: authConfig.Password,
})
// Only look at V2 registries
registryAddress := authConfig.ServerAddress
if !strings.Contains(authConfig.ServerAddress, "/v2") {
registryAddress = registryAddress + "/v2/"
}
registryURL, err := url.Parse(registryAddress)
if err != nil {
msg := fmt.Sprintf("Bad login address: %s", registryAddress)
log.Errorf(msg)
return msg, "", err
}
// Check if requested registry is in our list of allowed insecure registries
var insecureOk bool
insecureRegistries := InsecureRegistries()
for _, registry := range insecureRegistries {
if registry == registryURL.Host {
insecureOk = true
break
}
}
dologin := func(scheme string) (string, error) {
registryURL.Scheme = scheme
var authURL *url.URL
// Attempt to get the Auth URL from HEAD operation to the registry
hdr, err := fetcher.Head(registryURL)
if err == nil && fetcher.IsStatusUnauthorized() {
authURL, err = fetcher.ExtractOAuthURL(hdr.Get("www-authenticate"), nil)
}
if err != nil {
log.Errorf("Looking up OAuth URL failed: %s", err)
return "", err
}
log.Debugf("logging onto %s", authURL.String())
// Just check if we get a token back.
token, err := fetcher.FetchAuthToken(authURL)
if err != nil || token.Token == "" {
log.Errorf("Fetch auth token failed: %s", err)
return "", err
}
return token.Token, nil
}
_, err = dologin("https")
if err != nil && insecureOk {
_, err = dologin("http")
}
if err != nil {
return "", "", err
}
// We don't return the token. The config.json will store token if we return
// it, but the regular docker daemon doesn't seem to return it either.
return "Login Succeeded", "", nil
}