Exemple #1
0
// LearnRegistryURL returns the registry URL after making sure that it responds to queries
func LearnRegistryURL(options Options) (string, error) {
	defer trace.End(trace.Begin(options.Registry))

	req := func(schema string) (string, error) {
		registry := fmt.Sprintf("%s://%s/v2/", schema, options.Registry)

		url, err := url.Parse(registry)
		if err != nil {
			return "", err
		}
		log.Debugf("URL: %s", url)

		fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
			Timeout:            options.Timeout,
			Username:           options.Username,
			Password:           options.Password,
			InsecureSkipVerify: options.InsecureSkipVerify,
		})

		headers, err := fetcher.Head(url)
		if err != nil {
			return "", err
		}
		// v2 API requires this check
		if headers.Get("Docker-Distribution-API-Version") != "registry/2.0" {
			return "", fmt.Errorf("Missing Docker-Distribution-API-Version header")
		}
		return registry, nil
	}

	// first try https
	log.Debugf("Trying https scheme")
	registry, err := req("https")
	if err != nil && options.InsecureAllowHTTP {
		// fallback to http if it's allowed
		log.Debugf("Falling back to http scheme")
		registry, err = req("http")
	}

	return registry, err
}
Exemple #2
0
// FetchToken fetches the OAuth token from OAuth endpoint
func FetchToken(ctx context.Context, options Options, url *url.URL, progressOutput progress.Output) (*urlfetcher.Token, error) {
	defer trace.End(trace.Begin(url.String()))

	log.Debugf("URL: %s", url)

	fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
		Timeout:            options.Timeout,
		Username:           options.Username,
		Password:           options.Password,
		InsecureSkipVerify: options.InsecureSkipVerify,
	})

	token, err := fetcher.FetchAuthToken(url)
	if err != nil {
		err := fmt.Errorf("FetchToken (%s) failed: %s", url, err)
		log.Error(err)
		return nil, err
	}

	return token, nil
}
Exemple #3
0
// LearnAuthURL returns the URL of the OAuth endpoint
func LearnAuthURL(options Options) (*url.URL, error) {
	defer trace.End(trace.Begin(options.Image + "/" + options.Tag))

	url, err := url.Parse(options.Registry)
	if err != nil {
		return nil, err
	}
	url.Path = path.Join(url.Path, options.Image, "manifests", options.Tag)

	log.Debugf("URL: %s", url)

	fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
		Timeout:            options.Timeout,
		Username:           options.Username,
		Password:           options.Password,
		InsecureSkipVerify: options.InsecureSkipVerify,
	})

	// We expect docker registry to return a 401 to us - with a WWW-Authenticate header
	// We parse that header and learn the OAuth endpoint to fetch OAuth token.
	hdr, err := fetcher.Head(url)
	if err == nil && fetcher.IsStatusUnauthorized() {
		return fetcher.ExtractOAuthURL(hdr.Get("www-authenticate"), url)
	}

	// Private registry returned the manifest directly as auth option is optional.
	// https://github.com/docker/distribution/blob/master/docs/configuration.md#auth
	if err == nil && options.Registry != DefaultDockerURL && fetcher.IsStatusOK() {
		log.Debugf("%s does not support OAuth", url)
		return nil, nil
	}

	// Do we even have the image on that registry
	if err != nil && fetcher.IsStatusNotFound() {
		err = fmt.Errorf("image not found")
		return nil, urlfetcher.ImageNotFoundError{Err: err}
	}

	return nil, fmt.Errorf("%s returned an unexpected response: %s", url, err)
}
Exemple #4
0
// FetchImageManifest fetches the image manifest file
func FetchImageManifest(ctx context.Context, options Options, progressOutput progress.Output) (*Manifest, error) {
	defer trace.End(trace.Begin(options.Image + "/" + options.Tag))

	url, err := url.Parse(options.Registry)
	if err != nil {
		return nil, err
	}
	url.Path = path.Join(url.Path, options.Image, "manifests", options.Tag)

	log.Debugf("URL: %s", url)

	fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
		Timeout:            options.Timeout,
		Username:           options.Username,
		Password:           options.Password,
		Token:              options.Token,
		InsecureSkipVerify: options.InsecureSkipVerify,
	})

	manifestFileName, err := fetcher.Fetch(ctx, url, true, progressOutput)
	if err != nil {
		return nil, err
	}

	// Cleanup function for the error case
	defer func() {
		if err != nil {
			os.Remove(manifestFileName)
		}
	}()

	// Read the entire file into []byte for json.Unmarshal
	content, err := ioutil.ReadFile(manifestFileName)
	if err != nil {
		return nil, err
	}

	manifest := &Manifest{}

	err = json.Unmarshal(content, manifest)
	if err != nil {
		return nil, err
	}

	if manifest.Name != options.Image {
		return nil, fmt.Errorf("name doesn't match what was requested, expected: %s, downloaded: %s", options.Image, manifest.Name)
	}

	if manifest.Tag != options.Tag {
		return nil, fmt.Errorf("tag doesn't match what was requested, expected: %s, downloaded: %s", options.Tag, manifest.Tag)
	}

	digest, err := getManifestDigest(content)
	if err != nil {
		return nil, err
	}

	manifest.Digest = digest

	// Ensure the parent directory exists
	destination := DestinationDirectory(options)
	err = os.MkdirAll(destination, 0755) /* #nosec */
	if err != nil {
		return nil, err
	}

	// Move(rename) the temporary file to its final destination
	err = os.Rename(string(manifestFileName), path.Join(destination, "manifest.json"))
	if err != nil {
		return nil, err
	}

	return manifest, nil
}
Exemple #5
0
// FetchImageBlob fetches the image blob
func FetchImageBlob(ctx context.Context, options Options, image *ImageWithMeta, progressOutput progress.Output) (string, error) {
	defer trace.End(trace.Begin(options.Image + "/" + image.Layer.BlobSum))

	id := image.ID
	layer := image.Layer.BlobSum
	meta := image.Meta
	diffID := ""

	url, err := url.Parse(options.Registry)
	if err != nil {
		return diffID, err
	}
	url.Path = path.Join(url.Path, options.Image, "blobs", layer)

	log.Debugf("URL: %s\n ", url)

	fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
		Timeout:            options.Timeout,
		Username:           options.Username,
		Password:           options.Password,
		Token:              options.Token,
		InsecureSkipVerify: options.InsecureSkipVerify,
	})

	// ctx
	ctx, cancel := context.WithTimeout(ctx, options.Timeout)
	defer cancel()

	imageFileName, err := fetcher.Fetch(ctx, url, true, progressOutput, image.String())
	if err != nil {
		return diffID, err
	}

	// Cleanup function for the error case
	defer func() {
		if err != nil {
			os.Remove(imageFileName)
		}
	}()

	// Open the file so that we can use it as a io.Reader for sha256 calculation
	imageFile, err := os.Open(string(imageFileName))
	if err != nil {
		return diffID, err
	}
	defer imageFile.Close()

	// blobSum is the sha of the compressed layer
	blobSum := sha256.New()

	// diffIDSum is the sha of the uncompressed layer
	diffIDSum := sha256.New()

	// blobTr is an io.TeeReader that writes bytes to blobSum that it reads from imageFile
	// see https://golang.org/pkg/io/#TeeReader
	blobTr := io.TeeReader(imageFile, blobSum)

	progress.Update(progressOutput, image.String(), "Verifying Checksum")
	decompressedTar, err := archive.DecompressStream(blobTr)
	if err != nil {
		return diffID, err
	}

	// Copy bytes from decompressed layer into diffIDSum to calculate diffID
	_, cerr := io.Copy(diffIDSum, decompressedTar)
	if cerr != nil {
		return diffID, cerr
	}

	bs := fmt.Sprintf("sha256:%x", blobSum.Sum(nil))
	if bs != layer {
		return diffID, fmt.Errorf("Failed to validate layer checksum. Expected %s got %s", layer, bs)
	}

	diffID = fmt.Sprintf("sha256:%x", diffIDSum.Sum(nil))

	// this isn't an empty layer, so we need to calculate the size
	if diffID != string(DigestSHA256EmptyTar) {
		var layerSize int64

		// seek to the beginning of the file
		imageFile.Seek(0, 0)

		// recreate the decompressed tar Reader
		decompressedTar, err := archive.DecompressStream(imageFile)
		if err != nil {
			return "", err
		}

		// get a tar reader for access to the files in the archive
		tr := tar.NewReader(decompressedTar)

		// iterate through tar headers to get file sizes
		for {
			tarHeader, err := tr.Next()
			if err == io.EOF {
				break
			}
			if err != nil {
				return "", err
			}
			layerSize += tarHeader.Size
		}

		image.Size = layerSize
	}

	log.Infof("diffID for layer %s: %s", id, diffID)

	// Ensure the parent directory exists
	destination := path.Join(DestinationDirectory(options), id)
	err = os.MkdirAll(destination, 0755) /* #nosec */
	if err != nil {
		return diffID, err
	}

	// Move(rename) the temporary file to its final destination
	err = os.Rename(string(imageFileName), path.Join(destination, id+".tar"))
	if err != nil {
		return diffID, err
	}

	// Dump the history next to it
	err = ioutil.WriteFile(path.Join(destination, id+".json"), []byte(meta), 0644)
	if err != nil {
		return diffID, err
	}

	progress.Update(progressOutput, image.String(), "Download complete")

	return diffID, nil
}
Exemple #6
0
func (s *System) AuthenticateToRegistry(ctx context.Context, authConfig *types.AuthConfig) (string, string, error) {
	defer trace.End(trace.Begin(""))

	fetcher := urlfetcher.NewURLFetcher(urlfetcher.Options{
		Timeout:  loginTimeout,
		Username: authConfig.Username,
		Password: authConfig.Password,
	})

	// Only look at V2 registries
	registryAddress := authConfig.ServerAddress
	if !strings.Contains(authConfig.ServerAddress, "/v2") {
		registryAddress = registryAddress + "/v2/"
	}

	registryURL, err := url.Parse(registryAddress)
	if err != nil {
		msg := fmt.Sprintf("Bad login address: %s", registryAddress)
		log.Errorf(msg)
		return msg, "", err
	}

	// Check if requested registry is in our list of allowed insecure registries
	var insecureOk bool
	insecureRegistries := InsecureRegistries()
	for _, registry := range insecureRegistries {
		if registry == registryURL.Host {
			insecureOk = true
			break
		}
	}

	dologin := func(scheme string) (string, error) {
		registryURL.Scheme = scheme

		var authURL *url.URL

		// Attempt to get the Auth URL from HEAD operation to the registry
		hdr, err := fetcher.Head(registryURL)
		if err == nil && fetcher.IsStatusUnauthorized() {
			authURL, err = fetcher.ExtractOAuthURL(hdr.Get("www-authenticate"), nil)
		}
		if err != nil {
			log.Errorf("Looking up OAuth URL failed: %s", err)
			return "", err
		}

		log.Debugf("logging onto %s", authURL.String())

		// Just check if we get a token back.
		token, err := fetcher.FetchAuthToken(authURL)
		if err != nil || token.Token == "" {
			log.Errorf("Fetch auth token failed: %s", err)
			return "", err
		}

		return token.Token, nil
	}

	_, err = dologin("https")
	if err != nil && insecureOk {
		_, err = dologin("http")
	}

	if err != nil {
		return "", "", err
	}

	// We don't return the token.  The config.json will store token if we return
	// it, but the regular docker daemon doesn't seem to return it  either.
	return "Login Succeeded", "", nil
}