func (r *repositoryRetriever) Repository(ctx gocontext.Context, registry *url.URL, repoName string, insecure bool) (distribution.Repository, error) {
src := *registry
// ping the registry to get challenge headers
if err, ok := r.pings[src]; ok {
if err != nil {
return nil, err
}
if redirect, ok := r.redirect[src]; ok {
src = *redirect
}
} else {
redirect, err := r.ping(src, insecure)
r.pings[src] = err
if err != nil {
return nil, err
}
if redirect != nil {
r.redirect[src] = redirect
src = *redirect
}
}
rt := transport.NewTransport(
r.context.Transport,
// TODO: slightly smarter authorizer that retries unauthenticated requests
auth.NewAuthorizer(
r.context.Challenges,
auth.NewTokenHandler(r.context.Transport, r.credentials, repoName, "pull"),
auth.NewBasicHandler(r.credentials),
),
)
return registryclient.NewRepository(context.Context(ctx), repoName, src.String(), rt)
}
func uploadBlobsToRegistry(repostr string, file *os.File, layers []*Layer, manifest *manifest.SignedManifest) error {
/* rewind first */
file.Seek(0, 0)
archive := tar.NewReader(bufio.NewReader(file))
url, repo := splitUrlAndRepo(repostr)
tr := transport.NewTransport(http.DefaultTransport)
repository, err := client.NewRepository(context.Background(), repo, "http://"+url, tr)
for _, v := range layers {
/* probe remote endpoint */
dsc, err := repository.Blobs(context.Background()).Stat(context.Background(), v.BlobSum)
switch err {
case nil:
case distribution.ErrBlobUnknown:
default:
return err
}
if err == distribution.ErrBlobUnknown {
/* rewind after each seek */
file.Seek(0, 0)
bb, err := getLayerRaw(archive, v.Id)
if err != nil {
return err
}
if verbose {
fmt.Printf("Uploading layer: %q size: %d\n", v.BlobSum, len(bb))
}
dsc, err := repository.Blobs(context.Background()).Put(context.Background(), LayerType, bb)
if err != nil {
return err
}
if verbose {
fmt.Printf(" uploaded with digest: %q\n", dsc.Digest)
}
} else {
if verbose {
fmt.Printf("Already in blob store: %q\n", dsc.Digest)
}
}
}
manSvc, err := repository.Manifests(context.Background())
if err == nil {
return manSvc.Put(manifest)
}
return err
}
func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named) (distribution.Repository, error) {
c := pr.authChallenger
tr := transport.NewTransport(http.DefaultTransport,
auth.NewAuthorizer(c.challengeManager(), auth.NewTokenHandler(http.DefaultTransport, c.credentialStore(), name.Name(), "pull")))
localRepo, err := pr.embedded.Repository(ctx, name)
if err != nil {
return nil, err
}
localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification())
if err != nil {
return nil, err
}
remoteRepo, err := client.NewRepository(ctx, name, pr.remoteURL.String(), tr)
if err != nil {
return nil, err
}
remoteManifests, err := remoteRepo.Manifests(ctx)
if err != nil {
return nil, err
}
return &proxiedRepository{
blobStore: &proxyBlobStore{
localStore: localRepo.Blobs(ctx),
remoteStore: remoteRepo.Blobs(ctx),
scheduler: pr.scheduler,
repositoryName: name,
authChallenger: pr.authChallenger,
},
manifests: &proxyManifestStore{
repositoryName: name,
localManifests: localManifests, // Options?
remoteManifests: remoteManifests,
ctx: ctx,
scheduler: pr.scheduler,
authChallenger: pr.authChallenger,
},
name: name,
tags: &proxyTagService{
localTags: localRepo.Tags(ctx),
remoteTags: remoteRepo.Tags(ctx),
authChallenger: pr.authChallenger,
},
}, nil
}
func (d dialer) Dial(logger lager.Logger, host, repo string) (Conn, error) {
host, transport, err := newTransport(logger, d.InsecureRegistryList, host, repo)
if err != nil {
logger.Error("failed-to-construct-transport", err)
return nil, err
}
repoClient, err := client.NewRepository(context.TODO(), repo, host, transport)
if err != nil {
logger.Error("failed-to-construct-repository", err)
return nil, err
}
return &conn{client: repoClient}, nil
}
func (r *repositoryRetriever) Repository(ctx gocontext.Context, registry *url.URL, repoName string, insecure bool) (distribution.Repository, error) {
named, err := reference.ParseNamed(repoName)
if err != nil {
return nil, err
}
t := r.context.Transport
if insecure && r.context.InsecureTransport != nil {
t = r.context.InsecureTransport
}
src := *registry
// ping the registry to get challenge headers
if err, ok := r.pings[src]; ok {
if err != nil {
return nil, err
}
if redirect, ok := r.redirect[src]; ok {
src = *redirect
}
} else {
redirect, err := r.ping(src, insecure, t)
r.pings[src] = err
if err != nil {
return nil, err
}
if redirect != nil {
r.redirect[src] = redirect
src = *redirect
}
}
rt := transport.NewTransport(
t,
// TODO: slightly smarter authorizer that retries unauthenticated requests
// TODO: make multiple attempts if the first credential fails
auth.NewAuthorizer(
r.context.Challenges,
auth.NewTokenHandler(t, r.credentials, repoName, "pull"),
auth.NewBasicHandler(r.credentials),
),
)
repo, err := registryclient.NewRepository(context.Context(ctx), named, src.String(), rt)
if err != nil {
return nil, err
}
return NewRetryRepository(repo, 2, 3/2*time.Second), nil
}
func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distribution.Repository, error) {
tr := transport.NewTransport(http.DefaultTransport,
auth.NewAuthorizer(pr.challengeManager, auth.NewTokenHandler(http.DefaultTransport, pr.credentialStore, name, "pull")))
localRepo, err := pr.embedded.Repository(ctx, name)
if err != nil {
return nil, err
}
localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification)
if err != nil {
return nil, err
}
remoteRepo, err := client.NewRepository(ctx, name, pr.remoteURL, tr)
if err != nil {
return nil, err
}
remoteManifests, err := remoteRepo.Manifests(ctx)
if err != nil {
return nil, err
}
return &proxiedRepository{
blobStore: &proxyBlobStore{
localStore: localRepo.Blobs(ctx),
remoteStore: remoteRepo.Blobs(ctx),
scheduler: pr.scheduler,
},
manifests: proxyManifestStore{
repositoryName: name,
localManifests: localManifests, // Options?
remoteManifests: remoteManifests,
ctx: ctx,
scheduler: pr.scheduler,
},
name: name,
signatures: localRepo.Signatures(),
}, nil
}
// UploadTestBlob generates a random tar file and uploads it to the given repository.
func UploadTestBlob(serverURL *url.URL, repoName string) (distribution.Descriptor, []byte, error) {
rs, ds, err := CreateRandomTarFile()
if err != nil {
return distribution.Descriptor{}, nil, fmt.Errorf("unexpected error generating test layer file: %v", err)
}
dgst := digest.Digest(ds)
ctx := context.Background()
ref, err := reference.ParseNamed(repoName)
if err != nil {
return distribution.Descriptor{}, nil, err
}
repo, err := distclient.NewRepository(ctx, ref, serverURL.String(), nil)
if err != nil {
return distribution.Descriptor{}, nil, fmt.Errorf("failed to get repository %q: %v", repoName, err)
}
wr, err := repo.Blobs(ctx).Create(ctx)
if err != nil {
return distribution.Descriptor{}, nil, err
}
if _, err := io.Copy(wr, rs); err != nil {
return distribution.Descriptor{}, nil, fmt.Errorf("unexpected error copying to upload: %v", err)
}
desc, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst})
if err != nil {
return distribution.Descriptor{}, nil, err
}
if _, err := rs.Seek(0, 0); err != nil {
return distribution.Descriptor{}, nil, fmt.Errorf("failed to seak blob reader: %v", err)
}
content, err := ioutil.ReadAll(rs)
if err != nil {
return distribution.Descriptor{}, nil, fmt.Errorf("failed to read blob content: %v", err)
}
return desc, content, nil
}
// NewV2Repository returns a repository (v2 only). It creates a HTTP transport
// providing timeout settings and authentication support, and also verifies the
// remote API version.
func NewV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *cliconfig.AuthConfig, actions ...string) (distribution.Repository, error) {
ctx := context.Background()
repoName := repoInfo.CanonicalName
// If endpoint does not support CanonicalName, use the RemoteName instead
if endpoint.TrimHostname {
repoName = repoInfo.RemoteName
}
// TODO(dmcgowan): Call close idle connections when complete, use keep alive
base := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: endpoint.TLSConfig,
// TODO(dmcgowan): Call close idle connections when complete and use keep alive
DisableKeepAlives: true,
}
modifiers := registry.DockerHeaders(metaHeaders)
authTransport := transport.NewTransport(base, modifiers...)
pingClient := &http.Client{
Transport: authTransport,
Timeout: 15 * time.Second,
}
endpointStr := strings.TrimRight(endpoint.URL, "/") + "/v2/"
req, err := http.NewRequest("GET", endpointStr, nil)
if err != nil {
return nil, err
}
resp, err := pingClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
versions := auth.APIVersions(resp, endpoint.VersionHeader)
if endpoint.VersionHeader != "" && len(endpoint.Versions) > 0 {
var foundVersion bool
for _, version := range endpoint.Versions {
for _, pingVersion := range versions {
if version == pingVersion {
foundVersion = true
}
}
}
if !foundVersion {
return nil, errors.New("endpoint does not support v2 API")
}
}
challengeManager := auth.NewSimpleChallengeManager()
if err := challengeManager.AddResponse(resp); err != nil {
return nil, err
}
creds := dumbCredentialStore{auth: authConfig}
tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, actions...)
basicHandler := auth.NewBasicHandler(creds)
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
tr := transport.NewTransport(base, modifiers...)
return client.NewRepository(ctx, repoName, endpoint.URL, tr)
}
// NewV2Repository returns a repository (v2 only). It creates a HTTP transport
// providing timeout settings and authentication support, and also verifies the
// remote API version.
func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *types.AuthConfig, actions ...string) (repo distribution.Repository, foundVersion bool, err error) {
repoName := repoInfo.FullName()
// If endpoint does not support CanonicalName, use the RemoteName instead
if endpoint.TrimHostname {
repoName = repoInfo.RemoteName()
}
// TODO(dmcgowan): Call close idle connections when complete, use keep alive
base := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: func(network, address string) (net.Conn, error) {
dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}
netConn, err := dialer.Dial(network, address)
if err != nil {
return netConn, err
}
return &conn{
Conn: netConn,
readTimeout: time.Minute,
writeTimeout: time.Minute,
}, nil
},
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: endpoint.TLSConfig,
// TODO(dmcgowan): Call close idle connections when complete and use keep alive
DisableKeepAlives: true,
}
modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(), metaHeaders)
authTransport := transport.NewTransport(base, modifiers...)
pingClient := &http.Client{
Transport: authTransport,
Timeout: 15 * time.Second,
}
endpointStr := strings.TrimRight(endpoint.URL, "/") + "/v2/"
req, err := http.NewRequest("GET", endpointStr, nil)
if err != nil {
return nil, false, err
}
resp, err := pingClient.Do(req)
if err != nil {
return nil, false, err
}
defer resp.Body.Close()
v2Version := auth.APIVersion{
Type: "registry",
Version: "2.0",
}
versions := auth.APIVersions(resp, registry.DefaultRegistryVersionHeader)
for _, pingVersion := range versions {
if pingVersion == v2Version {
// The version header indicates we're definitely
// talking to a v2 registry. So don't allow future
// fallbacks to the v1 protocol.
foundVersion = true
break
}
}
challengeManager := auth.NewSimpleChallengeManager()
if err := challengeManager.AddResponse(resp); err != nil {
return nil, foundVersion, err
}
if authConfig.RegistryToken != "" {
passThruTokenHandler := &existingTokenHandler{token: authConfig.RegistryToken}
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, passThruTokenHandler))
} else {
creds := dumbCredentialStore{auth: authConfig}
tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, actions...)
basicHandler := auth.NewBasicHandler(creds)
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
}
tr := transport.NewTransport(base, modifiers...)
repo, err = client.NewRepository(ctx, repoName, endpoint.URL, tr)
return repo, foundVersion, err
}
// NewV2Repository returns a repository (v2 only). It creates a HTTP transport
// providing timeout settings and authentication support, and also verifies the
// remote API version.
func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *types.AuthConfig, actions ...string) (repo distribution.Repository, foundVersion bool, err error) {
repoName := repoInfo.FullName()
// If endpoint does not support CanonicalName, use the RemoteName instead
if endpoint.TrimHostname {
repoName = repoInfo.RemoteName()
}
// TODO(dmcgowan): Call close idle connections when complete, use keep alive
base := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: endpoint.TLSConfig,
// TODO(dmcgowan): Call close idle connections when complete and use keep alive
DisableKeepAlives: true,
}
modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(), metaHeaders)
authTransport := transport.NewTransport(base, modifiers...)
challengeManager, foundVersion, err := registry.PingV2Registry(endpoint, authTransport)
if err != nil {
transportOK := false
if responseErr, ok := err.(registry.PingResponseError); ok {
transportOK = true
err = responseErr.Err
}
return nil, foundVersion, fallbackError{
err: err,
confirmedV2: foundVersion,
transportOK: transportOK,
}
}
if authConfig.RegistryToken != "" {
passThruTokenHandler := &existingTokenHandler{token: authConfig.RegistryToken}
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, passThruTokenHandler))
} else {
creds := dumbCredentialStore{auth: authConfig}
tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, actions...)
basicHandler := auth.NewBasicHandler(creds)
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
}
tr := transport.NewTransport(base, modifiers...)
repoNameRef, err := distreference.ParseNamed(repoName)
if err != nil {
return nil, foundVersion, fallbackError{
err: err,
confirmedV2: foundVersion,
transportOK: true,
}
}
repo, err = client.NewRepository(ctx, repoNameRef, endpoint.URL.String(), tr)
if err != nil {
err = fallbackError{
err: err,
confirmedV2: foundVersion,
transportOK: true,
}
}
return
}
// NewV2Repository returns a repository (v2 only). It creates an HTTP transport
// providing timeout settings and authentication support, and also verifies the
// remote API version.
func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *types.AuthConfig, actions ...string) (repo distribution.Repository, foundVersion bool, err error) {
repoName := repoInfo.FullName()
// If endpoint does not support CanonicalName, use the RemoteName instead
if endpoint.TrimHostname {
repoName = repoInfo.RemoteName()
}
direct := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}
// TODO(dmcgowan): Call close idle connections when complete, use keep alive
base := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: direct.Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: endpoint.TLSConfig,
// TODO(dmcgowan): Call close idle connections when complete and use keep alive
DisableKeepAlives: true,
}
proxyDialer, err := sockets.DialerFromEnvironment(direct)
if err == nil {
base.Dial = proxyDialer.Dial
}
modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(ctx), metaHeaders)
authTransport := transport.NewTransport(base, modifiers...)
challengeManager, foundVersion, err := registry.PingV2Registry(endpoint.URL, authTransport)
if err != nil {
transportOK := false
if responseErr, ok := err.(registry.PingResponseError); ok {
transportOK = true
err = responseErr.Err
}
return nil, foundVersion, fallbackError{
err: err,
confirmedV2: foundVersion,
transportOK: transportOK,
}
}
if authConfig.RegistryToken != "" {
passThruTokenHandler := &existingTokenHandler{token: authConfig.RegistryToken}
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, passThruTokenHandler))
} else {
scope := auth.RepositoryScope{
Repository: repoName,
Actions: actions,
}
// Keep image repositories blank for scope compatibility
if repoInfo.Class != "image" {
scope.Class = repoInfo.Class
}
creds := registry.NewStaticCredentialStore(authConfig)
tokenHandlerOptions := auth.TokenHandlerOptions{
Transport: authTransport,
Credentials: creds,
Scopes: []auth.Scope{scope},
ClientID: registry.AuthClientID,
}
tokenHandler := auth.NewTokenHandlerWithOptions(tokenHandlerOptions)
basicHandler := auth.NewBasicHandler(creds)
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
}
tr := transport.NewTransport(base, modifiers...)
repoNameRef, err := distreference.ParseNamed(repoName)
if err != nil {
return nil, foundVersion, fallbackError{
err: err,
confirmedV2: foundVersion,
transportOK: true,
}
}
repo, err = client.NewRepository(ctx, repoNameRef, endpoint.URL.String(), tr)
if err != nil {
err = fallbackError{
err: err,
confirmedV2: foundVersion,
transportOK: true,
}
}
return
}
// Spool downloades Docker images from Distribution, builds base layer for Porto container
func (b *Box) Spool(ctx context.Context, name string, opts isolate.Profile) (err error) {
defer apexctx.GetLogger(ctx).WithField("name", name).Trace("spool").Stop(&err)
profile, err := docker.ConvertProfile(opts)
if err != nil {
apexctx.GetLogger(ctx).WithError(err).WithField("name", name).Info("unbale to convert raw profile to Porto/Docker specific profile")
return err
}
if profile.Registry == "" {
apexctx.GetLogger(ctx).WithField("name", name).Error("Registry must be non empty")
return fmt.Errorf("Registry must be non empty")
}
portoConn, err := portoConnect()
if err != nil {
apexctx.GetLogger(ctx).WithError(err).WithField("name", name).Error("Porto connection error")
return err
}
named, err := reference.ParseNamed(filepath.Join(profile.Repository, profile.Repository, name))
if err != nil {
apexctx.GetLogger(ctx).WithError(err).WithField("name", name).Error("name is invalid")
return err
}
var tr http.RoundTripper
if registryAuth, ok := b.config.RegistryAuth[profile.Registry]; ok {
tr = transport.NewTransport(b.transport, transport.NewHeaderRequestModifier(http.Header{
"Authorization": []string{registryAuth},
}))
} else {
tr = b.transport
}
var registry = profile.Registry
if !strings.HasPrefix(registry, "http") {
registry = "https://" + registry
}
apexctx.GetLogger(ctx).Debugf("Image URI generated at spawn with data: %s and %s", registry, named)
repo, err := client.NewRepository(ctx, named, registry, tr)
if err != nil {
return err
}
tagDescriptor, err := repo.Tags(ctx).Get(ctx, engineref.GetTagFromNamedRef(named))
if err != nil {
return err
}
layerName := b.appLayerName(name)
digest := tagDescriptor.Digest.String()
if b.journal.In(layerName, digest) {
apexctx.GetLogger(ctx).WithField("name", name).Infof("layer %s has been found in the cache", digest)
return nil
}
manifests, err := repo.Manifests(ctx)
if err != nil {
return err
}
manifest, err := manifests.Get(ctx, tagDescriptor.Digest)
if err != nil {
return err
}
if err = portoConn.RemoveLayer(layerName); err != nil && !isEqualPortoError(err, portorpc.EError_LayerNotFound) {
return err
}
apexctx.GetLogger(ctx).WithField("name", name).Infof("create a layer %s in Porto with merge", layerName)
var order layersOrder
switch manifest.(type) {
case schema1.SignedManifest, *schema1.SignedManifest:
order = layerOrderV1
case schema2.DeserializedManifest, *schema2.DeserializedManifest:
order = layerOrderV2
default:
return fmt.Errorf("unknown manifest type %T", manifest)
}
for _, descriptor := range order(manifest.References()) {
blobPath, err := b.blobRepo.Get(ctx, repo, descriptor.Digest)
if err != nil {
return err
}
entry := apexctx.GetLogger(ctx).WithField("layer", layerName).Trace("ImportLayer with merge")
err = portoConn.ImportLayer(layerName, blobPath, true)
entry.Stop(&err)
if err != nil {
return err
}
}
b.journal.Insert(layerName, digest)
// NOTE: Not so fast, but it's important to debug
journalContent.Set(b.journal.String())
return nil
}