func TestManifestFetchWithEtag(t *testing.T) {
repo, _ := reference.ParseNamed("test.example.com/repo/by/tag")
_, d1, p1 := newRandomSchemaV1Manifest(repo, "latest", 6)
var m testutil.RequestResponseMap
addTestManifestWithEtag(repo, "latest", p1, &m, d1.String())
e, c := testServer(m)
defer c()
ctx := context.Background()
r, err := NewRepository(ctx, repo, e, nil)
if err != nil {
t.Fatal(err)
}
ms, err := r.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
clientManifestService, ok := ms.(*manifests)
if !ok {
panic("wrong type for client manifest service")
}
_, err = clientManifestService.Get(ctx, d1, distribution.WithTag("latest"), AddEtagToTag("latest", d1.String()))
if err != distribution.ErrManifestNotModified {
t.Fatal(err)
}
}
func TestEventBridgeManifestPulledWithTag(t *testing.T) {
l := createTestEnv(t, testSinkFn(func(events ...Event) error {
checkCommonManifest(t, EventActionPull, events...)
if events[0].Target.Tag != "latest" {
t.Fatalf("missing or unexpected tag: %#v", events[0].Target)
}
return nil
}))
repoRef, _ := reference.ParseNamed(repo)
if err := l.ManifestPulled(repoRef, sm, distribution.WithTag(m.Tag)); err != nil {
t.Fatalf("unexpected error notifying manifest pull: %v", err)
}
}
// Push pushes a plugin to a registry.
func Push(name string, rs registry.Service, metaHeader http.Header, authConfig *types.AuthConfig, config io.ReadCloser, layers io.ReadCloser) (digest.Digest, error) {
ref, err := reference.ParseNamed(name)
if err != nil {
return "", err
}
repoInfo, err := rs.ResolveRepository(ref)
if err != nil {
return "", err
}
if err := dockerdist.ValidateRepoName(repoInfo.Name()); err != nil {
return "", err
}
endpoints, err := rs.LookupPushEndpoints(repoInfo.Hostname())
if err != nil {
return "", err
}
var confirmedV2 bool
var repository distribution.Repository
for _, endpoint := range endpoints {
if confirmedV2 && endpoint.Version == registry.APIVersion1 {
logrus.Debugf("Skipping v1 endpoint %s because v2 registry was detected", endpoint.URL)
continue
}
repository, confirmedV2, err = dockerdist.NewV2Repository(context.Background(), repoInfo, endpoint, metaHeader, authConfig, "push", "pull")
if err != nil {
return "", err
}
if !confirmedV2 {
return "", ErrUnSupportedRegistry
}
logrus.Debugf("Trying to push %s to %s %s", repoInfo.Name(), endpoint.URL, endpoint.Version)
// This means that we found an endpoint. and we are ready to push
break
}
// Returns a reference to the repository's blob service.
blobs := repository.Blobs(context.Background())
// Descriptor = {mediaType, size, digest}
var descs []distribution.Descriptor
for i, f := range []io.ReadCloser{config, layers} {
bw, err := blobs.Create(context.Background())
if err != nil {
logrus.Debugf("Error in blobs.Create: %v", err)
return "", err
}
h := sha256.New()
r := io.TeeReader(f, h)
_, err = io.Copy(bw, r)
if err != nil {
logrus.Debugf("Error in io.Copy: %v", err)
return "", err
}
f.Close()
mt := MediaTypeLayer
if i == 0 {
mt = MediaTypeConfig
}
// Commit completes the write process to the BlobService.
// The descriptor arg to Commit is called the "provisional" descriptor and
// used for validation.
// The returned descriptor should be the one used. Its called the "Canonical"
// descriptor.
desc, err := bw.Commit(context.Background(), distribution.Descriptor{
MediaType: mt,
// XXX: What about the Size?
Digest: digest.NewDigest("sha256", h),
})
if err != nil {
logrus.Debugf("Error in bw.Commit: %v", err)
return "", err
}
// The canonical descriptor is set the mediatype again, just in case.
// Dont touch the digest or the size here.
desc.MediaType = mt
logrus.Debugf("pushed blob: %s %s", desc.MediaType, desc.Digest)
descs = append(descs, desc)
}
// XXX: schema2.Versioned needs a MediaType as well.
// "application/vnd.docker.distribution.manifest.v2+json"
m, err := schema2.FromStruct(schema2.Manifest{Versioned: schema2.SchemaVersion, Config: descs[0], Layers: descs[1:]})
if err != nil {
logrus.Debugf("error in schema2.FromStruct: %v", err)
return "", err
}
msv, err := repository.Manifests(context.Background())
if err != nil {
logrus.Debugf("error in repository.Manifests: %v", err)
return "", err
}
_, pl, err := m.Payload()
if err != nil {
logrus.Debugf("error in m.Payload: %v", err)
return "", err
}
logrus.Debugf("Pushed manifest: %s", pl)
tag := DefaultTag
if tagged, ok := ref.(reference.NamedTagged); ok {
tag = tagged.Tag()
}
return msv.Put(context.Background(), m, distribution.WithTag(tag))
}
// Pull downloads the plugin from Store
func Pull(name string, rs registry.Service, metaheader http.Header, authConfig *types.AuthConfig) (PullData, error) {
ref, err := reference.ParseNamed(name)
if err != nil {
logrus.Debugf("pull.go: error in ParseNamed: %v", err)
return nil, err
}
repoInfo, err := rs.ResolveRepository(ref)
if err != nil {
logrus.Debugf("pull.go: error in ResolveRepository: %v", err)
return nil, err
}
if err := dockerdist.ValidateRepoName(repoInfo.Name()); err != nil {
logrus.Debugf("pull.go: error in ValidateRepoName: %v", err)
return nil, err
}
endpoints, err := rs.LookupPullEndpoints(repoInfo.Hostname())
if err != nil {
logrus.Debugf("pull.go: error in LookupPullEndpoints: %v", err)
return nil, err
}
var confirmedV2 bool
var repository distribution.Repository
for _, endpoint := range endpoints {
if confirmedV2 && endpoint.Version == registry.APIVersion1 {
logrus.Debugf("Skipping v1 endpoint %s because v2 registry was detected", endpoint.URL)
continue
}
// TODO: reuse contexts
repository, confirmedV2, err = dockerdist.NewV2Repository(context.Background(), repoInfo, endpoint, metaheader, authConfig, "pull")
if err != nil {
logrus.Debugf("pull.go: error in NewV2Repository: %v", err)
return nil, err
}
if !confirmedV2 {
logrus.Debugf("pull.go: !confirmedV2")
return nil, ErrUnSupportedRegistry
}
logrus.Debugf("Trying to pull %s from %s %s", repoInfo.Name(), endpoint.URL, endpoint.Version)
break
}
tag := DefaultTag
if ref, ok := ref.(reference.NamedTagged); ok {
tag = ref.Tag()
}
// tags := repository.Tags(context.Background())
// desc, err := tags.Get(context.Background(), tag)
// if err != nil {
// return nil, err
// }
//
msv, err := repository.Manifests(context.Background())
if err != nil {
logrus.Debugf("pull.go: error in repository.Manifests: %v", err)
return nil, err
}
manifest, err := msv.Get(context.Background(), "", distribution.WithTag(tag))
if err != nil {
// TODO: change 401 to 404
logrus.Debugf("pull.go: error in msv.Get(): %v", err)
return nil, err
}
_, pl, err := manifest.Payload()
if err != nil {
logrus.Debugf("pull.go: error in manifest.Payload(): %v", err)
return nil, err
}
var m schema2.Manifest
if err := json.Unmarshal(pl, &m); err != nil {
logrus.Debugf("pull.go: error in json.Unmarshal(): %v", err)
return nil, err
}
pd := &pullData{
repository: repository,
manifest: m,
}
logrus.Debugf("manifest: %s", pl)
return pd, nil
}
func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) {
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return false, err
}
var (
manifest distribution.Manifest
tagOrDigest string // Used for logging/progress only
)
if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
// NOTE: not using TagService.Get, since it uses HEAD requests
// against the manifests endpoint, which are not supported by
// all registry versions.
manifest, err = manSvc.Get(ctx, "", distribution.WithTag(tagged.Tag()))
if err != nil {
return false, allowV1Fallback(err)
}
tagOrDigest = tagged.Tag()
} else if digested, isDigested := ref.(reference.Canonical); isDigested {
manifest, err = manSvc.Get(ctx, digested.Digest())
if err != nil {
return false, err
}
tagOrDigest = digested.Digest().String()
} else {
return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String())
}
if manifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
}
// If manSvc.Get succeeded, we can be confident that the registry on
// the other side speaks the v2 protocol.
p.confirmedV2 = true
logrus.Debugf("Pulling ref from V2 registry: %s", ref.String())
progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+p.repo.Named().Name())
var (
imageID image.ID
manifestDigest digest.Digest
)
switch v := manifest.(type) {
case *schema1.SignedManifest:
imageID, manifestDigest, err = p.pullSchema1(ctx, ref, v)
if err != nil {
return false, err
}
case *schema2.DeserializedManifest:
imageID, manifestDigest, err = p.pullSchema2(ctx, ref, v)
if err != nil {
return false, err
}
case *manifestlist.DeserializedManifestList:
imageID, manifestDigest, err = p.pullManifestList(ctx, ref, v)
if err != nil {
return false, err
}
default:
return false, errors.New("unsupported manifest format")
}
progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
oldTagImageID, err := p.config.ReferenceStore.Get(ref)
if err == nil {
if oldTagImageID == imageID {
return false, addDigestReference(p.config.ReferenceStore, ref, manifestDigest, imageID)
}
} else if err != reference.ErrDoesNotExist {
return false, err
}
if canonical, ok := ref.(reference.Canonical); ok {
if err = p.config.ReferenceStore.AddDigest(canonical, imageID, true); err != nil {
return false, err
}
} else {
if err = addDigestReference(p.config.ReferenceStore, ref, manifestDigest, imageID); err != nil {
return false, err
}
if err = p.config.ReferenceStore.AddTag(ref, imageID, true); err != nil {
return false, err
}
}
return true, nil
}
func TestManifestPut(t *testing.T) {
repo, _ := reference.ParseNamed("test.example.com/repo/delete")
m1, dgst, _ := newRandomSchemaV1Manifest(repo, "other", 6)
_, payload, err := m1.Payload()
if err != nil {
t.Fatal(err)
}
var m testutil.RequestResponseMap
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "PUT",
Route: "/v2/" + repo.Name() + "/manifests/other",
Body: payload,
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
Headers: http.Header(map[string][]string{
"Content-Length": {"0"},
"Docker-Content-Digest": {dgst.String()},
}),
},
})
putDgst := digest.FromBytes(m1.Canonical)
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "PUT",
Route: "/v2/" + repo.Name() + "/manifests/" + putDgst.String(),
Body: payload,
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
Headers: http.Header(map[string][]string{
"Content-Length": {"0"},
"Docker-Content-Digest": {putDgst.String()},
}),
},
})
e, c := testServer(m)
defer c()
r, err := NewRepository(context.Background(), repo, e, nil)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
ms, err := r.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
if _, err := ms.Put(ctx, m1, distribution.WithTag(m1.Tag)); err != nil {
t.Fatal(err)
}
if _, err := ms.Put(ctx, m1); err != nil {
t.Fatal(err)
}
// TODO(dmcgowan): Check for invalid input error
}
func TestV1ManifestFetch(t *testing.T) {
ctx := context.Background()
repo, _ := reference.ParseNamed("test.example.com/repo")
m1, dgst, _ := newRandomSchemaV1Manifest(repo, "latest", 6)
var m testutil.RequestResponseMap
_, pl, err := m1.Payload()
if err != nil {
t.Fatal(err)
}
addTestManifest(repo, dgst.String(), schema1.MediaTypeSignedManifest, pl, &m)
addTestManifest(repo, "latest", schema1.MediaTypeSignedManifest, pl, &m)
addTestManifest(repo, "badcontenttype", "text/html", pl, &m)
e, c := testServer(m)
defer c()
r, err := NewRepository(context.Background(), repo, e, nil)
if err != nil {
t.Fatal(err)
}
ms, err := r.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
ok, err := ms.Exists(ctx, dgst)
if err != nil {
t.Fatal(err)
}
if !ok {
t.Fatal("Manifest does not exist")
}
manifest, err := ms.Get(ctx, dgst)
if err != nil {
t.Fatal(err)
}
v1manifest, ok := manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err := checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
var contentDigest digest.Digest
manifest, err = ms.Get(ctx, dgst, distribution.WithTag("latest"), ReturnContentDigest(&contentDigest))
if err != nil {
t.Fatal(err)
}
v1manifest, ok = manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err = checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
if contentDigest != dgst {
t.Fatalf("Unexpected returned content digest %v, expected %v", contentDigest, dgst)
}
manifest, err = ms.Get(ctx, dgst, distribution.WithTag("badcontenttype"))
if err != nil {
t.Fatal(err)
}
v1manifest, ok = manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err = checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
}
func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, imageID image.ID) error {
logrus.Debugf("Pushing repository: %s", ref.String())
img, err := p.config.ImageStore.Get(imageID)
if err != nil {
return fmt.Errorf("could not find image from tag %s: %v", ref.String(), err)
}
var l layer.Layer
topLayerID := img.RootFS.ChainID()
if topLayerID == "" {
l = layer.EmptyLayer
} else {
l, err = p.config.LayerStore.Get(topLayerID)
if err != nil {
return fmt.Errorf("failed to get top layer from image: %v", err)
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
}
var descriptors []xfer.UploadDescriptor
descriptorTemplate := v2PushDescriptor{
v2MetadataService: p.v2MetadataService,
repoInfo: p.repoInfo,
ref: p.ref,
repo: p.repo,
pushState: &p.pushState,
}
// Loop bounds condition is to avoid pushing the base layer on Windows.
for i := 0; i < len(img.RootFS.DiffIDs); i++ {
descriptor := descriptorTemplate
descriptor.layer = l
descriptors = append(descriptors, &descriptor)
l = l.Parent()
}
if err := p.config.UploadManager.Upload(ctx, descriptors, p.config.ProgressOutput); err != nil {
return err
}
// Try schema2 first
builder := schema2.NewManifestBuilder(p.repo.Blobs(ctx), img.RawJSON())
manifest, err := manifestFromBuilder(ctx, builder, descriptors)
if err != nil {
return err
}
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return err
}
putOptions := []distribution.ManifestServiceOption{distribution.WithTag(ref.Tag())}
if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
logrus.Warnf("failed to upload schema2 manifest: %v - falling back to schema1", err)
manifestRef, err := distreference.WithTag(p.repo.Named(), ref.Tag())
if err != nil {
return err
}
builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, manifestRef, img.RawJSON())
manifest, err = manifestFromBuilder(ctx, builder, descriptors)
if err != nil {
return err
}
if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
return err
}
}
var canonicalManifest []byte
switch v := manifest.(type) {
case *schema1.SignedManifest:
canonicalManifest = v.Canonical
case *schema2.DeserializedManifest:
_, canonicalManifest, err = v.Payload()
if err != nil {
return err
}
}
manifestDigest := digest.FromBytes(canonicalManifest)
progress.Messagef(p.config.ProgressOutput, "", "%s: digest: %s size: %d", ref.Tag(), manifestDigest, len(canonicalManifest))
if err := addDigestReference(p.config.ReferenceStore, ref, manifestDigest, imageID); err != nil {
return err
}
// Signal digest to the trust client so it can sign the
// push, if appropriate.
progress.Aux(p.config.ProgressOutput, PushResult{Tag: ref.Tag(), Digest: manifestDigest, Size: len(canonicalManifest)})
return nil
}
func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id digest.Digest) error {
logrus.Debugf("Pushing repository: %s", ref.String())
imgConfig, err := p.config.ImageStore.Get(id)
if err != nil {
return fmt.Errorf("could not find image from tag %s: %v", ref.String(), err)
}
rootfs, err := p.config.ImageStore.RootFSFromConfig(imgConfig)
if err != nil {
return fmt.Errorf("unable to get rootfs for image %s: %s", ref.String(), err)
}
l, err := p.config.LayerStore.Get(rootfs.ChainID())
if err != nil {
return fmt.Errorf("failed to get top layer from image: %v", err)
}
defer l.Release()
hmacKey, err := metadata.ComputeV2MetadataHMACKey(p.config.AuthConfig)
if err != nil {
return fmt.Errorf("failed to compute hmac key of auth config: %v", err)
}
var descriptors []xfer.UploadDescriptor
descriptorTemplate := v2PushDescriptor{
v2MetadataService: p.v2MetadataService,
hmacKey: hmacKey,
repoInfo: p.repoInfo,
ref: p.ref,
repo: p.repo,
pushState: &p.pushState,
}
// Loop bounds condition is to avoid pushing the base layer on Windows.
for i := 0; i < len(rootfs.DiffIDs); i++ {
descriptor := descriptorTemplate
descriptor.layer = l
descriptor.checkedDigests = make(map[digest.Digest]struct{})
descriptors = append(descriptors, &descriptor)
l = l.Parent()
}
if err := p.config.UploadManager.Upload(ctx, descriptors, p.config.ProgressOutput); err != nil {
return err
}
// Try schema2 first
builder := schema2.NewManifestBuilder(p.repo.Blobs(ctx), p.config.ConfigMediaType, imgConfig)
manifest, err := manifestFromBuilder(ctx, builder, descriptors)
if err != nil {
return err
}
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return err
}
putOptions := []distribution.ManifestServiceOption{distribution.WithTag(ref.Tag())}
if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
if runtime.GOOS == "windows" || p.config.TrustKey == nil || p.config.RequireSchema2 {
logrus.Warnf("failed to upload schema2 manifest: %v", err)
return err
}
logrus.Warnf("failed to upload schema2 manifest: %v - falling back to schema1", err)
manifestRef, err := distreference.WithTag(p.repo.Named(), ref.Tag())
if err != nil {
return err
}
builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, manifestRef, imgConfig)
manifest, err = manifestFromBuilder(ctx, builder, descriptors)
if err != nil {
return err
}
if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
return err
}
}
var canonicalManifest []byte
switch v := manifest.(type) {
case *schema1.SignedManifest:
canonicalManifest = v.Canonical
case *schema2.DeserializedManifest:
_, canonicalManifest, err = v.Payload()
if err != nil {
return err
}
}
manifestDigest := digest.FromBytes(canonicalManifest)
progress.Messagef(p.config.ProgressOutput, "", "%s: digest: %s size: %d", ref.Tag(), manifestDigest, len(canonicalManifest))
if err := addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil {
return err
}
// Signal digest to the trust client so it can sign the
// push, if appropriate.
progress.Aux(p.config.ProgressOutput, PushResult{Tag: ref.Tag(), Digest: manifestDigest, Size: len(canonicalManifest)})
return nil
}
// GetImageManifest fetches the image manifest from the storage backend, if it exists.
func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("GetImageManifest")
manifests, err := imh.Repository.Manifests(imh)
if err != nil {
imh.Errors = append(imh.Errors, err)
return
}
var manifest distribution.Manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
desc, err := tags.Get(imh, imh.Tag)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
imh.Digest = desc.Digest
}
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
var options []distribution.ManifestServiceOption
if imh.Tag != "" {
options = append(options, distribution.WithTag(imh.Tag))
}
manifest, err = manifests.Get(imh, imh.Digest, options...)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
supportsSchema2 := false
supportsManifestList := false
if acceptHeaders, ok := r.Header["Accept"]; ok {
for _, mediaType := range acceptHeaders {
if mediaType == schema2.MediaTypeManifest {
supportsSchema2 = true
}
if mediaType == manifestlist.MediaTypeManifestList {
supportsManifestList = true
}
}
}
schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest)
manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList)
// Only rewrite schema2 manifests when they are being fetched by tag.
// If they are being fetched by digest, we can't return something not
// matching the digest.
if imh.Tag != "" && isSchema2 && !supportsSchema2 {
// Rewrite manifest in schema1 format
ctxu.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String())
manifest, err = imh.convertSchema2Manifest(schema2Manifest)
if err != nil {
return
}
} else if imh.Tag != "" && isManifestList && !supportsManifestList {
// Rewrite manifest in schema1 format
ctxu.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String())
// Find the image manifest corresponding to the default
// platform
var manifestDigest digest.Digest
for _, manifestDescriptor := range manifestList.Manifests {
if manifestDescriptor.Platform.Architecture == defaultArch && manifestDescriptor.Platform.OS == defaultOS {
manifestDigest = manifestDescriptor.Digest
break
}
}
if manifestDigest == "" {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown)
return
}
manifest, err = manifests.Get(imh, manifestDigest)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
// If necessary, convert the image manifest
if schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 && !supportsSchema2 {
manifest, err = imh.convertSchema2Manifest(schema2Manifest)
if err != nil {
return
}
}
}
ct, p, err := manifest.Payload()
if err != nil {
return
}
w.Header().Set("Content-Type", ct)
w.Header().Set("Content-Length", fmt.Sprint(len(p)))
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest))
w.Write(p)
}
// PutImageManifest validates and stores an image in the registry.
func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("PutImageManifest")
manifests, err := imh.Repository.Manifests(imh)
if err != nil {
imh.Errors = append(imh.Errors, err)
return
}
var jsonBuf bytes.Buffer
if err := copyFullPayload(w, r, &jsonBuf, imh, "image manifest PUT", &imh.Errors); err != nil {
// copyFullPayload reports the error if necessary
return
}
mediaType := r.Header.Get("Content-Type")
manifest, desc, err := distribution.UnmarshalManifest(mediaType, jsonBuf.Bytes())
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err))
return
}
if imh.Digest != "" {
if desc.Digest != imh.Digest {
ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", desc.Digest, imh.Digest)
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid)
return
}
} else if imh.Tag != "" {
imh.Digest = desc.Digest
} else {
imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid.WithDetail("no tag or digest specified"))
return
}
var options []distribution.ManifestServiceOption
if imh.Tag != "" {
options = append(options, distribution.WithTag(imh.Tag))
}
_, err = manifests.Put(imh, manifest, options...)
if err != nil {
// TODO(stevvooe): These error handling switches really need to be
// handled by an app global mapper.
if err == distribution.ErrUnsupported {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnsupported)
return
}
if err == distribution.ErrAccessDenied {
imh.Errors = append(imh.Errors, errcode.ErrorCodeDenied)
return
}
switch err := err.(type) {
case distribution.ErrManifestVerification:
for _, verificationError := range err {
switch verificationError := verificationError.(type) {
case distribution.ErrManifestBlobUnknown:
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestBlobUnknown.WithDetail(verificationError.Digest))
case distribution.ErrManifestNameInvalid:
imh.Errors = append(imh.Errors, v2.ErrorCodeNameInvalid.WithDetail(err))
case distribution.ErrManifestUnverified:
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnverified)
default:
if verificationError == digest.ErrDigestInvalidFormat {
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid)
} else {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown, verificationError)
}
}
}
default:
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
// Tag this manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
err = tags.Tag(imh, imh.Tag, desc)
if err != nil {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
}
// Construct a canonical url for the uploaded manifest.
ref, err := reference.WithDigest(imh.Repository.Named(), imh.Digest)
if err != nil {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
location, err := imh.urlBuilder.BuildManifestURL(ref)
if err != nil {
// NOTE(stevvooe): Given the behavior above, this absurdly unlikely to
// happen. We'll log the error here but proceed as if it worked. Worst
// case, we set an empty location header.
ctxu.GetLogger(imh).Errorf("error building manifest url from digest: %v", err)
}
w.Header().Set("Location", location)
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.WriteHeader(http.StatusCreated)
}
// importRepositoryFromDocker loads the tags and images requested in the passed importRepository, obeying the
// optional rate limiter. Errors are set onto the individual tags and digest objects.
func (isi *ImageStreamImporter) importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetriever, repository *importRepository, limiter flowcontrol.RateLimiter) {
glog.V(5).Infof("importing remote Docker repository registry=%s repository=%s insecure=%t", repository.Registry, repository.Name, repository.Insecure)
// retrieve the repository
repo, err := retriever.Repository(ctx, repository.Registry, repository.Name, repository.Insecure)
if err != nil {
glog.V(5).Infof("unable to access repository %#v: %#v", repository, err)
switch {
case err == reference.ErrReferenceInvalidFormat:
err = field.Invalid(field.NewPath("from", "name"), repository.Name, "the provided repository name is not valid")
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
case strings.Contains(err.Error(), "tls: oversized record received with length") && !repository.Insecure:
err = kapierrors.NewBadRequest("this repository is HTTP only and requires the insecure flag to import")
case strings.HasSuffix(err.Error(), "no basic auth credentials"):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact()))
case strings.HasSuffix(err.Error(), "does not support v2 API"):
importRepositoryFromDockerV1(ctx, repository, limiter)
return
}
applyErrorToRepository(repository, err)
return
}
// get a manifest context
s, err := repo.Manifests(ctx)
if err != nil {
glog.V(5).Infof("unable to access manifests for repository %#v: %#v", repository, err)
switch {
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
case strings.HasSuffix(err.Error(), "no basic auth credentials"):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact()))
}
applyErrorToRepository(repository, err)
return
}
// get a blob context
b := repo.Blobs(ctx)
// if repository import is requested (MaximumTags), attempt to load the tags, sort them, and request the first N
if count := repository.MaximumTags; count > 0 || count == -1 {
tags, err := repo.Tags(ctx).All(ctx)
if err != nil {
glog.V(5).Infof("unable to access tags for repository %#v: %#v", repository, err)
switch {
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
}
repository.Err = err
return
}
// some images on the Hub have empty tags - treat those as "latest"
set := sets.NewString(tags...)
if set.Has("") {
set.Delete("")
set.Insert(api.DefaultImageTag)
}
tags = set.List()
// include only the top N tags in the result, put the rest in AdditionalTags
api.PrioritizeTags(tags)
for _, s := range tags {
if count <= 0 && repository.MaximumTags != -1 {
repository.AdditionalTags = append(repository.AdditionalTags, s)
continue
}
count--
repository.Tags = append(repository.Tags, importTag{
Name: s,
})
}
}
// load digests
for i := range repository.Digests {
importDigest := &repository.Digests[i]
if importDigest.Err != nil || importDigest.Image != nil {
continue
}
d, err := digest.ParseDigest(importDigest.Name)
if err != nil {
importDigest.Err = err
continue
}
limiter.Accept()
manifest, err := s.Get(ctx, d)
if err != nil {
glog.V(5).Infof("unable to access digest %q for repository %#v: %#v", d, repository, err)
importDigest.Err = formatRepositoryError(repository, "", importDigest.Name, err)
continue
}
if signedManifest, isSchema1 := manifest.(*schema1.SignedManifest); isSchema1 {
importDigest.Image, err = schema1ToImage(signedManifest, d)
} else if deserializedManifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 {
imageConfig, err := b.Get(ctx, deserializedManifest.Config.Digest)
if err != nil {
glog.V(5).Infof("unable to access the image config using digest %q for repository %#v: %#v", d, repository, err)
if isDockerError(err, v2.ErrorCodeManifestUnknown) {
ref := repository.Ref
ref.ID = deserializedManifest.Config.Digest.String()
importDigest.Err = kapierrors.NewNotFound(api.Resource("dockerimage"), ref.Exact())
} else {
importDigest.Err = formatRepositoryError(repository, "", importDigest.Name, err)
}
continue
}
importDigest.Image, err = schema2ToImage(deserializedManifest, imageConfig, d)
} else {
glog.V(5).Infof("unsupported manifest type: %T", manifest)
continue
}
if err != nil {
importDigest.Err = err
continue
}
if err := api.ImageWithMetadata(importDigest.Image); err != nil {
importDigest.Err = err
continue
}
if importDigest.Image.DockerImageMetadata.Size == 0 {
if err := isi.calculateImageSize(ctx, repo, importDigest.Image); err != nil {
importDigest.Err = err
continue
}
}
}
for i := range repository.Tags {
importTag := &repository.Tags[i]
if importTag.Err != nil || importTag.Image != nil {
continue
}
limiter.Accept()
manifest, err := s.Get(ctx, "", distribution.WithTag(importTag.Name))
if err != nil {
glog.V(5).Infof("unable to get manifest by tag %q for repository %#v: %#v", importTag.Name, repository, err)
// try to resolve the tag and fetch manifest by digest instead
desc, getTagErr := repo.Tags(ctx).Get(ctx, importTag.Name)
if getTagErr != nil {
glog.V(5).Infof("unable to get tag %q for repository %#v: %#v", importTag.Name, repository, getTagErr)
importTag.Err = formatRepositoryError(repository, importTag.Name, "", err)
continue
}
m, getManifestErr := s.Get(ctx, desc.Digest)
if getManifestErr != nil {
glog.V(5).Infof("unable to access digest %q for tag %q for repository %#v: %#v", desc.Digest, importTag.Name, repository, getManifestErr)
importTag.Err = formatRepositoryError(repository, importTag.Name, "", err)
continue
}
manifest = m
}
if signedManifest, isSchema1 := manifest.(*schema1.SignedManifest); isSchema1 {
importTag.Image, err = schema1ToImage(signedManifest, "")
} else if deserializedManifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 {
imageConfig, err := b.Get(ctx, deserializedManifest.Config.Digest)
if err != nil {
glog.V(5).Infof("unable to access image config using digest %q for tag %q for repository %#v: %#v", deserializedManifest.Config.Digest, importTag.Name, repository, err)
importTag.Err = formatRepositoryError(repository, importTag.Name, "", err)
continue
}
importTag.Image, err = schema2ToImage(deserializedManifest, imageConfig, "")
} else {
glog.V(5).Infof("unsupported manifest type: %T", manifest)
continue
}
if err != nil {
importTag.Err = err
continue
}
if err := api.ImageWithMetadata(importTag.Image); err != nil {
importTag.Err = err
continue
}
if importTag.Image.DockerImageMetadata.Size == 0 {
if err := isi.calculateImageSize(ctx, repo, importTag.Image); err != nil {
importTag.Err = err
continue
}
}
}
}
func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) {
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return false, err
}
var (
manifest distribution.Manifest
tagOrDigest string // Used for logging/progress only
)
if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
manifest, err = manSvc.Get(ctx, "", distribution.WithTag(tagged.Tag()))
if err != nil {
return false, allowV1Fallback(err)
}
tagOrDigest = tagged.Tag()
} else if digested, isDigested := ref.(reference.Canonical); isDigested {
manifest, err = manSvc.Get(ctx, digested.Digest())
if err != nil {
return false, err
}
tagOrDigest = digested.Digest().String()
} else {
return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String())
}
if manifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
}
if m, ok := manifest.(*schema2.DeserializedManifest); ok {
var allowedMediatype bool
for _, t := range p.config.Schema2Types {
if m.Manifest.Config.MediaType == t {
allowedMediatype = true
break
}
}
if !allowedMediatype {
configClass := mediaTypeClasses[m.Manifest.Config.MediaType]
if configClass == "" {
configClass = "unknown"
}
return false, fmt.Errorf("target is %s", configClass)
}
}
// If manSvc.Get succeeded, we can be confident that the registry on
// the other side speaks the v2 protocol.
p.confirmedV2 = true
logrus.Debugf("Pulling ref from V2 registry: %s", ref.String())
progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+p.repo.Named().Name())
var (
id digest.Digest
manifestDigest digest.Digest
)
switch v := manifest.(type) {
case *schema1.SignedManifest:
if p.config.RequireSchema2 {
return false, fmt.Errorf("invalid manifest: not schema2")
}
id, manifestDigest, err = p.pullSchema1(ctx, ref, v)
if err != nil {
return false, err
}
case *schema2.DeserializedManifest:
id, manifestDigest, err = p.pullSchema2(ctx, ref, v)
if err != nil {
return false, err
}
case *manifestlist.DeserializedManifestList:
id, manifestDigest, err = p.pullManifestList(ctx, ref, v)
if err != nil {
return false, err
}
default:
return false, errors.New("unsupported manifest format")
}
progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
if p.config.ReferenceStore != nil {
oldTagID, err := p.config.ReferenceStore.Get(ref)
if err == nil {
if oldTagID == id {
return false, addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id)
}
} else if err != reference.ErrDoesNotExist {
return false, err
}
if canonical, ok := ref.(reference.Canonical); ok {
if err = p.config.ReferenceStore.AddDigest(canonical, id, true); err != nil {
return false, err
}
} else {
if err = addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil {
return false, err
}
if err = p.config.ReferenceStore.AddTag(ref, id, true); err != nil {
return false, err
}
}
}
return true, nil
}
// GetImageManifest fetches the image manifest from the storage backend, if it exists.
func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("GetImageManifest")
manifests, err := imh.Repository.Manifests(imh)
if err != nil {
imh.Errors = append(imh.Errors, err)
return
}
var manifest distribution.Manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
desc, err := tags.Get(imh, imh.Tag)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
imh.Digest = desc.Digest
}
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
var options []distribution.ManifestServiceOption
if imh.Tag != "" {
options = append(options, distribution.WithTag(imh.Tag))
}
manifest, err = manifests.Get(imh, imh.Digest, options...)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
supportsSchema2 := false
supportsManifestList := false
// this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values
// https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202
for _, acceptHeader := range r.Header["Accept"] {
// r.Header[...] is a slice in case the request contains the same header more than once
// if the header isn't set, we'll get the zero value, which "range" will handle gracefully
// we need to split each header value on "," to get the full list of "Accept" values (per RFC 2616)
// https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1
for _, mediaType := range strings.Split(acceptHeader, ",") {
// remove "; q=..." if present
if i := strings.Index(mediaType, ";"); i >= 0 {
mediaType = mediaType[:i]
}
// it's common (but not required) for Accept values to be space separated ("a/b, c/d, e/f")
mediaType = strings.TrimSpace(mediaType)
if mediaType == schema2.MediaTypeManifest {
supportsSchema2 = true
}
if mediaType == manifestlist.MediaTypeManifestList {
supportsManifestList = true
}
}
}
schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest)
manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList)
// Only rewrite schema2 manifests when they are being fetched by tag.
// If they are being fetched by digest, we can't return something not
// matching the digest.
if imh.Tag != "" && isSchema2 && !supportsSchema2 {
// Rewrite manifest in schema1 format
ctxu.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String())
manifest, err = imh.convertSchema2Manifest(schema2Manifest)
if err != nil {
return
}
} else if imh.Tag != "" && isManifestList && !supportsManifestList {
// Rewrite manifest in schema1 format
ctxu.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String())
// Find the image manifest corresponding to the default
// platform
var manifestDigest digest.Digest
for _, manifestDescriptor := range manifestList.Manifests {
if manifestDescriptor.Platform.Architecture == defaultArch && manifestDescriptor.Platform.OS == defaultOS {
manifestDigest = manifestDescriptor.Digest
break
}
}
if manifestDigest == "" {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown)
return
}
manifest, err = manifests.Get(imh, manifestDigest)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
// If necessary, convert the image manifest
if schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 && !supportsSchema2 {
manifest, err = imh.convertSchema2Manifest(schema2Manifest)
if err != nil {
return
}
}
}
ct, p, err := manifest.Payload()
if err != nil {
return
}
w.Header().Set("Content-Type", ct)
w.Header().Set("Content-Length", fmt.Sprint(len(p)))
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest))
w.Write(p)
}
func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) {
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return false, err
}
var (
manifest distribution.Manifest
tagOrDigest string // Used for logging/progress only
)
if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
manifest, err = manSvc.Get(ctx, "", distribution.WithTag(tagged.Tag()))
if err != nil {
return false, allowV1Fallback(err)
}
tagOrDigest = tagged.Tag()
} else if digested, isDigested := ref.(reference.Canonical); isDigested {
manifest, err = manSvc.Get(ctx, digested.Digest())
if err != nil {
return false, err
}
tagOrDigest = digested.Digest().String()
} else {
return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String())
}
if manifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
}
if m, ok := manifest.(*schema2.DeserializedManifest); ok {
if m.Manifest.Config.MediaType == schema2.MediaTypePluginConfig ||
m.Manifest.Config.MediaType == "application/vnd.docker.plugin.image.v0+json" { //TODO: remove this v0 before 1.13 GA
return false, errMediaTypePlugin
}
}
// If manSvc.Get succeeded, we can be confident that the registry on
// the other side speaks the v2 protocol.
p.confirmedV2 = true
logrus.Debugf("Pulling ref from V2 registry: %s", ref.String())
progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+p.repo.Named().Name())
var (
id digest.Digest
manifestDigest digest.Digest
)
switch v := manifest.(type) {
case *schema1.SignedManifest:
id, manifestDigest, err = p.pullSchema1(ctx, ref, v)
if err != nil {
return false, err
}
case *schema2.DeserializedManifest:
id, manifestDigest, err = p.pullSchema2(ctx, ref, v)
if err != nil {
return false, err
}
case *manifestlist.DeserializedManifestList:
id, manifestDigest, err = p.pullManifestList(ctx, ref, v)
if err != nil {
return false, err
}
default:
return false, errors.New("unsupported manifest format")
}
progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
oldTagID, err := p.config.ReferenceStore.Get(ref)
if err == nil {
if oldTagID == id {
return false, addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id)
}
} else if err != reference.ErrDoesNotExist {
return false, err
}
if canonical, ok := ref.(reference.Canonical); ok {
if err = p.config.ReferenceStore.AddDigest(canonical, id, true); err != nil {
return false, err
}
} else {
if err = addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil {
return false, err
}
if err = p.config.ReferenceStore.AddTag(ref, id, true); err != nil {
return false, err
}
}
return true, nil
}