// fillImageWithMetadata fills a given image with metadata. Also correct layer sizes with blob sizes. Newer
// Docker client versions don't set layer sizes in the manifest at all. Origin master needs correct layer
// sizes for proper image quota support. That's why we need to fill the metadata in the registry.
func (r *repository) fillImageWithMetadata(manifest *schema1.SignedManifest, image *imageapi.Image) error {
if err := imageapi.ImageWithMetadata(image); err != nil {
return err
}
layerSet := sets.NewString()
size := int64(0)
blobs := r.Blobs(r.ctx)
for i := range image.DockerImageLayers {
layer := &image.DockerImageLayers[i]
// DockerImageLayers represents manifest.Manifest.FSLayers in reversed order
desc, err := blobs.Stat(r.ctx, manifest.Manifest.FSLayers[len(image.DockerImageLayers)-i-1].BlobSum)
if err != nil {
context.GetLogger(r.ctx).Errorf("Failed to stat blobs %s of image %s", layer.Name, image.DockerImageReference)
return err
}
layer.Size = desc.Size
// count empty layer just once (empty layer may actually have non-zero size)
if !layerSet.Has(layer.Name) {
size += desc.Size
layerSet.Insert(layer.Name)
}
}
image.DockerImageMetadata.Size = size
context.GetLogger(r.ctx).Infof("Total size of image %s with docker ref %s: %d", image.Name, image.DockerImageReference, size)
return nil
}
func NewImageForManifest(repoName string, rawManifest string, managedByOpenShift bool) (*imageapi.Image, error) {
var versioned manifest.Versioned
if err := json.Unmarshal([]byte(rawManifest), &versioned); err != nil {
return nil, err
}
_, desc, err := distribution.UnmarshalManifest(versioned.MediaType, []byte(rawManifest))
if err != nil {
return nil, err
}
annotations := make(map[string]string)
if managedByOpenShift {
annotations[imageapi.ManagedByOpenShiftAnnotation] = "true"
}
img := &imageapi.Image{
ObjectMeta: kapi.ObjectMeta{
Name: desc.Digest.String(),
Annotations: annotations,
},
DockerImageReference: fmt.Sprintf("localhost:5000/%s@%s", repoName, desc.Digest.String()),
DockerImageManifest: string(rawManifest),
}
if err := imageapi.ImageWithMetadata(img); err != nil {
return nil, err
}
return img, nil
}
// PrepareForUpdate clears fields that are not allowed to be set by end users on update.
// It extracts the latest info from the manifest and sets that on the object. It allows a user
// to update the manifest so that it matches the digest (in case an older server stored a manifest
// that was malformed, it can always be corrected).
func (imageStrategy) PrepareForUpdate(obj, old runtime.Object) {
newImage := obj.(*api.Image)
oldImage := old.(*api.Image)
// image metadata cannot be altered
newImage.DockerImageReference = oldImage.DockerImageReference
newImage.DockerImageMetadata = oldImage.DockerImageMetadata
newImage.DockerImageMetadataVersion = oldImage.DockerImageMetadataVersion
newImage.DockerImageLayers = oldImage.DockerImageLayers
// allow an image update that results in the manifest matching the digest (the name)
newManifest := newImage.DockerImageManifest
newImage.DockerImageManifest = oldImage.DockerImageManifest
if newManifest != oldImage.DockerImageManifest && len(newManifest) > 0 {
ok, err := api.ManifestMatchesImage(oldImage, []byte(newManifest))
if err != nil {
utilruntime.HandleError(fmt.Errorf("attempted to validate that a manifest change to %q matched the signature, but failed: %v", oldImage.Name, err))
} else if ok {
newImage.DockerImageManifest = newManifest
}
}
if err := api.ImageWithMetadata(newImage); err != nil {
utilruntime.HandleError(fmt.Errorf("Unable to update image metadata for %q: %v", newImage.Name, err))
}
}
// PrepareForCreate clears fields that are not allowed to be set by end users on creation.
// It extracts the latest information from the manifest (if available) and sets that onto the object.
func (imageStrategy) PrepareForCreate(obj runtime.Object) {
newImage := obj.(*api.Image)
// ignore errors, change in place
if err := api.ImageWithMetadata(newImage); err != nil {
util.HandleError(fmt.Errorf("Unable to update image metadata for %q: %v", newImage.Name, err))
}
}
// PrepareForCreate clears fields that are not allowed to be set by end users on creation.
// It extracts the latest information from the manifest (if available) and sets that onto the object.
func (s imageStrategy) PrepareForCreate(ctx kapi.Context, obj runtime.Object) {
newImage := obj.(*api.Image)
// ignore errors, change in place
if err := api.ImageWithMetadata(newImage); err != nil {
utilruntime.HandleError(fmt.Errorf("Unable to update image metadata for %q: %v", newImage.Name, err))
}
// clear signature fields that will be later set by server once it's able to parse the content
s.clearSignatureDetails(newImage)
}
// imageHasBlob returns true if the image identified by imageName refers to the given blob. The image is
// fetched. If requireManaged is true and the image is not managed (it refers to remote registry), the image
// will not be processed. Fetched image will update local cache of blobs -> repositories with (blobDigest,
// cacheName) pairs.
func imageHasBlob(
r *repository,
cacheName,
imageName,
blobDigest string,
requireManaged bool,
) bool {
context.GetLogger(r.ctx).Debugf("getting image %s", imageName)
image, err := r.getImage(digest.Digest(imageName))
if err != nil {
if kerrors.IsNotFound(err) {
context.GetLogger(r.ctx).Debugf("image %q not found: imageName")
} else {
context.GetLogger(r.ctx).Errorf("failed to get image: %v", err)
}
return false
}
// in case of pullthrough disabled, client won't be able to download a blob belonging to not managed image
// (image stored in external registry), thus don't consider them as candidates
if managed := image.Annotations[imageapi.ManagedByOpenShiftAnnotation]; requireManaged && managed != "true" {
context.GetLogger(r.ctx).Debugf("skipping not managed image")
return false
}
if len(image.DockerImageLayers) == 0 {
if len(image.DockerImageManifestMediaType) > 0 {
// If the media type is set, we can safely assume that the best effort to fill the image layers
// has already been done. There are none.
return false
}
err = imageapi.ImageWithMetadata(image)
if err != nil {
context.GetLogger(r.ctx).Errorf("failed to get metadata for image %s: %v", imageName, err)
return false
}
}
for _, layer := range image.DockerImageLayers {
if layer.Name == blobDigest {
// remember all the layers of matching image
r.rememberLayersOfImage(image, cacheName)
return true
}
}
// only manifest V2 schema2 has docker image config filled where dockerImage.Metadata.id is its digest
if len(image.DockerImageConfig) > 0 && image.DockerImageMetadata.ID == blobDigest {
// remember manifest config reference of schema 2 as well
r.rememberLayersOfImage(image, cacheName)
return true
}
return false
}
func newISTag(tag string, imageStream *api.ImageStream, image *api.Image) (*api.ImageStreamTag, error) {
istagName := api.JoinImageStreamTag(imageStream.Name, tag)
event := api.LatestTaggedImage(imageStream, tag)
if event == nil || len(event.Image) == 0 {
return nil, kapierrors.NewNotFound("imageStreamTag", istagName)
}
ist := &api.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: imageStream.Namespace,
Name: istagName,
CreationTimestamp: event.Created,
Annotations: map[string]string{},
ResourceVersion: imageStream.ResourceVersion,
},
}
// if the imageStream has Spec.Tags[tag].Annotations[k] = v, copy it to the image's annotations
// and add them to the istag's annotations
if imageStream.Spec.Tags != nil {
if tagRef, ok := imageStream.Spec.Tags[tag]; ok {
if image != nil && image.Annotations == nil {
image.Annotations = make(map[string]string)
}
for k, v := range tagRef.Annotations {
ist.Annotations[k] = v
if image != nil {
image.Annotations[k] = v
}
}
}
}
if image != nil {
imageWithMetadata, err := api.ImageWithMetadata(*image)
if err != nil {
return nil, err
}
ist.Image = *imageWithMetadata
} else {
ist.Image = api.Image{}
ist.Image.Name = event.Image
}
// Replace the DockerImageReference with the value from event, which contains
// real value from status. This should fix the problem for v1 registries,
// where mutliple tags point to a single id and only the first image's metadata
// is saved. This in turn will always return the pull spec from the first
// imported image, which might be different than the requested tag.
ist.Image.DockerImageReference = event.DockerImageReference
return ist, nil
}
// PrepareForUpdate clears fields that are not allowed to be set by end users on update.
// It extracts the latest info from the manifest and sets that on the object. It allows a user
// to update the manifest so that it matches the digest (in case an older server stored a manifest
// that was malformed, it can always be corrected).
func (s imageStrategy) PrepareForUpdate(ctx kapi.Context, obj, old runtime.Object) {
newImage := obj.(*api.Image)
oldImage := old.(*api.Image)
// image metadata cannot be altered
newImage.DockerImageMetadata = oldImage.DockerImageMetadata
newImage.DockerImageMetadataVersion = oldImage.DockerImageMetadataVersion
newImage.DockerImageLayers = oldImage.DockerImageLayers
if oldImage.DockerImageSignatures != nil {
newImage.DockerImageSignatures = nil
for _, v := range oldImage.DockerImageSignatures {
newImage.DockerImageSignatures = append(newImage.DockerImageSignatures, v)
}
}
var err error
// allow an image update that results in the manifest matching the digest (the name)
if newImage.DockerImageManifest != oldImage.DockerImageManifest {
ok := true
if len(newImage.DockerImageManifest) > 0 {
ok, err = api.ManifestMatchesImage(oldImage, []byte(newImage.DockerImageManifest))
if err != nil {
utilruntime.HandleError(fmt.Errorf("attempted to validate that a manifest change to %q matched the signature, but failed: %v", oldImage.Name, err))
}
}
if !ok {
newImage.DockerImageManifest = oldImage.DockerImageManifest
}
}
if newImage.DockerImageConfig != oldImage.DockerImageConfig {
ok := true
if len(newImage.DockerImageConfig) > 0 {
ok, err = api.ImageConfigMatchesImage(newImage, []byte(newImage.DockerImageConfig))
if err != nil {
utilruntime.HandleError(fmt.Errorf("attempted to validate that a new config for %q mentioned in the manifest, but failed: %v", oldImage.Name, err))
}
}
if !ok {
newImage.DockerImageConfig = oldImage.DockerImageConfig
}
}
if err = api.ImageWithMetadata(newImage); err != nil {
utilruntime.HandleError(fmt.Errorf("Unable to update image metadata for %q: %v", newImage.Name, err))
}
// clear signature fields that will be later set by server once it's able to parse the content
s.clearSignatureDetails(newImage)
}
// Get retrieves an image by ID that has previously been tagged into an image stream.
// `id` is of the form <repo name>@<image id>.
func (r *REST) Get(ctx kapi.Context, id string) (runtime.Object, error) {
name, imageID, err := ParseNameAndID(id)
if err != nil {
return nil, err
}
repo, err := r.imageStreamRegistry.GetImageStream(ctx, name)
if err != nil {
return nil, err
}
if repo.Status.Tags == nil {
return nil, errors.NewNotFound("imageStreamImage", imageID)
}
set := api.ResolveImageID(repo, imageID)
switch len(set) {
case 1:
imageName := set.List()[0]
image, err := r.imageRegistry.GetImage(ctx, imageName)
if err != nil {
return nil, err
}
imageWithMetadata, err := api.ImageWithMetadata(*image)
if err != nil {
return nil, err
}
if d, err := digest.ParseDigest(imageName); err == nil {
imageName = d.Hex()
}
if len(imageName) > 7 {
imageName = imageName[:7]
}
isi := api.ImageStreamImage{
ObjectMeta: kapi.ObjectMeta{
Namespace: kapi.NamespaceValue(ctx),
Name: fmt.Sprintf("%s@%s", name, imageName),
},
Image: *imageWithMetadata,
}
return &isi, nil
case 0:
return nil, errors.NewNotFound("imageStreamImage", imageID)
default:
return nil, errors.NewConflict("imageStreamImage", imageID, fmt.Errorf("multiple images match the prefix %q: %s", imageID, strings.Join(set.List(), ", ")))
}
}
// deserializedManifestFillImageMetadata fills a given image with metadata.
func (r *repository) deserializedManifestFillImageMetadata(manifest *schema2.DeserializedManifest, image *imageapi.Image) error {
configBytes, err := r.Blobs(r.ctx).Get(r.ctx, manifest.Config.Digest)
if err != nil {
context.GetLogger(r.ctx).Errorf("failed to get image config %s: %v", manifest.Config.Digest.String(), err)
return err
}
image.DockerImageConfig = string(configBytes)
if err := imageapi.ImageWithMetadata(image); err != nil {
return err
}
return nil
}
// signedManifestFillImageMetadata fills a given image with metadata. It also corrects layer sizes with blob sizes. Newer
// Docker client versions don't set layer sizes in the manifest at all. Origin master needs correct layer
// sizes for proper image quota support. That's why we need to fill the metadata in the registry.
func (r *repository) signedManifestFillImageMetadata(manifest *schema1.SignedManifest, image *imageapi.Image) error {
signatures, err := manifest.Signatures()
if err != nil {
return err
}
for _, signDigest := range signatures {
image.DockerImageSignatures = append(image.DockerImageSignatures, signDigest)
}
if err := imageapi.ImageWithMetadata(image); err != nil {
return err
}
refs := manifest.References()
blobSet := sets.NewString()
image.DockerImageMetadata.Size = int64(0)
blobs := r.Blobs(r.ctx)
for i := range image.DockerImageLayers {
layer := &image.DockerImageLayers[i]
// DockerImageLayers represents manifest.Manifest.FSLayers in reversed order
desc, err := blobs.Stat(r.ctx, refs[len(image.DockerImageLayers)-i-1].Digest)
if err != nil {
context.GetLogger(r.ctx).Errorf("failed to stat blobs %s of image %s", layer.Name, image.DockerImageReference)
return err
}
if layer.MediaType == "" {
if desc.MediaType != "" {
layer.MediaType = desc.MediaType
} else {
layer.MediaType = schema1.MediaTypeManifestLayer
}
}
layer.LayerSize = desc.Size
// count empty layer just once (empty layer may actually have non-zero size)
if !blobSet.Has(layer.Name) {
image.DockerImageMetadata.Size += desc.Size
blobSet.Insert(layer.Name)
}
}
if len(image.DockerImageConfig) > 0 && !blobSet.Has(image.DockerImageMetadata.ID) {
blobSet.Insert(image.DockerImageMetadata.ID)
image.DockerImageMetadata.Size += int64(len(image.DockerImageConfig))
}
return nil
}
// PrepareForUpdate clears fields that are not allowed to be set by end users on update.
// It extracts the latest info from the manifest and sets that on the object.
func (imageStrategy) PrepareForUpdate(obj, old runtime.Object) {
newImage := obj.(*api.Image)
oldImage := old.(*api.Image)
// image metadata cannot be altered
newImage.DockerImageReference = oldImage.DockerImageReference
newImage.DockerImageMetadata = oldImage.DockerImageMetadata
newImage.DockerImageManifest = oldImage.DockerImageManifest
newImage.DockerImageMetadataVersion = oldImage.DockerImageMetadataVersion
newImage.DockerImageLayers = oldImage.DockerImageLayers
if err := api.ImageWithMetadata(newImage); err != nil {
util.HandleError(fmt.Errorf("Unable to update image metadata for %q: %v", newImage.Name, err))
}
}
// Get retrieves an image that has been tagged by stream and tag. `id` is of the format
// <stream name>:<tag>.
func (r *REST) Get(ctx kapi.Context, id string) (runtime.Object, error) {
name, tag, err := nameAndTag(id)
if err != nil {
return nil, err
}
stream, err := r.imageStreamRegistry.GetImageStream(ctx, name)
if err != nil {
return nil, err
}
event := api.LatestTaggedImage(stream, tag)
if event == nil || len(event.Image) == 0 {
return nil, errors.NewNotFound("imageStreamTag", id)
}
image, err := r.imageRegistry.GetImage(ctx, event.Image)
if err != nil {
return nil, err
}
// if the stream has Spec.Tags[tag].Annotations[k] = v, copy it to the image's annotations
if stream.Spec.Tags != nil {
if tagRef, ok := stream.Spec.Tags[tag]; ok {
if image.Annotations == nil {
image.Annotations = make(map[string]string)
}
for k, v := range tagRef.Annotations {
image.Annotations[k] = v
}
}
}
imageWithMetadata, err := api.ImageWithMetadata(*image)
if err != nil {
return nil, err
}
ist := api.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: kapi.NamespaceValue(ctx),
Name: id,
CreationTimestamp: event.Created,
},
Image: *imageWithMetadata,
}
return &ist, nil
}
func (h *manifestSchema2Handler) FillImageMetadata(ctx context.Context, image *imageapi.Image) error {
// The manifest.Config references a configuration object for a container by its digest.
// It needs to be fetched in order to fill an image object metadata below.
configBytes, err := h.repo.Blobs(ctx).Get(ctx, h.manifest.Config.Digest)
if err != nil {
context.GetLogger(ctx).Errorf("failed to get image config %s: %v", h.manifest.Config.Digest.String(), err)
return err
}
image.DockerImageConfig = string(configBytes)
if err := imageapi.ImageWithMetadata(image); err != nil {
return err
}
return nil
}
// Get retrieves an image by ID that has previously been tagged into an image stream.
// `id` is of the form <repo name>@<image id>.
func (r *REST) Get(ctx kapi.Context, id string) (runtime.Object, error) {
name, imageID, err := ParseNameAndID(id)
if err != nil {
return nil, err
}
repo, err := r.imageStreamRegistry.GetImageStream(ctx, name)
if err != nil {
return nil, err
}
if repo.Status.Tags == nil {
return nil, errors.NewNotFound("imageStreamImage", imageID)
}
event, err := api.ResolveImageID(repo, imageID)
if err != nil {
return nil, err
}
imageName := event.Image
image, err := r.imageRegistry.GetImage(ctx, imageName)
if err != nil {
return nil, err
}
imageWithMetadata, err := api.ImageWithMetadata(*image)
if err != nil {
return nil, err
}
if d, err := digest.ParseDigest(imageName); err == nil {
imageName = d.Hex()
}
if len(imageName) > 7 {
imageName = imageName[:7]
}
isi := api.ImageStreamImage{
ObjectMeta: kapi.ObjectMeta{
Namespace: kapi.NamespaceValue(ctx),
Name: fmt.Sprintf("%s@%s", name, imageName),
},
Image: *imageWithMetadata,
}
return &isi, nil
}
// Limit is the limit range implementation that checks resource against the
// image limit ranges.
// Implements the LimitRangerActions interface
func (a *imageLimitRangerPlugin) Limit(limitRange *kapi.LimitRange, kind string, obj runtime.Object) error {
isObj, ok := obj.(*imageapi.ImageStreamMapping)
if !ok {
glog.V(5).Infof("%s: received object other than ImageStreamMapping (%T)", PluginName, obj)
return nil
}
image := &isObj.Image
if err := imageapi.ImageWithMetadata(image); err != nil {
return err
}
for _, limit := range limitRange.Spec.Limits {
if err := AdmitImage(image.DockerImageMetadata.Size, limit); err != nil {
return err
}
}
return nil
}
// Get retrieves an image by ID that has previously been tagged into an image stream.
// `id` is of the form <repo name>@<image id>.
func (r *REST) Get(ctx kapi.Context, id string) (runtime.Object, error) {
name, imageID, err := parseNameAndID(id)
if err != nil {
return nil, err
}
repo, err := r.imageStreamRegistry.GetImageStream(ctx, name)
if err != nil {
return nil, err
}
if repo.Status.Tags == nil {
return nil, errors.NewNotFound(api.Resource("imagestreamimage"), id)
}
event, err := api.ResolveImageID(repo, imageID)
if err != nil {
return nil, err
}
imageName := event.Image
image, err := r.imageRegistry.GetImage(ctx, imageName)
if err != nil {
return nil, err
}
if err := api.ImageWithMetadata(image); err != nil {
return nil, err
}
image.DockerImageManifest = ""
isi := api.ImageStreamImage{
ObjectMeta: kapi.ObjectMeta{
Namespace: kapi.NamespaceValue(ctx),
Name: api.MakeImageStreamImageName(name, imageID),
CreationTimestamp: image.ObjectMeta.CreationTimestamp,
},
Image: *image,
}
return &isi, nil
}
func storeTestImage(
ctx context.Context,
reg distribution.Namespace,
imageReference reference.NamedTagged,
schemaVersion int,
managedByOpenShift bool,
) (*imageapi.Image, error) {
repo, err := reg.Repository(ctx, imageReference)
if err != nil {
return nil, fmt.Errorf("unexpected error getting repo %q: %v", imageReference.Name(), err)
}
var (
m distribution.Manifest
m1 schema1.Manifest
)
switch schemaVersion {
case 1:
m1 = schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: imageReference.Name(),
Tag: imageReference.Tag(),
}
case 2:
// TODO
fallthrough
default:
return nil, fmt.Errorf("unsupported manifest version %d", schemaVersion)
}
for i := 0; i < testImageLayerCount; i++ {
rs, ds, err := registrytest.CreateRandomTarFile()
if err != nil {
return nil, fmt.Errorf("unexpected error generating test layer file: %v", err)
}
dgst := digest.Digest(ds)
wr, err := repo.Blobs(ctx).Create(ctx)
if err != nil {
return nil, fmt.Errorf("unexpected error creating test upload: %v", err)
}
defer wr.Close()
n, err := io.Copy(wr, rs)
if err != nil {
return nil, fmt.Errorf("unexpected error copying to upload: %v", err)
}
if schemaVersion == 1 {
m1.FSLayers = append(m1.FSLayers, schema1.FSLayer{BlobSum: dgst})
m1.History = append(m1.History, schema1.History{V1Compatibility: fmt.Sprintf(`{"size":%d}`, n)})
} // TODO v2
if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst, MediaType: schema1.MediaTypeManifestLayer}); err != nil {
return nil, fmt.Errorf("unexpected error finishing upload: %v", err)
}
}
var dgst digest.Digest
var payload []byte
if schemaVersion == 1 {
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
return nil, fmt.Errorf("unexpected error generating private key: %v", err)
}
m, err = schema1.Sign(&m1, pk)
if err != nil {
return nil, fmt.Errorf("error signing manifest: %v", err)
}
_, payload, err = m.Payload()
if err != nil {
return nil, fmt.Errorf("error getting payload %#v", err)
}
dgst = digest.FromBytes(payload)
} //TODO v2
image := &imageapi.Image{
ObjectMeta: kapi.ObjectMeta{
Name: dgst.String(),
},
DockerImageManifest: string(payload),
DockerImageReference: imageReference.Name() + "@" + dgst.String(),
}
if managedByOpenShift {
image.Annotations = map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"}
}
if schemaVersion == 1 {
signedManifest := m.(*schema1.SignedManifest)
signatures, err := signedManifest.Signatures()
if err != nil {
return nil, err
}
for _, signDigest := range signatures {
image.DockerImageSignatures = append(image.DockerImageSignatures, signDigest)
}
}
err = imageapi.ImageWithMetadata(image)
if err != nil {
return nil, fmt.Errorf("failed to fill image with metadata: %v", err)
}
return image, nil
}
func (r *REST) Create(ctx kapi.Context, obj runtime.Object) (runtime.Object, error) {
isi, ok := obj.(*api.ImageStreamImport)
if !ok {
return nil, kapierrors.NewBadRequest(fmt.Sprintf("obj is not an ImageStreamImport: %#v", obj))
}
inputMeta := isi.ObjectMeta
if err := rest.BeforeCreate(Strategy, ctx, obj); err != nil {
return nil, err
}
namespace, ok := kapi.NamespaceFrom(ctx)
if !ok {
return nil, kapierrors.NewBadRequest("a namespace must be specified to import images")
}
secrets, err := r.secrets.ImageStreamSecrets(namespace).Secrets(isi.Name, kapi.ListOptions{})
if err != nil {
util.HandleError(fmt.Errorf("unable to load secrets for namespace %q: %v", namespace, err))
secrets = &kapi.SecretList{}
}
if r.clientFn != nil {
if client := r.clientFn(); client != nil {
ctx = kapi.WithValue(ctx, importer.ContextKeyV1RegistryClient, client)
}
}
credentials := importer.NewCredentialsForSecrets(secrets.Items)
importCtx := importer.NewContext(r.transport).WithCredentials(credentials)
imports := r.importFn(importCtx)
if err := imports.Import(ctx.(gocontext.Context), isi); err != nil {
return nil, kapierrors.NewInternalError(err)
}
// TODO: perform the transformation of the image stream and return it with the ISI if import is false
// so that clients can see what the resulting object would look like.
if !isi.Spec.Import {
clearManifests(isi)
return isi, nil
}
create := false
stream, err := r.streams.GetImageStream(ctx, isi.Name)
if err != nil {
if !kapierrors.IsNotFound(err) {
return nil, err
}
// consistency check, stream must exist
if len(inputMeta.ResourceVersion) > 0 || len(inputMeta.UID) > 0 {
return nil, err
}
create = true
stream = &api.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Name: isi.Name,
Namespace: namespace,
Generation: 0,
},
}
} else {
if len(inputMeta.ResourceVersion) > 0 && inputMeta.ResourceVersion != stream.ResourceVersion {
glog.V(4).Infof("DEBUG: mismatch between requested UID %s and located UID %s", inputMeta.UID, stream.UID)
return nil, kapierrors.NewConflict("imageStream", inputMeta.Name, fmt.Errorf("the image stream was updated from %q to %q", inputMeta.ResourceVersion, stream.ResourceVersion))
}
if len(inputMeta.UID) > 0 && inputMeta.UID != stream.UID {
glog.V(4).Infof("DEBUG: mismatch between requested UID %s and located UID %s", inputMeta.UID, stream.UID)
return nil, kapierrors.NewNotFound("imageStream", inputMeta.Name)
}
}
if stream.Annotations == nil {
stream.Annotations = make(map[string]string)
}
now := unversioned.Now()
stream.Annotations[api.DockerImageRepositoryCheckAnnotation] = now.UTC().Format(time.RFC3339)
gen := stream.Generation + 1
zero := int64(0)
importedImages := make(map[string]error)
updatedImages := make(map[string]*api.Image)
if spec := isi.Spec.Repository; spec != nil {
for i, imageStatus := range isi.Status.Repository.Images {
image := imageStatus.Image
if image == nil {
continue
}
// update the spec tag
ref, err := api.ParseDockerImageReference(image.DockerImageReference)
if err != nil {
// ???
continue
}
tag := ref.Tag
if len(imageStatus.Tag) > 0 {
tag = imageStatus.Tag
}
if _, ok := stream.Spec.Tags[tag]; !ok {
if stream.Spec.Tags == nil {
stream.Spec.Tags = make(map[string]api.TagReference)
}
stream.Spec.Tags[tag] = api.TagReference{
From: &kapi.ObjectReference{
Kind: "DockerImage",
Name: image.DockerImageReference,
},
Generation: &gen,
ImportPolicy: api.TagImportPolicy{Insecure: spec.ImportPolicy.Insecure},
}
}
// import or reuse the image
importErr, imported := importedImages[image.Name]
if importErr != nil {
api.SetTagConditions(stream, tag, newImportFailedCondition(err, gen, now))
}
pullSpec, _ := api.MostAccuratePullSpec(image.DockerImageReference, image.Name, "")
api.AddTagEventToImageStream(stream, tag, api.TagEvent{
Created: now,
DockerImageReference: pullSpec,
Image: image.Name,
Generation: gen,
})
if imported {
if updatedImage, ok := updatedImages[image.Name]; ok {
isi.Status.Repository.Images[i].Image = updatedImage
}
continue
}
// establish the image into the store
updated, err := r.images.Create(ctx, image)
switch {
case kapierrors.IsAlreadyExists(err):
if err := api.ImageWithMetadata(image); err != nil {
glog.V(4).Infof("Unable to update image metadata during image import when image already exists %q: err", image.Name, err)
}
updated = image
fallthrough
case err == nil:
updatedImage := updated.(*api.Image)
updatedImages[image.Name] = updatedImage
isi.Status.Repository.Images[i].Image = updatedImage
importedImages[image.Name] = nil
default:
importedImages[image.Name] = err
}
}
}
for i, spec := range isi.Spec.Images {
if spec.To == nil {
continue
}
tag := spec.To.Name
if stream.Spec.Tags == nil {
stream.Spec.Tags = make(map[string]api.TagReference)
}
specTag := stream.Spec.Tags[tag]
from := spec.From
specTag.From = &from
specTag.Generation = &zero
specTag.ImportPolicy.Insecure = spec.ImportPolicy.Insecure
stream.Spec.Tags[tag] = specTag
status := isi.Status.Images[i]
if status.Image == nil || status.Status.Status == unversioned.StatusFailure {
message := status.Status.Message
if len(message) == 0 {
message = "unknown error prevented import"
}
api.SetTagConditions(stream, tag, api.TagEventCondition{
Type: api.ImportSuccess,
Status: kapi.ConditionFalse,
Message: message,
Reason: string(status.Status.Reason),
Generation: gen,
LastTransitionTime: now,
})
continue
}
image := status.Image
importErr, imported := importedImages[image.Name]
if importErr != nil {
api.SetTagConditions(stream, tag, newImportFailedCondition(err, gen, now))
}
pullSpec, _ := api.MostAccuratePullSpec(image.DockerImageReference, image.Name, "")
api.AddTagEventToImageStream(stream, tag, api.TagEvent{
Created: now,
DockerImageReference: pullSpec,
Image: image.Name,
Generation: gen,
})
if imported {
continue
}
_, err = r.images.Create(ctx, image)
if kapierrors.IsAlreadyExists(err) {
err = nil
}
importedImages[image.Name] = err
}
// TODO: should we allow partial failure?
for _, err := range importedImages {
if err != nil {
return nil, err
}
}
clearManifests(isi)
if create {
obj, err = r.internalStreams.Create(ctx, stream)
} else {
obj, _, err = r.internalStreams.Update(ctx, stream)
}
if err != nil {
return nil, err
}
isi.Status.Import = obj.(*api.ImageStream)
return isi, nil
}
// newISTag initializes an image stream tag from an image stream and image. The allowEmptyEvent will create a tag even
// in the event that the status tag does does not exist yet (no image has successfully been tagged) or the image is nil.
func newISTag(tag string, imageStream *api.ImageStream, image *api.Image, allowEmptyEvent bool) (*api.ImageStreamTag, error) {
istagName := api.JoinImageStreamTag(imageStream.Name, tag)
event := api.LatestTaggedImage(imageStream, tag)
if event == nil || len(event.Image) == 0 {
if !allowEmptyEvent {
return nil, kapierrors.NewNotFound(api.Resource("imagestreamtags"), istagName)
}
event = &api.TagEvent{
Created: imageStream.CreationTimestamp,
}
}
ist := &api.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: imageStream.Namespace,
Name: istagName,
CreationTimestamp: event.Created,
Annotations: map[string]string{},
ResourceVersion: imageStream.ResourceVersion,
},
Generation: event.Generation,
Conditions: imageStream.Status.Tags[tag].Conditions,
}
if imageStream.Spec.Tags != nil {
if tagRef, ok := imageStream.Spec.Tags[tag]; ok {
// copy the spec tag
ist.Tag = &tagRef
if from := ist.Tag.From; from != nil {
copied := *from
ist.Tag.From = &copied
}
if gen := ist.Tag.Generation; gen != nil {
copied := *gen
ist.Tag.Generation = &copied
}
// if the imageStream has Spec.Tags[tag].Annotations[k] = v, copy it to the image's annotations
// and add them to the istag's annotations
if image != nil && image.Annotations == nil {
image.Annotations = make(map[string]string)
}
for k, v := range tagRef.Annotations {
ist.Annotations[k] = v
if image != nil {
image.Annotations[k] = v
}
}
}
}
if image != nil {
if err := api.ImageWithMetadata(image); err != nil {
return nil, err
}
image.DockerImageManifest = ""
ist.Image = *image
} else {
ist.Image = api.Image{}
ist.Image.Name = event.Image
}
// Replace the DockerImageReference with the value from event, which contains
// real value from status. This should fix the problem for v1 registries,
// where mutliple tags point to a single id and only the first image's metadata
// is saved. This in turn will always return the pull spec from the first
// imported image, which might be different than the requested tag.
ist.Image.DockerImageReference = event.DockerImageReference
return ist, nil
}
// importSuccessful records a successful import into an image stream, setting the spec tag, status tag or conditions, and ensuring
// the image is created in etcd. Images are cached so they are not created multiple times in a row (when multiple tags point to the
// same image), and a failure to persist the image will be summarized before we update the stream. If an image was imported by this
// operation, it *replaces* the imported image (from the remote repository) with the updated image.
func (r *REST) importSuccessful(
ctx kapi.Context,
image *api.Image, stream *api.ImageStream, tag string, from string, nextGeneration int64, now unversioned.Time, importPolicy api.TagImportPolicy,
importedImages map[string]error, updatedImages map[string]*api.Image,
) (*api.Image, bool) {
Strategy.PrepareImageForCreate(image)
pullSpec, _ := api.MostAccuratePullSpec(image.DockerImageReference, image.Name, "")
tagEvent := api.TagEvent{
Created: now,
DockerImageReference: pullSpec,
Image: image.Name,
Generation: nextGeneration,
}
if stream.Spec.Tags == nil {
stream.Spec.Tags = make(map[string]api.TagReference)
}
// ensure the spec and status tag match the imported image
changed := api.DifferentTagEvent(stream, tag, tagEvent)
specTag, ok := stream.Spec.Tags[tag]
if changed || !ok {
specTag = ensureSpecTag(stream, tag, from, importPolicy, true)
api.AddTagEventToImageStream(stream, tag, tagEvent)
}
// always reset the import policy
specTag.ImportPolicy = importPolicy
stream.Spec.Tags[tag] = specTag
// import or reuse the image, and ensure tag conditions are set
importErr, alreadyImported := importedImages[image.Name]
if importErr != nil {
api.SetTagConditions(stream, tag, newImportFailedCondition(importErr, nextGeneration, now))
} else {
api.SetTagConditions(stream, tag)
}
// create the image if it does not exist, otherwise cache the updated status from the store for use by other tags
if alreadyImported {
if updatedImage, ok := updatedImages[image.Name]; ok {
return updatedImage, true
}
return nil, false
}
updated, err := r.images.Create(ctx, image)
switch {
case kapierrors.IsAlreadyExists(err):
if err := api.ImageWithMetadata(image); err != nil {
glog.V(4).Infof("Unable to update image metadata during image import when image already exists %q: err", image.Name, err)
}
updated = image
fallthrough
case err == nil:
updatedImage := updated.(*api.Image)
updatedImages[image.Name] = updatedImage
//isi.Status.Repository.Images[i].Image = updatedImage
importedImages[image.Name] = nil
return updatedImage, true
default:
importedImages[image.Name] = err
}
return nil, false
}
// importRepositoryFromDocker loads the tags and images requested in the passed importRepository, obeying the
// optional rate limiter. Errors are set onto the individual tags and digest objects.
func (isi *ImageStreamImporter) importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetriever, repository *importRepository, limiter flowcontrol.RateLimiter) {
glog.V(5).Infof("importing remote Docker repository registry=%s repository=%s insecure=%t", repository.Registry, repository.Name, repository.Insecure)
// retrieve the repository
repo, err := retriever.Repository(ctx, repository.Registry, repository.Name, repository.Insecure)
if err != nil {
glog.V(5).Infof("unable to access repository %#v: %#v", repository, err)
switch {
case err == reference.ErrReferenceInvalidFormat:
err = field.Invalid(field.NewPath("from", "name"), repository.Name, "the provided repository name is not valid")
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
case strings.Contains(err.Error(), "tls: oversized record received with length") && !repository.Insecure:
err = kapierrors.NewBadRequest("this repository is HTTP only and requires the insecure flag to import")
case strings.HasSuffix(err.Error(), "no basic auth credentials"):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact()))
case strings.HasSuffix(err.Error(), "does not support v2 API"):
importRepositoryFromDockerV1(ctx, repository, limiter)
return
}
applyErrorToRepository(repository, err)
return
}
// get a manifest context
s, err := repo.Manifests(ctx)
if err != nil {
glog.V(5).Infof("unable to access manifests for repository %#v: %#v", repository, err)
switch {
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
case strings.HasSuffix(err.Error(), "no basic auth credentials"):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact()))
}
applyErrorToRepository(repository, err)
return
}
// get a blob context
b := repo.Blobs(ctx)
// if repository import is requested (MaximumTags), attempt to load the tags, sort them, and request the first N
if count := repository.MaximumTags; count > 0 || count == -1 {
tags, err := repo.Tags(ctx).All(ctx)
if err != nil {
glog.V(5).Infof("unable to access tags for repository %#v: %#v", repository, err)
switch {
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
}
repository.Err = err
return
}
// some images on the Hub have empty tags - treat those as "latest"
set := sets.NewString(tags...)
if set.Has("") {
set.Delete("")
set.Insert(api.DefaultImageTag)
}
tags = set.List()
// include only the top N tags in the result, put the rest in AdditionalTags
api.PrioritizeTags(tags)
for _, s := range tags {
if count <= 0 && repository.MaximumTags != -1 {
repository.AdditionalTags = append(repository.AdditionalTags, s)
continue
}
count--
repository.Tags = append(repository.Tags, importTag{
Name: s,
})
}
}
// load digests
for i := range repository.Digests {
importDigest := &repository.Digests[i]
if importDigest.Err != nil || importDigest.Image != nil {
continue
}
d, err := digest.ParseDigest(importDigest.Name)
if err != nil {
importDigest.Err = err
continue
}
limiter.Accept()
manifest, err := s.Get(ctx, d)
if err != nil {
glog.V(5).Infof("unable to access digest %q for repository %#v: %#v", d, repository, err)
importDigest.Err = formatRepositoryError(repository, "", importDigest.Name, err)
continue
}
if signedManifest, isSchema1 := manifest.(*schema1.SignedManifest); isSchema1 {
importDigest.Image, err = schema1ToImage(signedManifest, d)
} else if deserializedManifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 {
imageConfig, err := b.Get(ctx, deserializedManifest.Config.Digest)
if err != nil {
glog.V(5).Infof("unable to access the image config using digest %q for repository %#v: %#v", d, repository, err)
if isDockerError(err, v2.ErrorCodeManifestUnknown) {
ref := repository.Ref
ref.ID = deserializedManifest.Config.Digest.String()
importDigest.Err = kapierrors.NewNotFound(api.Resource("dockerimage"), ref.Exact())
} else {
importDigest.Err = formatRepositoryError(repository, "", importDigest.Name, err)
}
continue
}
importDigest.Image, err = schema2ToImage(deserializedManifest, imageConfig, d)
} else {
glog.V(5).Infof("unsupported manifest type: %T", manifest)
continue
}
if err != nil {
importDigest.Err = err
continue
}
if err := api.ImageWithMetadata(importDigest.Image); err != nil {
importDigest.Err = err
continue
}
if importDigest.Image.DockerImageMetadata.Size == 0 {
if err := isi.calculateImageSize(ctx, repo, importDigest.Image); err != nil {
importDigest.Err = err
continue
}
}
}
for i := range repository.Tags {
importTag := &repository.Tags[i]
if importTag.Err != nil || importTag.Image != nil {
continue
}
limiter.Accept()
desc, err := repo.Tags(ctx).Get(ctx, importTag.Name)
if err != nil {
glog.V(5).Infof("unable to get tag %q for repository %#v: %#v", importTag.Name, repository, err)
importTag.Err = formatRepositoryError(repository, importTag.Name, "", err)
continue
}
manifest, err := s.Get(ctx, desc.Digest)
if err != nil {
glog.V(5).Infof("unable to access digest %q for tag %q for repository %#v: %#v", desc.Digest, importTag.Name, repository, err)
importTag.Err = formatRepositoryError(repository, importTag.Name, "", err)
continue
}
if signedManifest, isSchema1 := manifest.(*schema1.SignedManifest); isSchema1 {
importTag.Image, err = schema1ToImage(signedManifest, "")
} else if deserializedManifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 {
imageConfig, err := b.Get(ctx, deserializedManifest.Config.Digest)
if err != nil {
glog.V(5).Infof("unable to access image config using digest %q for tag %q for repository %#v: %#v", desc.Digest, importTag.Name, repository, err)
importTag.Err = formatRepositoryError(repository, importTag.Name, "", err)
continue
}
importTag.Image, err = schema2ToImage(deserializedManifest, imageConfig, "")
} else {
glog.V(5).Infof("unsupported manifest type: %T", manifest)
continue
}
if err != nil {
importTag.Err = err
continue
}
if err := api.ImageWithMetadata(importTag.Image); err != nil {
importTag.Err = err
continue
}
if importTag.Image.DockerImageMetadata.Size == 0 {
if err := isi.calculateImageSize(ctx, repo, importTag.Image); err != nil {
importTag.Err = err
continue
}
}
}
}
// importRepositoryFromDocker loads the tags and images requested in the passed importRepository, obeying the
// optional rate limiter. Errors are set onto the individual tags and digest objects.
func importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetriever, repository *importRepository, limiter util.RateLimiter) {
// retrieve the repository
repo, err := retriever.Repository(ctx, repository.Registry, repository.Name, repository.Insecure)
if err != nil {
glog.V(5).Infof("unable to access repository %#v: %#v", repository, err)
switch {
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound("DockerImage", repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
case strings.Contains(err.Error(), "tls: oversized record received with length") && !repository.Insecure:
err = kapierrors.NewBadRequest("this repository is HTTP only and requires the insecure flag to import")
case strings.HasSuffix(err.Error(), "no basic auth credentials"):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact()))
case strings.HasSuffix(err.Error(), "does not support v2 API"):
importRepositoryFromDockerV1(ctx, repository, limiter)
return
}
applyErrorToRepository(repository, err)
return
}
// get a manifest context
s, err := repo.Manifests(ctx)
if err != nil {
glog.V(5).Infof("unable to access manifests for repository %#v: %#v", repository, err)
switch {
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound("DockerImage", repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
case strings.HasSuffix(err.Error(), "no basic auth credentials"):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact()))
}
applyErrorToRepository(repository, err)
return
}
// if repository import is requested (MaximumTags), attempt to load the tags, sort them, and request the first N
if count := repository.MaximumTags; count > 0 || count == -1 {
tags, err := s.Tags()
if err != nil {
glog.V(5).Infof("unable to access tags for repository %#v: %#v", repository, err)
switch {
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound("DockerImage", repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
}
repository.Err = err
return
}
// some images on the Hub have empty tags - treat those as "latest"
set := sets.NewString(tags...)
if set.Has("") {
set.Delete("")
set.Insert(api.DefaultImageTag)
}
tags = set.List()
// include only the top N tags in the result, put the rest in AdditionalTags
api.PrioritizeTags(tags)
for _, s := range tags {
if count <= 0 && repository.MaximumTags != -1 {
repository.AdditionalTags = append(repository.AdditionalTags, s)
continue
}
count--
repository.Tags = append(repository.Tags, importTag{
Name: s,
})
}
}
// load digests
for i := range repository.Digests {
importDigest := &repository.Digests[i]
if importDigest.Err != nil || importDigest.Image != nil {
continue
}
d, err := digest.ParseDigest(importDigest.Name)
if err != nil {
importDigest.Err = err
continue
}
limiter.Accept()
m, err := s.Get(d)
if err != nil {
glog.V(5).Infof("unable to access digest %q for repository %#v: %#v", d, repository, err)
switch {
case isDockerError(err, v2.ErrorCodeManifestUnknown):
ref := repository.Ref
ref.Tag, ref.ID = "", importDigest.Name
err = kapierrors.NewNotFound("DockerImage", ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
case strings.HasSuffix(err.Error(), "no basic auth credentials"):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
}
importDigest.Err = err
continue
}
importDigest.Image, err = schema1ToImage(m, d)
if err != nil {
importDigest.Err = err
continue
}
if err := api.ImageWithMetadata(importDigest.Image); err != nil {
importDigest.Err = err
continue
}
}
for i := range repository.Tags {
importTag := &repository.Tags[i]
if importTag.Err != nil || importTag.Image != nil {
continue
}
limiter.Accept()
m, err := s.GetByTag(importTag.Name)
if err != nil {
glog.V(5).Infof("unable to access tag %q for repository %#v: %#v", importTag.Name, repository, err)
switch {
case isDockerError(err, v2.ErrorCodeManifestUnknown):
ref := repository.Ref
ref.Tag = importTag.Name
err = kapierrors.NewNotFound("DockerImage", ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
case strings.HasSuffix(err.Error(), "no basic auth credentials"):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
}
importTag.Err = err
continue
}
importTag.Image, err = schema1ToImage(m, "")
if err != nil {
importTag.Err = err
continue
}
if err := api.ImageWithMetadata(importTag.Image); err != nil {
importTag.Err = err
continue
}
}
}