// fillImageWithMetadata fills a given image with metadata. Also correct layer sizes with blob sizes. Newer // Docker client versions don't set layer sizes in the manifest at all. Origin master needs correct layer // sizes for proper image quota support. That's why we need to fill the metadata in the registry. func (r *repository) fillImageWithMetadata(manifest *schema1.SignedManifest, image *imageapi.Image) error { if err := imageapi.ImageWithMetadata(image); err != nil { return err } layerSet := sets.NewString() size := int64(0) blobs := r.Blobs(r.ctx) for i := range image.DockerImageLayers { layer := &image.DockerImageLayers[i] // DockerImageLayers represents manifest.Manifest.FSLayers in reversed order desc, err := blobs.Stat(r.ctx, manifest.Manifest.FSLayers[len(image.DockerImageLayers)-i-1].BlobSum) if err != nil { context.GetLogger(r.ctx).Errorf("Failed to stat blobs %s of image %s", layer.Name, image.DockerImageReference) return err } layer.Size = desc.Size // count empty layer just once (empty layer may actually have non-zero size) if !layerSet.Has(layer.Name) { size += desc.Size layerSet.Insert(layer.Name) } } image.DockerImageMetadata.Size = size context.GetLogger(r.ctx).Infof("Total size of image %s with docker ref %s: %d", image.Name, image.DockerImageReference, size) return nil }
func NewImageForManifest(repoName string, rawManifest string, managedByOpenShift bool) (*imageapi.Image, error) { var versioned manifest.Versioned if err := json.Unmarshal([]byte(rawManifest), &versioned); err != nil { return nil, err } _, desc, err := distribution.UnmarshalManifest(versioned.MediaType, []byte(rawManifest)) if err != nil { return nil, err } annotations := make(map[string]string) if managedByOpenShift { annotations[imageapi.ManagedByOpenShiftAnnotation] = "true" } img := &imageapi.Image{ ObjectMeta: kapi.ObjectMeta{ Name: desc.Digest.String(), Annotations: annotations, }, DockerImageReference: fmt.Sprintf("localhost:5000/%s@%s", repoName, desc.Digest.String()), DockerImageManifest: string(rawManifest), } if err := imageapi.ImageWithMetadata(img); err != nil { return nil, err } return img, nil }
// PrepareForUpdate clears fields that are not allowed to be set by end users on update. // It extracts the latest info from the manifest and sets that on the object. It allows a user // to update the manifest so that it matches the digest (in case an older server stored a manifest // that was malformed, it can always be corrected). func (imageStrategy) PrepareForUpdate(obj, old runtime.Object) { newImage := obj.(*api.Image) oldImage := old.(*api.Image) // image metadata cannot be altered newImage.DockerImageReference = oldImage.DockerImageReference newImage.DockerImageMetadata = oldImage.DockerImageMetadata newImage.DockerImageMetadataVersion = oldImage.DockerImageMetadataVersion newImage.DockerImageLayers = oldImage.DockerImageLayers // allow an image update that results in the manifest matching the digest (the name) newManifest := newImage.DockerImageManifest newImage.DockerImageManifest = oldImage.DockerImageManifest if newManifest != oldImage.DockerImageManifest && len(newManifest) > 0 { ok, err := api.ManifestMatchesImage(oldImage, []byte(newManifest)) if err != nil { utilruntime.HandleError(fmt.Errorf("attempted to validate that a manifest change to %q matched the signature, but failed: %v", oldImage.Name, err)) } else if ok { newImage.DockerImageManifest = newManifest } } if err := api.ImageWithMetadata(newImage); err != nil { utilruntime.HandleError(fmt.Errorf("Unable to update image metadata for %q: %v", newImage.Name, err)) } }
// PrepareForCreate clears fields that are not allowed to be set by end users on creation. // It extracts the latest information from the manifest (if available) and sets that onto the object. func (imageStrategy) PrepareForCreate(obj runtime.Object) { newImage := obj.(*api.Image) // ignore errors, change in place if err := api.ImageWithMetadata(newImage); err != nil { util.HandleError(fmt.Errorf("Unable to update image metadata for %q: %v", newImage.Name, err)) } }
// PrepareForCreate clears fields that are not allowed to be set by end users on creation. // It extracts the latest information from the manifest (if available) and sets that onto the object. func (s imageStrategy) PrepareForCreate(ctx kapi.Context, obj runtime.Object) { newImage := obj.(*api.Image) // ignore errors, change in place if err := api.ImageWithMetadata(newImage); err != nil { utilruntime.HandleError(fmt.Errorf("Unable to update image metadata for %q: %v", newImage.Name, err)) } // clear signature fields that will be later set by server once it's able to parse the content s.clearSignatureDetails(newImage) }
// imageHasBlob returns true if the image identified by imageName refers to the given blob. The image is // fetched. If requireManaged is true and the image is not managed (it refers to remote registry), the image // will not be processed. Fetched image will update local cache of blobs -> repositories with (blobDigest, // cacheName) pairs. func imageHasBlob( r *repository, cacheName, imageName, blobDigest string, requireManaged bool, ) bool { context.GetLogger(r.ctx).Debugf("getting image %s", imageName) image, err := r.getImage(digest.Digest(imageName)) if err != nil { if kerrors.IsNotFound(err) { context.GetLogger(r.ctx).Debugf("image %q not found: imageName") } else { context.GetLogger(r.ctx).Errorf("failed to get image: %v", err) } return false } // in case of pullthrough disabled, client won't be able to download a blob belonging to not managed image // (image stored in external registry), thus don't consider them as candidates if managed := image.Annotations[imageapi.ManagedByOpenShiftAnnotation]; requireManaged && managed != "true" { context.GetLogger(r.ctx).Debugf("skipping not managed image") return false } if len(image.DockerImageLayers) == 0 { if len(image.DockerImageManifestMediaType) > 0 { // If the media type is set, we can safely assume that the best effort to fill the image layers // has already been done. There are none. return false } err = imageapi.ImageWithMetadata(image) if err != nil { context.GetLogger(r.ctx).Errorf("failed to get metadata for image %s: %v", imageName, err) return false } } for _, layer := range image.DockerImageLayers { if layer.Name == blobDigest { // remember all the layers of matching image r.rememberLayersOfImage(image, cacheName) return true } } // only manifest V2 schema2 has docker image config filled where dockerImage.Metadata.id is its digest if len(image.DockerImageConfig) > 0 && image.DockerImageMetadata.ID == blobDigest { // remember manifest config reference of schema 2 as well r.rememberLayersOfImage(image, cacheName) return true } return false }
func newISTag(tag string, imageStream *api.ImageStream, image *api.Image) (*api.ImageStreamTag, error) { istagName := api.JoinImageStreamTag(imageStream.Name, tag) event := api.LatestTaggedImage(imageStream, tag) if event == nil || len(event.Image) == 0 { return nil, kapierrors.NewNotFound("imageStreamTag", istagName) } ist := &api.ImageStreamTag{ ObjectMeta: kapi.ObjectMeta{ Namespace: imageStream.Namespace, Name: istagName, CreationTimestamp: event.Created, Annotations: map[string]string{}, ResourceVersion: imageStream.ResourceVersion, }, } // if the imageStream has Spec.Tags[tag].Annotations[k] = v, copy it to the image's annotations // and add them to the istag's annotations if imageStream.Spec.Tags != nil { if tagRef, ok := imageStream.Spec.Tags[tag]; ok { if image != nil && image.Annotations == nil { image.Annotations = make(map[string]string) } for k, v := range tagRef.Annotations { ist.Annotations[k] = v if image != nil { image.Annotations[k] = v } } } } if image != nil { imageWithMetadata, err := api.ImageWithMetadata(*image) if err != nil { return nil, err } ist.Image = *imageWithMetadata } else { ist.Image = api.Image{} ist.Image.Name = event.Image } // Replace the DockerImageReference with the value from event, which contains // real value from status. This should fix the problem for v1 registries, // where mutliple tags point to a single id and only the first image's metadata // is saved. This in turn will always return the pull spec from the first // imported image, which might be different than the requested tag. ist.Image.DockerImageReference = event.DockerImageReference return ist, nil }
// PrepareForUpdate clears fields that are not allowed to be set by end users on update. // It extracts the latest info from the manifest and sets that on the object. It allows a user // to update the manifest so that it matches the digest (in case an older server stored a manifest // that was malformed, it can always be corrected). func (s imageStrategy) PrepareForUpdate(ctx kapi.Context, obj, old runtime.Object) { newImage := obj.(*api.Image) oldImage := old.(*api.Image) // image metadata cannot be altered newImage.DockerImageMetadata = oldImage.DockerImageMetadata newImage.DockerImageMetadataVersion = oldImage.DockerImageMetadataVersion newImage.DockerImageLayers = oldImage.DockerImageLayers if oldImage.DockerImageSignatures != nil { newImage.DockerImageSignatures = nil for _, v := range oldImage.DockerImageSignatures { newImage.DockerImageSignatures = append(newImage.DockerImageSignatures, v) } } var err error // allow an image update that results in the manifest matching the digest (the name) if newImage.DockerImageManifest != oldImage.DockerImageManifest { ok := true if len(newImage.DockerImageManifest) > 0 { ok, err = api.ManifestMatchesImage(oldImage, []byte(newImage.DockerImageManifest)) if err != nil { utilruntime.HandleError(fmt.Errorf("attempted to validate that a manifest change to %q matched the signature, but failed: %v", oldImage.Name, err)) } } if !ok { newImage.DockerImageManifest = oldImage.DockerImageManifest } } if newImage.DockerImageConfig != oldImage.DockerImageConfig { ok := true if len(newImage.DockerImageConfig) > 0 { ok, err = api.ImageConfigMatchesImage(newImage, []byte(newImage.DockerImageConfig)) if err != nil { utilruntime.HandleError(fmt.Errorf("attempted to validate that a new config for %q mentioned in the manifest, but failed: %v", oldImage.Name, err)) } } if !ok { newImage.DockerImageConfig = oldImage.DockerImageConfig } } if err = api.ImageWithMetadata(newImage); err != nil { utilruntime.HandleError(fmt.Errorf("Unable to update image metadata for %q: %v", newImage.Name, err)) } // clear signature fields that will be later set by server once it's able to parse the content s.clearSignatureDetails(newImage) }
// Get retrieves an image by ID that has previously been tagged into an image stream. // `id` is of the form <repo name>@<image id>. func (r *REST) Get(ctx kapi.Context, id string) (runtime.Object, error) { name, imageID, err := ParseNameAndID(id) if err != nil { return nil, err } repo, err := r.imageStreamRegistry.GetImageStream(ctx, name) if err != nil { return nil, err } if repo.Status.Tags == nil { return nil, errors.NewNotFound("imageStreamImage", imageID) } set := api.ResolveImageID(repo, imageID) switch len(set) { case 1: imageName := set.List()[0] image, err := r.imageRegistry.GetImage(ctx, imageName) if err != nil { return nil, err } imageWithMetadata, err := api.ImageWithMetadata(*image) if err != nil { return nil, err } if d, err := digest.ParseDigest(imageName); err == nil { imageName = d.Hex() } if len(imageName) > 7 { imageName = imageName[:7] } isi := api.ImageStreamImage{ ObjectMeta: kapi.ObjectMeta{ Namespace: kapi.NamespaceValue(ctx), Name: fmt.Sprintf("%s@%s", name, imageName), }, Image: *imageWithMetadata, } return &isi, nil case 0: return nil, errors.NewNotFound("imageStreamImage", imageID) default: return nil, errors.NewConflict("imageStreamImage", imageID, fmt.Errorf("multiple images match the prefix %q: %s", imageID, strings.Join(set.List(), ", "))) } }
// deserializedManifestFillImageMetadata fills a given image with metadata. func (r *repository) deserializedManifestFillImageMetadata(manifest *schema2.DeserializedManifest, image *imageapi.Image) error { configBytes, err := r.Blobs(r.ctx).Get(r.ctx, manifest.Config.Digest) if err != nil { context.GetLogger(r.ctx).Errorf("failed to get image config %s: %v", manifest.Config.Digest.String(), err) return err } image.DockerImageConfig = string(configBytes) if err := imageapi.ImageWithMetadata(image); err != nil { return err } return nil }
// signedManifestFillImageMetadata fills a given image with metadata. It also corrects layer sizes with blob sizes. Newer // Docker client versions don't set layer sizes in the manifest at all. Origin master needs correct layer // sizes for proper image quota support. That's why we need to fill the metadata in the registry. func (r *repository) signedManifestFillImageMetadata(manifest *schema1.SignedManifest, image *imageapi.Image) error { signatures, err := manifest.Signatures() if err != nil { return err } for _, signDigest := range signatures { image.DockerImageSignatures = append(image.DockerImageSignatures, signDigest) } if err := imageapi.ImageWithMetadata(image); err != nil { return err } refs := manifest.References() blobSet := sets.NewString() image.DockerImageMetadata.Size = int64(0) blobs := r.Blobs(r.ctx) for i := range image.DockerImageLayers { layer := &image.DockerImageLayers[i] // DockerImageLayers represents manifest.Manifest.FSLayers in reversed order desc, err := blobs.Stat(r.ctx, refs[len(image.DockerImageLayers)-i-1].Digest) if err != nil { context.GetLogger(r.ctx).Errorf("failed to stat blobs %s of image %s", layer.Name, image.DockerImageReference) return err } if layer.MediaType == "" { if desc.MediaType != "" { layer.MediaType = desc.MediaType } else { layer.MediaType = schema1.MediaTypeManifestLayer } } layer.LayerSize = desc.Size // count empty layer just once (empty layer may actually have non-zero size) if !blobSet.Has(layer.Name) { image.DockerImageMetadata.Size += desc.Size blobSet.Insert(layer.Name) } } if len(image.DockerImageConfig) > 0 && !blobSet.Has(image.DockerImageMetadata.ID) { blobSet.Insert(image.DockerImageMetadata.ID) image.DockerImageMetadata.Size += int64(len(image.DockerImageConfig)) } return nil }
// PrepareForUpdate clears fields that are not allowed to be set by end users on update. // It extracts the latest info from the manifest and sets that on the object. func (imageStrategy) PrepareForUpdate(obj, old runtime.Object) { newImage := obj.(*api.Image) oldImage := old.(*api.Image) // image metadata cannot be altered newImage.DockerImageReference = oldImage.DockerImageReference newImage.DockerImageMetadata = oldImage.DockerImageMetadata newImage.DockerImageManifest = oldImage.DockerImageManifest newImage.DockerImageMetadataVersion = oldImage.DockerImageMetadataVersion newImage.DockerImageLayers = oldImage.DockerImageLayers if err := api.ImageWithMetadata(newImage); err != nil { util.HandleError(fmt.Errorf("Unable to update image metadata for %q: %v", newImage.Name, err)) } }
// Get retrieves an image that has been tagged by stream and tag. `id` is of the format // <stream name>:<tag>. func (r *REST) Get(ctx kapi.Context, id string) (runtime.Object, error) { name, tag, err := nameAndTag(id) if err != nil { return nil, err } stream, err := r.imageStreamRegistry.GetImageStream(ctx, name) if err != nil { return nil, err } event := api.LatestTaggedImage(stream, tag) if event == nil || len(event.Image) == 0 { return nil, errors.NewNotFound("imageStreamTag", id) } image, err := r.imageRegistry.GetImage(ctx, event.Image) if err != nil { return nil, err } // if the stream has Spec.Tags[tag].Annotations[k] = v, copy it to the image's annotations if stream.Spec.Tags != nil { if tagRef, ok := stream.Spec.Tags[tag]; ok { if image.Annotations == nil { image.Annotations = make(map[string]string) } for k, v := range tagRef.Annotations { image.Annotations[k] = v } } } imageWithMetadata, err := api.ImageWithMetadata(*image) if err != nil { return nil, err } ist := api.ImageStreamTag{ ObjectMeta: kapi.ObjectMeta{ Namespace: kapi.NamespaceValue(ctx), Name: id, CreationTimestamp: event.Created, }, Image: *imageWithMetadata, } return &ist, nil }
func (h *manifestSchema2Handler) FillImageMetadata(ctx context.Context, image *imageapi.Image) error { // The manifest.Config references a configuration object for a container by its digest. // It needs to be fetched in order to fill an image object metadata below. configBytes, err := h.repo.Blobs(ctx).Get(ctx, h.manifest.Config.Digest) if err != nil { context.GetLogger(ctx).Errorf("failed to get image config %s: %v", h.manifest.Config.Digest.String(), err) return err } image.DockerImageConfig = string(configBytes) if err := imageapi.ImageWithMetadata(image); err != nil { return err } return nil }
// Get retrieves an image by ID that has previously been tagged into an image stream. // `id` is of the form <repo name>@<image id>. func (r *REST) Get(ctx kapi.Context, id string) (runtime.Object, error) { name, imageID, err := ParseNameAndID(id) if err != nil { return nil, err } repo, err := r.imageStreamRegistry.GetImageStream(ctx, name) if err != nil { return nil, err } if repo.Status.Tags == nil { return nil, errors.NewNotFound("imageStreamImage", imageID) } event, err := api.ResolveImageID(repo, imageID) if err != nil { return nil, err } imageName := event.Image image, err := r.imageRegistry.GetImage(ctx, imageName) if err != nil { return nil, err } imageWithMetadata, err := api.ImageWithMetadata(*image) if err != nil { return nil, err } if d, err := digest.ParseDigest(imageName); err == nil { imageName = d.Hex() } if len(imageName) > 7 { imageName = imageName[:7] } isi := api.ImageStreamImage{ ObjectMeta: kapi.ObjectMeta{ Namespace: kapi.NamespaceValue(ctx), Name: fmt.Sprintf("%s@%s", name, imageName), }, Image: *imageWithMetadata, } return &isi, nil }
// Limit is the limit range implementation that checks resource against the // image limit ranges. // Implements the LimitRangerActions interface func (a *imageLimitRangerPlugin) Limit(limitRange *kapi.LimitRange, kind string, obj runtime.Object) error { isObj, ok := obj.(*imageapi.ImageStreamMapping) if !ok { glog.V(5).Infof("%s: received object other than ImageStreamMapping (%T)", PluginName, obj) return nil } image := &isObj.Image if err := imageapi.ImageWithMetadata(image); err != nil { return err } for _, limit := range limitRange.Spec.Limits { if err := AdmitImage(image.DockerImageMetadata.Size, limit); err != nil { return err } } return nil }
// Get retrieves an image by ID that has previously been tagged into an image stream. // `id` is of the form <repo name>@<image id>. func (r *REST) Get(ctx kapi.Context, id string) (runtime.Object, error) { name, imageID, err := parseNameAndID(id) if err != nil { return nil, err } repo, err := r.imageStreamRegistry.GetImageStream(ctx, name) if err != nil { return nil, err } if repo.Status.Tags == nil { return nil, errors.NewNotFound(api.Resource("imagestreamimage"), id) } event, err := api.ResolveImageID(repo, imageID) if err != nil { return nil, err } imageName := event.Image image, err := r.imageRegistry.GetImage(ctx, imageName) if err != nil { return nil, err } if err := api.ImageWithMetadata(image); err != nil { return nil, err } image.DockerImageManifest = "" isi := api.ImageStreamImage{ ObjectMeta: kapi.ObjectMeta{ Namespace: kapi.NamespaceValue(ctx), Name: api.MakeImageStreamImageName(name, imageID), CreationTimestamp: image.ObjectMeta.CreationTimestamp, }, Image: *image, } return &isi, nil }
func storeTestImage( ctx context.Context, reg distribution.Namespace, imageReference reference.NamedTagged, schemaVersion int, managedByOpenShift bool, ) (*imageapi.Image, error) { repo, err := reg.Repository(ctx, imageReference) if err != nil { return nil, fmt.Errorf("unexpected error getting repo %q: %v", imageReference.Name(), err) } var ( m distribution.Manifest m1 schema1.Manifest ) switch schemaVersion { case 1: m1 = schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: imageReference.Name(), Tag: imageReference.Tag(), } case 2: // TODO fallthrough default: return nil, fmt.Errorf("unsupported manifest version %d", schemaVersion) } for i := 0; i < testImageLayerCount; i++ { rs, ds, err := registrytest.CreateRandomTarFile() if err != nil { return nil, fmt.Errorf("unexpected error generating test layer file: %v", err) } dgst := digest.Digest(ds) wr, err := repo.Blobs(ctx).Create(ctx) if err != nil { return nil, fmt.Errorf("unexpected error creating test upload: %v", err) } defer wr.Close() n, err := io.Copy(wr, rs) if err != nil { return nil, fmt.Errorf("unexpected error copying to upload: %v", err) } if schemaVersion == 1 { m1.FSLayers = append(m1.FSLayers, schema1.FSLayer{BlobSum: dgst}) m1.History = append(m1.History, schema1.History{V1Compatibility: fmt.Sprintf(`{"size":%d}`, n)}) } // TODO v2 if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst, MediaType: schema1.MediaTypeManifestLayer}); err != nil { return nil, fmt.Errorf("unexpected error finishing upload: %v", err) } } var dgst digest.Digest var payload []byte if schemaVersion == 1 { pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { return nil, fmt.Errorf("unexpected error generating private key: %v", err) } m, err = schema1.Sign(&m1, pk) if err != nil { return nil, fmt.Errorf("error signing manifest: %v", err) } _, payload, err = m.Payload() if err != nil { return nil, fmt.Errorf("error getting payload %#v", err) } dgst = digest.FromBytes(payload) } //TODO v2 image := &imageapi.Image{ ObjectMeta: kapi.ObjectMeta{ Name: dgst.String(), }, DockerImageManifest: string(payload), DockerImageReference: imageReference.Name() + "@" + dgst.String(), } if managedByOpenShift { image.Annotations = map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"} } if schemaVersion == 1 { signedManifest := m.(*schema1.SignedManifest) signatures, err := signedManifest.Signatures() if err != nil { return nil, err } for _, signDigest := range signatures { image.DockerImageSignatures = append(image.DockerImageSignatures, signDigest) } } err = imageapi.ImageWithMetadata(image) if err != nil { return nil, fmt.Errorf("failed to fill image with metadata: %v", err) } return image, nil }
func (r *REST) Create(ctx kapi.Context, obj runtime.Object) (runtime.Object, error) { isi, ok := obj.(*api.ImageStreamImport) if !ok { return nil, kapierrors.NewBadRequest(fmt.Sprintf("obj is not an ImageStreamImport: %#v", obj)) } inputMeta := isi.ObjectMeta if err := rest.BeforeCreate(Strategy, ctx, obj); err != nil { return nil, err } namespace, ok := kapi.NamespaceFrom(ctx) if !ok { return nil, kapierrors.NewBadRequest("a namespace must be specified to import images") } secrets, err := r.secrets.ImageStreamSecrets(namespace).Secrets(isi.Name, kapi.ListOptions{}) if err != nil { util.HandleError(fmt.Errorf("unable to load secrets for namespace %q: %v", namespace, err)) secrets = &kapi.SecretList{} } if r.clientFn != nil { if client := r.clientFn(); client != nil { ctx = kapi.WithValue(ctx, importer.ContextKeyV1RegistryClient, client) } } credentials := importer.NewCredentialsForSecrets(secrets.Items) importCtx := importer.NewContext(r.transport).WithCredentials(credentials) imports := r.importFn(importCtx) if err := imports.Import(ctx.(gocontext.Context), isi); err != nil { return nil, kapierrors.NewInternalError(err) } // TODO: perform the transformation of the image stream and return it with the ISI if import is false // so that clients can see what the resulting object would look like. if !isi.Spec.Import { clearManifests(isi) return isi, nil } create := false stream, err := r.streams.GetImageStream(ctx, isi.Name) if err != nil { if !kapierrors.IsNotFound(err) { return nil, err } // consistency check, stream must exist if len(inputMeta.ResourceVersion) > 0 || len(inputMeta.UID) > 0 { return nil, err } create = true stream = &api.ImageStream{ ObjectMeta: kapi.ObjectMeta{ Name: isi.Name, Namespace: namespace, Generation: 0, }, } } else { if len(inputMeta.ResourceVersion) > 0 && inputMeta.ResourceVersion != stream.ResourceVersion { glog.V(4).Infof("DEBUG: mismatch between requested UID %s and located UID %s", inputMeta.UID, stream.UID) return nil, kapierrors.NewConflict("imageStream", inputMeta.Name, fmt.Errorf("the image stream was updated from %q to %q", inputMeta.ResourceVersion, stream.ResourceVersion)) } if len(inputMeta.UID) > 0 && inputMeta.UID != stream.UID { glog.V(4).Infof("DEBUG: mismatch between requested UID %s and located UID %s", inputMeta.UID, stream.UID) return nil, kapierrors.NewNotFound("imageStream", inputMeta.Name) } } if stream.Annotations == nil { stream.Annotations = make(map[string]string) } now := unversioned.Now() stream.Annotations[api.DockerImageRepositoryCheckAnnotation] = now.UTC().Format(time.RFC3339) gen := stream.Generation + 1 zero := int64(0) importedImages := make(map[string]error) updatedImages := make(map[string]*api.Image) if spec := isi.Spec.Repository; spec != nil { for i, imageStatus := range isi.Status.Repository.Images { image := imageStatus.Image if image == nil { continue } // update the spec tag ref, err := api.ParseDockerImageReference(image.DockerImageReference) if err != nil { // ??? continue } tag := ref.Tag if len(imageStatus.Tag) > 0 { tag = imageStatus.Tag } if _, ok := stream.Spec.Tags[tag]; !ok { if stream.Spec.Tags == nil { stream.Spec.Tags = make(map[string]api.TagReference) } stream.Spec.Tags[tag] = api.TagReference{ From: &kapi.ObjectReference{ Kind: "DockerImage", Name: image.DockerImageReference, }, Generation: &gen, ImportPolicy: api.TagImportPolicy{Insecure: spec.ImportPolicy.Insecure}, } } // import or reuse the image importErr, imported := importedImages[image.Name] if importErr != nil { api.SetTagConditions(stream, tag, newImportFailedCondition(err, gen, now)) } pullSpec, _ := api.MostAccuratePullSpec(image.DockerImageReference, image.Name, "") api.AddTagEventToImageStream(stream, tag, api.TagEvent{ Created: now, DockerImageReference: pullSpec, Image: image.Name, Generation: gen, }) if imported { if updatedImage, ok := updatedImages[image.Name]; ok { isi.Status.Repository.Images[i].Image = updatedImage } continue } // establish the image into the store updated, err := r.images.Create(ctx, image) switch { case kapierrors.IsAlreadyExists(err): if err := api.ImageWithMetadata(image); err != nil { glog.V(4).Infof("Unable to update image metadata during image import when image already exists %q: err", image.Name, err) } updated = image fallthrough case err == nil: updatedImage := updated.(*api.Image) updatedImages[image.Name] = updatedImage isi.Status.Repository.Images[i].Image = updatedImage importedImages[image.Name] = nil default: importedImages[image.Name] = err } } } for i, spec := range isi.Spec.Images { if spec.To == nil { continue } tag := spec.To.Name if stream.Spec.Tags == nil { stream.Spec.Tags = make(map[string]api.TagReference) } specTag := stream.Spec.Tags[tag] from := spec.From specTag.From = &from specTag.Generation = &zero specTag.ImportPolicy.Insecure = spec.ImportPolicy.Insecure stream.Spec.Tags[tag] = specTag status := isi.Status.Images[i] if status.Image == nil || status.Status.Status == unversioned.StatusFailure { message := status.Status.Message if len(message) == 0 { message = "unknown error prevented import" } api.SetTagConditions(stream, tag, api.TagEventCondition{ Type: api.ImportSuccess, Status: kapi.ConditionFalse, Message: message, Reason: string(status.Status.Reason), Generation: gen, LastTransitionTime: now, }) continue } image := status.Image importErr, imported := importedImages[image.Name] if importErr != nil { api.SetTagConditions(stream, tag, newImportFailedCondition(err, gen, now)) } pullSpec, _ := api.MostAccuratePullSpec(image.DockerImageReference, image.Name, "") api.AddTagEventToImageStream(stream, tag, api.TagEvent{ Created: now, DockerImageReference: pullSpec, Image: image.Name, Generation: gen, }) if imported { continue } _, err = r.images.Create(ctx, image) if kapierrors.IsAlreadyExists(err) { err = nil } importedImages[image.Name] = err } // TODO: should we allow partial failure? for _, err := range importedImages { if err != nil { return nil, err } } clearManifests(isi) if create { obj, err = r.internalStreams.Create(ctx, stream) } else { obj, _, err = r.internalStreams.Update(ctx, stream) } if err != nil { return nil, err } isi.Status.Import = obj.(*api.ImageStream) return isi, nil }
// newISTag initializes an image stream tag from an image stream and image. The allowEmptyEvent will create a tag even // in the event that the status tag does does not exist yet (no image has successfully been tagged) or the image is nil. func newISTag(tag string, imageStream *api.ImageStream, image *api.Image, allowEmptyEvent bool) (*api.ImageStreamTag, error) { istagName := api.JoinImageStreamTag(imageStream.Name, tag) event := api.LatestTaggedImage(imageStream, tag) if event == nil || len(event.Image) == 0 { if !allowEmptyEvent { return nil, kapierrors.NewNotFound(api.Resource("imagestreamtags"), istagName) } event = &api.TagEvent{ Created: imageStream.CreationTimestamp, } } ist := &api.ImageStreamTag{ ObjectMeta: kapi.ObjectMeta{ Namespace: imageStream.Namespace, Name: istagName, CreationTimestamp: event.Created, Annotations: map[string]string{}, ResourceVersion: imageStream.ResourceVersion, }, Generation: event.Generation, Conditions: imageStream.Status.Tags[tag].Conditions, } if imageStream.Spec.Tags != nil { if tagRef, ok := imageStream.Spec.Tags[tag]; ok { // copy the spec tag ist.Tag = &tagRef if from := ist.Tag.From; from != nil { copied := *from ist.Tag.From = &copied } if gen := ist.Tag.Generation; gen != nil { copied := *gen ist.Tag.Generation = &copied } // if the imageStream has Spec.Tags[tag].Annotations[k] = v, copy it to the image's annotations // and add them to the istag's annotations if image != nil && image.Annotations == nil { image.Annotations = make(map[string]string) } for k, v := range tagRef.Annotations { ist.Annotations[k] = v if image != nil { image.Annotations[k] = v } } } } if image != nil { if err := api.ImageWithMetadata(image); err != nil { return nil, err } image.DockerImageManifest = "" ist.Image = *image } else { ist.Image = api.Image{} ist.Image.Name = event.Image } // Replace the DockerImageReference with the value from event, which contains // real value from status. This should fix the problem for v1 registries, // where mutliple tags point to a single id and only the first image's metadata // is saved. This in turn will always return the pull spec from the first // imported image, which might be different than the requested tag. ist.Image.DockerImageReference = event.DockerImageReference return ist, nil }
// importSuccessful records a successful import into an image stream, setting the spec tag, status tag or conditions, and ensuring // the image is created in etcd. Images are cached so they are not created multiple times in a row (when multiple tags point to the // same image), and a failure to persist the image will be summarized before we update the stream. If an image was imported by this // operation, it *replaces* the imported image (from the remote repository) with the updated image. func (r *REST) importSuccessful( ctx kapi.Context, image *api.Image, stream *api.ImageStream, tag string, from string, nextGeneration int64, now unversioned.Time, importPolicy api.TagImportPolicy, importedImages map[string]error, updatedImages map[string]*api.Image, ) (*api.Image, bool) { Strategy.PrepareImageForCreate(image) pullSpec, _ := api.MostAccuratePullSpec(image.DockerImageReference, image.Name, "") tagEvent := api.TagEvent{ Created: now, DockerImageReference: pullSpec, Image: image.Name, Generation: nextGeneration, } if stream.Spec.Tags == nil { stream.Spec.Tags = make(map[string]api.TagReference) } // ensure the spec and status tag match the imported image changed := api.DifferentTagEvent(stream, tag, tagEvent) specTag, ok := stream.Spec.Tags[tag] if changed || !ok { specTag = ensureSpecTag(stream, tag, from, importPolicy, true) api.AddTagEventToImageStream(stream, tag, tagEvent) } // always reset the import policy specTag.ImportPolicy = importPolicy stream.Spec.Tags[tag] = specTag // import or reuse the image, and ensure tag conditions are set importErr, alreadyImported := importedImages[image.Name] if importErr != nil { api.SetTagConditions(stream, tag, newImportFailedCondition(importErr, nextGeneration, now)) } else { api.SetTagConditions(stream, tag) } // create the image if it does not exist, otherwise cache the updated status from the store for use by other tags if alreadyImported { if updatedImage, ok := updatedImages[image.Name]; ok { return updatedImage, true } return nil, false } updated, err := r.images.Create(ctx, image) switch { case kapierrors.IsAlreadyExists(err): if err := api.ImageWithMetadata(image); err != nil { glog.V(4).Infof("Unable to update image metadata during image import when image already exists %q: err", image.Name, err) } updated = image fallthrough case err == nil: updatedImage := updated.(*api.Image) updatedImages[image.Name] = updatedImage //isi.Status.Repository.Images[i].Image = updatedImage importedImages[image.Name] = nil return updatedImage, true default: importedImages[image.Name] = err } return nil, false }
// importRepositoryFromDocker loads the tags and images requested in the passed importRepository, obeying the // optional rate limiter. Errors are set onto the individual tags and digest objects. func (isi *ImageStreamImporter) importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetriever, repository *importRepository, limiter flowcontrol.RateLimiter) { glog.V(5).Infof("importing remote Docker repository registry=%s repository=%s insecure=%t", repository.Registry, repository.Name, repository.Insecure) // retrieve the repository repo, err := retriever.Repository(ctx, repository.Registry, repository.Name, repository.Insecure) if err != nil { glog.V(5).Infof("unable to access repository %#v: %#v", repository, err) switch { case err == reference.ErrReferenceInvalidFormat: err = field.Invalid(field.NewPath("from", "name"), repository.Name, "the provided repository name is not valid") case isDockerError(err, v2.ErrorCodeNameUnknown): err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact()) case isDockerError(err, errcode.ErrorCodeUnauthorized): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact())) case strings.Contains(err.Error(), "tls: oversized record received with length") && !repository.Insecure: err = kapierrors.NewBadRequest("this repository is HTTP only and requires the insecure flag to import") case strings.HasSuffix(err.Error(), "no basic auth credentials"): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact())) case strings.HasSuffix(err.Error(), "does not support v2 API"): importRepositoryFromDockerV1(ctx, repository, limiter) return } applyErrorToRepository(repository, err) return } // get a manifest context s, err := repo.Manifests(ctx) if err != nil { glog.V(5).Infof("unable to access manifests for repository %#v: %#v", repository, err) switch { case isDockerError(err, v2.ErrorCodeNameUnknown): err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact()) case isDockerError(err, errcode.ErrorCodeUnauthorized): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact())) case strings.HasSuffix(err.Error(), "no basic auth credentials"): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact())) } applyErrorToRepository(repository, err) return } // get a blob context b := repo.Blobs(ctx) // if repository import is requested (MaximumTags), attempt to load the tags, sort them, and request the first N if count := repository.MaximumTags; count > 0 || count == -1 { tags, err := repo.Tags(ctx).All(ctx) if err != nil { glog.V(5).Infof("unable to access tags for repository %#v: %#v", repository, err) switch { case isDockerError(err, v2.ErrorCodeNameUnknown): err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact()) case isDockerError(err, errcode.ErrorCodeUnauthorized): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact())) } repository.Err = err return } // some images on the Hub have empty tags - treat those as "latest" set := sets.NewString(tags...) if set.Has("") { set.Delete("") set.Insert(api.DefaultImageTag) } tags = set.List() // include only the top N tags in the result, put the rest in AdditionalTags api.PrioritizeTags(tags) for _, s := range tags { if count <= 0 && repository.MaximumTags != -1 { repository.AdditionalTags = append(repository.AdditionalTags, s) continue } count-- repository.Tags = append(repository.Tags, importTag{ Name: s, }) } } // load digests for i := range repository.Digests { importDigest := &repository.Digests[i] if importDigest.Err != nil || importDigest.Image != nil { continue } d, err := digest.ParseDigest(importDigest.Name) if err != nil { importDigest.Err = err continue } limiter.Accept() manifest, err := s.Get(ctx, d) if err != nil { glog.V(5).Infof("unable to access digest %q for repository %#v: %#v", d, repository, err) importDigest.Err = formatRepositoryError(repository, "", importDigest.Name, err) continue } if signedManifest, isSchema1 := manifest.(*schema1.SignedManifest); isSchema1 { importDigest.Image, err = schema1ToImage(signedManifest, d) } else if deserializedManifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 { imageConfig, err := b.Get(ctx, deserializedManifest.Config.Digest) if err != nil { glog.V(5).Infof("unable to access the image config using digest %q for repository %#v: %#v", d, repository, err) if isDockerError(err, v2.ErrorCodeManifestUnknown) { ref := repository.Ref ref.ID = deserializedManifest.Config.Digest.String() importDigest.Err = kapierrors.NewNotFound(api.Resource("dockerimage"), ref.Exact()) } else { importDigest.Err = formatRepositoryError(repository, "", importDigest.Name, err) } continue } importDigest.Image, err = schema2ToImage(deserializedManifest, imageConfig, d) } else { glog.V(5).Infof("unsupported manifest type: %T", manifest) continue } if err != nil { importDigest.Err = err continue } if err := api.ImageWithMetadata(importDigest.Image); err != nil { importDigest.Err = err continue } if importDigest.Image.DockerImageMetadata.Size == 0 { if err := isi.calculateImageSize(ctx, repo, importDigest.Image); err != nil { importDigest.Err = err continue } } } for i := range repository.Tags { importTag := &repository.Tags[i] if importTag.Err != nil || importTag.Image != nil { continue } limiter.Accept() desc, err := repo.Tags(ctx).Get(ctx, importTag.Name) if err != nil { glog.V(5).Infof("unable to get tag %q for repository %#v: %#v", importTag.Name, repository, err) importTag.Err = formatRepositoryError(repository, importTag.Name, "", err) continue } manifest, err := s.Get(ctx, desc.Digest) if err != nil { glog.V(5).Infof("unable to access digest %q for tag %q for repository %#v: %#v", desc.Digest, importTag.Name, repository, err) importTag.Err = formatRepositoryError(repository, importTag.Name, "", err) continue } if signedManifest, isSchema1 := manifest.(*schema1.SignedManifest); isSchema1 { importTag.Image, err = schema1ToImage(signedManifest, "") } else if deserializedManifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 { imageConfig, err := b.Get(ctx, deserializedManifest.Config.Digest) if err != nil { glog.V(5).Infof("unable to access image config using digest %q for tag %q for repository %#v: %#v", desc.Digest, importTag.Name, repository, err) importTag.Err = formatRepositoryError(repository, importTag.Name, "", err) continue } importTag.Image, err = schema2ToImage(deserializedManifest, imageConfig, "") } else { glog.V(5).Infof("unsupported manifest type: %T", manifest) continue } if err != nil { importTag.Err = err continue } if err := api.ImageWithMetadata(importTag.Image); err != nil { importTag.Err = err continue } if importTag.Image.DockerImageMetadata.Size == 0 { if err := isi.calculateImageSize(ctx, repo, importTag.Image); err != nil { importTag.Err = err continue } } } }
// importRepositoryFromDocker loads the tags and images requested in the passed importRepository, obeying the // optional rate limiter. Errors are set onto the individual tags and digest objects. func importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetriever, repository *importRepository, limiter util.RateLimiter) { // retrieve the repository repo, err := retriever.Repository(ctx, repository.Registry, repository.Name, repository.Insecure) if err != nil { glog.V(5).Infof("unable to access repository %#v: %#v", repository, err) switch { case isDockerError(err, v2.ErrorCodeNameUnknown): err = kapierrors.NewNotFound("DockerImage", repository.Ref.Exact()) case isDockerError(err, errcode.ErrorCodeUnauthorized): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact())) case strings.Contains(err.Error(), "tls: oversized record received with length") && !repository.Insecure: err = kapierrors.NewBadRequest("this repository is HTTP only and requires the insecure flag to import") case strings.HasSuffix(err.Error(), "no basic auth credentials"): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact())) case strings.HasSuffix(err.Error(), "does not support v2 API"): importRepositoryFromDockerV1(ctx, repository, limiter) return } applyErrorToRepository(repository, err) return } // get a manifest context s, err := repo.Manifests(ctx) if err != nil { glog.V(5).Infof("unable to access manifests for repository %#v: %#v", repository, err) switch { case isDockerError(err, v2.ErrorCodeNameUnknown): err = kapierrors.NewNotFound("DockerImage", repository.Ref.Exact()) case isDockerError(err, errcode.ErrorCodeUnauthorized): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact())) case strings.HasSuffix(err.Error(), "no basic auth credentials"): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact())) } applyErrorToRepository(repository, err) return } // if repository import is requested (MaximumTags), attempt to load the tags, sort them, and request the first N if count := repository.MaximumTags; count > 0 || count == -1 { tags, err := s.Tags() if err != nil { glog.V(5).Infof("unable to access tags for repository %#v: %#v", repository, err) switch { case isDockerError(err, v2.ErrorCodeNameUnknown): err = kapierrors.NewNotFound("DockerImage", repository.Ref.Exact()) case isDockerError(err, errcode.ErrorCodeUnauthorized): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact())) } repository.Err = err return } // some images on the Hub have empty tags - treat those as "latest" set := sets.NewString(tags...) if set.Has("") { set.Delete("") set.Insert(api.DefaultImageTag) } tags = set.List() // include only the top N tags in the result, put the rest in AdditionalTags api.PrioritizeTags(tags) for _, s := range tags { if count <= 0 && repository.MaximumTags != -1 { repository.AdditionalTags = append(repository.AdditionalTags, s) continue } count-- repository.Tags = append(repository.Tags, importTag{ Name: s, }) } } // load digests for i := range repository.Digests { importDigest := &repository.Digests[i] if importDigest.Err != nil || importDigest.Image != nil { continue } d, err := digest.ParseDigest(importDigest.Name) if err != nil { importDigest.Err = err continue } limiter.Accept() m, err := s.Get(d) if err != nil { glog.V(5).Infof("unable to access digest %q for repository %#v: %#v", d, repository, err) switch { case isDockerError(err, v2.ErrorCodeManifestUnknown): ref := repository.Ref ref.Tag, ref.ID = "", importDigest.Name err = kapierrors.NewNotFound("DockerImage", ref.Exact()) case isDockerError(err, errcode.ErrorCodeUnauthorized): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact())) case strings.HasSuffix(err.Error(), "no basic auth credentials"): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact())) } importDigest.Err = err continue } importDigest.Image, err = schema1ToImage(m, d) if err != nil { importDigest.Err = err continue } if err := api.ImageWithMetadata(importDigest.Image); err != nil { importDigest.Err = err continue } } for i := range repository.Tags { importTag := &repository.Tags[i] if importTag.Err != nil || importTag.Image != nil { continue } limiter.Accept() m, err := s.GetByTag(importTag.Name) if err != nil { glog.V(5).Infof("unable to access tag %q for repository %#v: %#v", importTag.Name, repository, err) switch { case isDockerError(err, v2.ErrorCodeManifestUnknown): ref := repository.Ref ref.Tag = importTag.Name err = kapierrors.NewNotFound("DockerImage", ref.Exact()) case isDockerError(err, errcode.ErrorCodeUnauthorized): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact())) case strings.HasSuffix(err.Error(), "no basic auth credentials"): err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact())) } importTag.Err = err continue } importTag.Image, err = schema1ToImage(m, "") if err != nil { importTag.Err = err continue } if err := api.ImageWithMetadata(importTag.Image); err != nil { importTag.Err = err continue } } }