func (bla byLikeness) Less(i, j int) bool { aMacMatch := metadata.CheckV2MetadataHMAC(&bla.arr[i], bla.hmacKey) bMacMatch := metadata.CheckV2MetadataHMAC(&bla.arr[j], bla.hmacKey) if aMacMatch != bMacMatch { return aMacMatch } aMatch := numOfMatchingPathComponents(bla.arr[i].SourceRepository, bla.pathComponents) bMatch := numOfMatchingPathComponents(bla.arr[j].SourceRepository, bla.pathComponents) return aMatch > bMatch }
// layerAlreadyExists checks if the registry already knows about any of the metadata passed in the "metadata" // slice. If it finds one that the registry knows about, it returns the known digest and "true". If // "checkOtherRepositories" is true, stat will be performed also with digests mapped to any other repository // (not just the target one). func (pd *v2PushDescriptor) layerAlreadyExists( ctx context.Context, progressOutput progress.Output, diffID layer.DiffID, checkOtherRepositories bool, maxExistenceCheckAttempts int, v2Metadata []metadata.V2Metadata, ) (desc distribution.Descriptor, exists bool, err error) { // filter the metadata candidates := []metadata.V2Metadata{} for _, meta := range v2Metadata { if len(meta.SourceRepository) > 0 && !checkOtherRepositories && meta.SourceRepository != pd.repoInfo.FullName() { continue } candidates = append(candidates, meta) } // sort the candidates by similarity sortV2MetadataByLikenessAndAge(pd.repoInfo, pd.hmacKey, candidates) digestToMetadata := make(map[digest.Digest]*metadata.V2Metadata) // an array of unique blob digests ordered from the best mount candidates to worst layerDigests := []digest.Digest{} for i := 0; i < len(candidates); i++ { if len(layerDigests) >= maxExistenceCheckAttempts { break } meta := &candidates[i] if _, exists := digestToMetadata[meta.Digest]; exists { // keep reference just to the first mapping (the best mount candidate) continue } if _, exists := pd.checkedDigests[meta.Digest]; exists { // existence of this digest has already been tested continue } digestToMetadata[meta.Digest] = meta layerDigests = append(layerDigests, meta.Digest) } attempts: for _, dgst := range layerDigests { meta := digestToMetadata[dgst] logrus.Debugf("Checking for presence of layer %s (%s) in %s", diffID, dgst, pd.repoInfo.FullName()) desc, err = pd.repo.Blobs(ctx).Stat(ctx, dgst) pd.checkedDigests[meta.Digest] = struct{}{} switch err { case nil: if m, ok := digestToMetadata[desc.Digest]; !ok || m.SourceRepository != pd.repoInfo.FullName() || !metadata.CheckV2MetadataHMAC(m, pd.hmacKey) { // cache mapping from this layer's DiffID to the blobsum if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{ Digest: desc.Digest, SourceRepository: pd.repoInfo.FullName(), }); err != nil { return distribution.Descriptor{}, false, xfer.DoNotRetry{Err: err} } } desc.MediaType = schema2.MediaTypeLayer exists = true break attempts case distribution.ErrBlobUnknown: if meta.SourceRepository == pd.repoInfo.FullName() { // remove the mapping to the target repository pd.v2MetadataService.Remove(*meta) } default: logrus.WithError(err).Debugf("Failed to check for presence of layer %s (%s) in %s", diffID, dgst, pd.repoInfo.FullName()) } } if exists { progress.Update(progressOutput, pd.ID(), "Layer already exists") pd.pushState.Lock() pd.pushState.remoteLayers[diffID] = desc pd.pushState.Unlock() } return desc, exists, nil }
func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) { if fs, ok := pd.layer.(distribution.Describable); ok { if d := fs.Descriptor(); len(d.URLs) > 0 { progress.Update(progressOutput, pd.ID(), "Skipped foreign layer") return d, nil } } diffID := pd.DiffID() pd.pushState.Lock() if descriptor, ok := pd.pushState.remoteLayers[diffID]; ok { // it is already known that the push is not needed and // therefore doing a stat is unnecessary pd.pushState.Unlock() progress.Update(progressOutput, pd.ID(), "Layer already exists") return descriptor, nil } pd.pushState.Unlock() maxMountAttempts, maxExistenceChecks, checkOtherRepositories := getMaxMountAndExistenceCheckAttempts(pd.layer) // Do we have any metadata associated with this layer's DiffID? v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID) if err == nil { // check for blob existence in the target repository if we have a mapping with it descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, false, 1, v2Metadata) if exists || err != nil { return descriptor, err } } // if digest was empty or not saved, or if blob does not exist on the remote repository, // then push the blob. bs := pd.repo.Blobs(ctx) var layerUpload distribution.BlobWriter // Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload candidates := getRepositoryMountCandidates(pd.repoInfo, pd.hmacKey, maxMountAttempts, v2Metadata) for _, mountCandidate := range candidates { logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountCandidate.Digest, mountCandidate.SourceRepository) createOpts := []distribution.BlobCreateOption{} if len(mountCandidate.SourceRepository) > 0 { namedRef, err := reference.WithName(mountCandidate.SourceRepository) if err != nil { logrus.Errorf("failed to parse source repository reference %v: %v", namedRef.String(), err) pd.v2MetadataService.Remove(mountCandidate) continue } // TODO (brianbland): We need to construct a reference where the Name is // only the full remote name, so clean this up when distribution has a // richer reference package remoteRef, err := distreference.WithName(namedRef.RemoteName()) if err != nil { logrus.Errorf("failed to make remote reference out of %q: %v", namedRef.RemoteName(), namedRef.RemoteName()) continue } canonicalRef, err := distreference.WithDigest(distreference.TrimNamed(remoteRef), mountCandidate.Digest) if err != nil { logrus.Errorf("failed to make canonical reference: %v", err) continue } createOpts = append(createOpts, client.WithMountFrom(canonicalRef)) } // send the layer lu, err := bs.Create(ctx, createOpts...) switch err := err.(type) { case nil: // noop case distribution.ErrBlobMounted: progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name()) err.Descriptor.MediaType = schema2.MediaTypeLayer pd.pushState.Lock() pd.pushState.confirmedV2 = true pd.pushState.remoteLayers[diffID] = err.Descriptor pd.pushState.Unlock() // Cache mapping from this layer's DiffID to the blobsum if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{ Digest: err.Descriptor.Digest, SourceRepository: pd.repoInfo.FullName(), }); err != nil { return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} } return err.Descriptor, nil default: logrus.Infof("failed to mount layer %s (%s) from %s: %v", diffID, mountCandidate.Digest, mountCandidate.SourceRepository, err) } if len(mountCandidate.SourceRepository) > 0 && (metadata.CheckV2MetadataHMAC(&mountCandidate, pd.hmacKey) || len(mountCandidate.HMAC) == 0) { cause := "blob mount failure" if err != nil { cause = fmt.Sprintf("an error: %v", err.Error()) } logrus.Debugf("removing association between layer %s and %s due to %s", mountCandidate.Digest, mountCandidate.SourceRepository, cause) pd.v2MetadataService.Remove(mountCandidate) } if lu != nil { // cancel previous upload cancelLayerUpload(ctx, mountCandidate.Digest, layerUpload) layerUpload = lu } } if maxExistenceChecks-len(pd.checkedDigests) > 0 { // do additional layer existence checks with other known digests if any descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, checkOtherRepositories, maxExistenceChecks-len(pd.checkedDigests), v2Metadata) if exists || err != nil { return descriptor, err } } logrus.Debugf("Pushing layer: %s", diffID) if layerUpload == nil { layerUpload, err = bs.Create(ctx) if err != nil { return distribution.Descriptor{}, retryOnError(err) } } defer layerUpload.Close() // upload the blob desc, err := pd.uploadUsingSession(ctx, progressOutput, diffID, layerUpload) if err != nil { return desc, err } return desc, nil }