// NewRepositoryInfo validates and breaks down a repository name into a RepositoryInfo func (config *ServiceConfig) NewRepositoryInfo(reposName reference.Named) (*RepositoryInfo, error) { if err := validateNoSchema(reposName.Name()); err != nil { return nil, err } repoInfo := &RepositoryInfo{} var ( indexName string err error ) indexName, repoInfo.RemoteName, err = loadRepositoryName(reposName) if err != nil { return nil, err } repoInfo.Index, err = config.NewIndexInfo(indexName) if err != nil { return nil, err } if repoInfo.Index.Official { repoInfo.LocalName, err = normalizeLibraryRepoName(repoInfo.RemoteName) if err != nil { return nil, err } repoInfo.RemoteName = repoInfo.LocalName // If the normalized name does not contain a '/' (e.g. "foo") // then it is an official repo. if strings.IndexRune(repoInfo.RemoteName.Name(), '/') == -1 { repoInfo.Official = true // Fix up remote name for official repos. repoInfo.RemoteName, err = reference.WithName("library/" + repoInfo.RemoteName.Name()) if err != nil { return nil, err } } repoInfo.CanonicalName, err = reference.WithName("docker.io/" + repoInfo.RemoteName.Name()) if err != nil { return nil, err } } else { repoInfo.LocalName, err = localNameFromRemote(repoInfo.Index.Name, repoInfo.RemoteName) if err != nil { return nil, err } repoInfo.CanonicalName = repoInfo.LocalName } return repoInfo, nil }
// normalizeLibraryRepoName removes the library prefix from // the repository name for official repos. func normalizeLibraryRepoName(name reference.Named) (reference.Named, error) { if strings.HasPrefix(name.Name(), "library/") { // If pull "library/foo", it's stored locally under "foo" return reference.WithName(strings.SplitN(name.Name(), "/", 2)[1]) } return name, nil }
func TestMirrorEndpointLookup(t *testing.T) { containsMirror := func(endpoints []APIEndpoint) bool { for _, pe := range endpoints { if pe.URL == "my.mirror" { return true } } return false } s := Service{Config: makeServiceConfig([]string{"my.mirror"}, nil)} imageName, err := reference.WithName(IndexName + "/test/image") if err != nil { t.Error(err) } pushAPIEndpoints, err := s.LookupPushEndpoints(imageName) if err != nil { t.Fatal(err) } if containsMirror(pushAPIEndpoints) { t.Fatal("Push endpoint should not contain mirror") } pullAPIEndpoints, err := s.LookupPullEndpoints(imageName) if err != nil { t.Fatal(err) } if !containsMirror(pullAPIEndpoints) { t.Fatal("Pull endpoint should contain mirror") } }
func TestValidateRepositoryName(t *testing.T) { validRepoNames := []string{ "docker/docker", "library/debian", "debian", "docker.io/docker/docker", "docker.io/library/debian", "docker.io/debian", "index.docker.io/docker/docker", "index.docker.io/library/debian", "index.docker.io/debian", "127.0.0.1:5000/docker/docker", "127.0.0.1:5000/library/debian", "127.0.0.1:5000/debian", "thisisthesongthatneverendsitgoesonandonandonthisisthesongthatnev", } invalidRepoNames := []string{ "https://github.com/docker/docker", "docker/Docker", "-docker", "-docker/docker", "-docker.io/docker/docker", "docker///docker", "docker.io/docker/Docker", "docker.io/docker///docker", "1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", "docker.io/1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", } for _, name := range invalidRepoNames { named, err := reference.WithName(name) if err == nil { err := ValidateRepositoryName(named) assertNotEqual(t, err, nil, "Expected invalid repo name: "+name) } } for _, name := range validRepoNames { named, err := reference.WithName(name) if err != nil { t.Fatalf("could not parse valid name: %s", name) } err = ValidateRepositoryName(named) assertEqual(t, err, nil, "Expected valid repo name: "+name) } }
func (l *tarexporter) legacyLoad(tmpDir string, outStream io.Writer) error { legacyLoadedMap := make(map[string]image.ID) dirs, err := ioutil.ReadDir(tmpDir) if err != nil { return err } // every dir represents an image for _, d := range dirs { if d.IsDir() { if err := l.legacyLoadImage(d.Name(), tmpDir, legacyLoadedMap); err != nil { return err } } } // load tags from repositories file repositoriesPath, err := safePath(tmpDir, legacyRepositoriesFileName) if err != nil { return err } repositoriesFile, err := os.Open(repositoriesPath) if err != nil { if !os.IsNotExist(err) { return err } return repositoriesFile.Close() } defer repositoriesFile.Close() repositories := make(map[string]map[string]string) if err := json.NewDecoder(repositoriesFile).Decode(&repositories); err != nil { return err } for name, tagMap := range repositories { for tag, oldID := range tagMap { imgID, ok := legacyLoadedMap[oldID] if !ok { return fmt.Errorf("invalid target ID: %v", oldID) } named, err := reference.WithName(name) if err != nil { return err } ref, err := reference.WithTag(named, tag) if err != nil { return err } l.setLoadedTag(ref, imgID, outStream) } } return nil }
// WithName returns a named object representing the given string. If the input // is invalid ErrReferenceInvalidFormat will be returned. func WithName(name string) (Named, error) { name = normalize(name) if err := validateName(name); err != nil { return nil, err } r, err := distreference.WithName(name) if err != nil { return nil, err } return &namedRef{r}, nil }
func validateRemoteName(remoteName string) error { if !strings.Contains(remoteName, "/") { // the repository name must not be a valid image ID if err := image.ValidateID(remoteName); err == nil { return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", remoteName) } } _, err := reference.WithName(remoteName) return err }
// splitReposName breaks a reposName into an index name and remote name func splitReposName(reposName reference.Named) (indexName string, remoteName reference.Named, err error) { var remoteNameStr string indexName, remoteNameStr = reference.SplitHostname(reposName) if indexName == "" || (!strings.Contains(indexName, ".") && !strings.Contains(indexName, ":") && indexName != "localhost") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) // 'docker.io' indexName = IndexName remoteName = reposName } else { remoteName, err = reference.WithName(remoteNameStr) } return }
func (s *router) postImagesTag(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if err := httputils.ParseForm(r); err != nil { return err } repo := r.Form.Get("repo") tag := r.Form.Get("tag") newTag, err := reference.WithName(repo) if err != nil { return err } if tag != "" { if newTag, err = reference.WithTag(newTag, tag); err != nil { return err } } if err := s.daemon.TagImage(newTag, vars["name"]); err != nil { return err } w.WriteHeader(http.StatusCreated) return nil }
func handlerPutTag(w http.ResponseWriter, r *http.Request) { if !requiresAuth(w, r) { return } vars := mux.Vars(r) repositoryName, err := reference.WithName(vars["repository"]) if err != nil { apiError(w, "Could not parse repository", 400) return } repositoryName = NormalizeLocalName(repositoryName) tagName := vars["tag"] tags, exists := testRepositories[repositoryName.String()] if !exists { tags = make(map[string]string) testRepositories[repositoryName.String()] = tags } tagValue := "" readJSON(r, tagValue) tags[tagName] = tagValue writeResponse(w, true, 200) }
func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) { if !requiresAuth(w, r) { return } repositoryName, err := reference.WithName(mux.Vars(r)["repository"]) if err != nil { apiError(w, "Could not parse repository", 400) return } repositoryName = NormalizeLocalName(repositoryName) tags, exists := testRepositories[repositoryName.String()] if !exists { apiError(w, "Repository not found", 404) return } if r.Method == "DELETE" { delete(testRepositories, repositoryName.String()) writeResponse(w, true, 200) return } writeResponse(w, tags, 200) }
func handlerGetTag(w http.ResponseWriter, r *http.Request) { if !requiresAuth(w, r) { return } vars := mux.Vars(r) repositoryName, err := reference.WithName(vars["repository"]) if err != nil { apiError(w, "Could not parse repository", 400) return } repositoryName = NormalizeLocalName(repositoryName) tagName := vars["tag"] tags, exists := testRepositories[repositoryName.String()] if !exists { apiError(w, "Repository not found", 404) return } tag, exists := tags[tagName] if !exists { apiError(w, "Tag not found", 404) return } writeResponse(w, tag, 200) }
// Commit creates a new filesystem image from the current state of a container. // The image can optionally be tagged into a repository. func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (string, error) { container, err := daemon.Get(name) if err != nil { return "", err } // It is not possible to commit a running container on Windows if runtime.GOOS == "windows" && container.IsRunning() { return "", fmt.Errorf("Windows does not support commit of a running container") } if c.Pause && !container.isPaused() { daemon.containerPause(container) defer daemon.containerUnpause(container) } if c.MergeConfigs { if err := runconfig.Merge(c.Config, container.Config); err != nil { return "", err } } rwTar, err := daemon.exportContainerRw(container) if err != nil { return "", err } defer func() { if rwTar != nil { rwTar.Close() } }() var history []image.History rootFS := image.NewRootFS() if container.ImageID != "" { img, err := daemon.imageStore.Get(container.ImageID) if err != nil { return "", err } history = img.History rootFS = img.RootFS } l, err := daemon.layerStore.Register(rwTar, rootFS.ChainID()) if err != nil { return "", err } defer layer.ReleaseAndLog(daemon.layerStore, l) h := image.History{ Author: c.Author, Created: time.Now().UTC(), CreatedBy: strings.Join(container.Config.Cmd.Slice(), " "), Comment: c.Comment, EmptyLayer: true, } if diffID := l.DiffID(); layer.DigestSHA256EmptyTar != diffID { h.EmptyLayer = false rootFS.Append(diffID) } history = append(history, h) config, err := json.Marshal(&image.Image{ V1Image: image.V1Image{ DockerVersion: dockerversion.Version, Config: c.Config, Architecture: runtime.GOARCH, OS: runtime.GOOS, Container: container.ID, ContainerConfig: *container.Config, Author: c.Author, Created: h.Created, }, RootFS: rootFS, History: history, }) if err != nil { return "", err } id, err := daemon.imageStore.Create(config) if err != nil { return "", err } if container.ImageID != "" { if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil { return "", err } } if c.Repo != "" { newTag, err := reference.WithName(c.Repo) // todo: should move this to API layer if err != nil { return "", err } if c.Tag != "" { if newTag, err = reference.WithTag(newTag, c.Tag); err != nil { return "", err } } if err := daemon.TagImage(newTag, id.String(), true); err != nil { return "", err } } daemon.LogContainerEvent(container, "commit") return id.String(), nil }
func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) { if fs, ok := pd.layer.(distribution.Describable); ok { if d := fs.Descriptor(); len(d.URLs) > 0 { progress.Update(progressOutput, pd.ID(), "Skipped foreign layer") return d, nil } } diffID := pd.DiffID() pd.pushState.Lock() if descriptor, ok := pd.pushState.remoteLayers[diffID]; ok { // it is already known that the push is not needed and // therefore doing a stat is unnecessary pd.pushState.Unlock() progress.Update(progressOutput, pd.ID(), "Layer already exists") return descriptor, nil } pd.pushState.Unlock() // Do we have any metadata associated with this layer's DiffID? v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID) if err == nil { descriptor, exists, err := layerAlreadyExists(ctx, v2Metadata, pd.repoInfo, pd.repo, pd.pushState) if err != nil { progress.Update(progressOutput, pd.ID(), "Image push failed") return distribution.Descriptor{}, retryOnError(err) } if exists { progress.Update(progressOutput, pd.ID(), "Layer already exists") pd.pushState.Lock() pd.pushState.remoteLayers[diffID] = descriptor pd.pushState.Unlock() return descriptor, nil } } logrus.Debugf("Pushing layer: %s", diffID) // if digest was empty or not saved, or if blob does not exist on the remote repository, // then push the blob. bs := pd.repo.Blobs(ctx) var layerUpload distribution.BlobWriter mountAttemptsRemaining := 3 // Attempt to find another repository in the same registry to mount the layer // from to avoid an unnecessary upload. // Note: metadata is stored from oldest to newest, so we iterate through this // slice in reverse to maximize our chances of the blob still existing in the // remote repository. for i := len(v2Metadata) - 1; i >= 0 && mountAttemptsRemaining > 0; i-- { mountFrom := v2Metadata[i] sourceRepo, err := reference.ParseNamed(mountFrom.SourceRepository) if err != nil { continue } if pd.repoInfo.Hostname() != sourceRepo.Hostname() { // don't mount blobs from another registry continue } namedRef, err := reference.WithName(mountFrom.SourceRepository) if err != nil { continue } // TODO (brianbland): We need to construct a reference where the Name is // only the full remote name, so clean this up when distribution has a // richer reference package remoteRef, err := distreference.WithName(namedRef.RemoteName()) if err != nil { continue } canonicalRef, err := distreference.WithDigest(remoteRef, mountFrom.Digest) if err != nil { continue } logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountFrom.Digest, sourceRepo.FullName()) layerUpload, err = bs.Create(ctx, client.WithMountFrom(canonicalRef)) switch err := err.(type) { case distribution.ErrBlobMounted: progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name()) err.Descriptor.MediaType = schema2.MediaTypeLayer pd.pushState.Lock() pd.pushState.confirmedV2 = true pd.pushState.remoteLayers[diffID] = err.Descriptor pd.pushState.Unlock() // Cache mapping from this layer's DiffID to the blobsum if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: mountFrom.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil { return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} } return err.Descriptor, nil case nil: // blob upload session created successfully, so begin the upload mountAttemptsRemaining = 0 default: // unable to mount layer from this repository, so this source mapping is no longer valid logrus.Debugf("unassociating layer %s (%s) with %s", diffID, mountFrom.Digest, mountFrom.SourceRepository) pd.v2MetadataService.Remove(mountFrom) mountAttemptsRemaining-- } } if layerUpload == nil { layerUpload, err = bs.Create(ctx) if err != nil { return distribution.Descriptor{}, retryOnError(err) } } defer layerUpload.Close() arch, err := pd.layer.TarStream() if err != nil { return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} } // don't care if this fails; best effort size, _ := pd.layer.DiffSize() reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, arch), progressOutput, size, pd.ID(), "Pushing") compressedReader, compressionDone := compress(reader) defer func() { reader.Close() <-compressionDone }() digester := digest.Canonical.New() tee := io.TeeReader(compressedReader, digester.Hash()) nn, err := layerUpload.ReadFrom(tee) compressedReader.Close() if err != nil { return distribution.Descriptor{}, retryOnError(err) } pushDigest := digester.Digest() if _, err := layerUpload.Commit(ctx, distribution.Descriptor{Digest: pushDigest}); err != nil { return distribution.Descriptor{}, retryOnError(err) } logrus.Debugf("uploaded layer %s (%s), %d bytes", diffID, pushDigest, nn) progress.Update(progressOutput, pd.ID(), "Pushed") // Cache mapping from this layer's DiffID to the blobsum if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: pushDigest, SourceRepository: pd.repoInfo.FullName()}); err != nil { return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} } pd.pushState.Lock() // If Commit succeeded, that's an indication that the remote registry // speaks the v2 protocol. pd.pushState.confirmedV2 = true descriptor := distribution.Descriptor{ Digest: pushDigest, MediaType: schema2.MediaTypeLayer, Size: nn, } pd.pushState.remoteLayers[diffID] = descriptor pd.pushState.Unlock() return descriptor, nil }
// localNameFromRemote combines the index name and the repo remote name // to generate a repo local name. func localNameFromRemote(indexName string, remoteName reference.Named) (reference.Named, error) { return reference.WithName(indexName + "/" + remoteName.Name()) }
func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) { if fs, ok := pd.layer.(distribution.Describable); ok { if d := fs.Descriptor(); len(d.URLs) > 0 { progress.Update(progressOutput, pd.ID(), "Skipped foreign layer") return d, nil } } diffID := pd.DiffID() pd.pushState.Lock() if descriptor, ok := pd.pushState.remoteLayers[diffID]; ok { // it is already known that the push is not needed and // therefore doing a stat is unnecessary pd.pushState.Unlock() progress.Update(progressOutput, pd.ID(), "Layer already exists") return descriptor, nil } pd.pushState.Unlock() maxMountAttempts, maxExistenceChecks, checkOtherRepositories := getMaxMountAndExistenceCheckAttempts(pd.layer) // Do we have any metadata associated with this layer's DiffID? v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID) if err == nil { // check for blob existence in the target repository if we have a mapping with it descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, false, 1, v2Metadata) if exists || err != nil { return descriptor, err } } // if digest was empty or not saved, or if blob does not exist on the remote repository, // then push the blob. bs := pd.repo.Blobs(ctx) var layerUpload distribution.BlobWriter // Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload candidates := getRepositoryMountCandidates(pd.repoInfo, pd.hmacKey, maxMountAttempts, v2Metadata) for _, mountCandidate := range candidates { logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountCandidate.Digest, mountCandidate.SourceRepository) createOpts := []distribution.BlobCreateOption{} if len(mountCandidate.SourceRepository) > 0 { namedRef, err := reference.WithName(mountCandidate.SourceRepository) if err != nil { logrus.Errorf("failed to parse source repository reference %v: %v", namedRef.String(), err) pd.v2MetadataService.Remove(mountCandidate) continue } // TODO (brianbland): We need to construct a reference where the Name is // only the full remote name, so clean this up when distribution has a // richer reference package remoteRef, err := distreference.WithName(namedRef.RemoteName()) if err != nil { logrus.Errorf("failed to make remote reference out of %q: %v", namedRef.RemoteName(), namedRef.RemoteName()) continue } canonicalRef, err := distreference.WithDigest(distreference.TrimNamed(remoteRef), mountCandidate.Digest) if err != nil { logrus.Errorf("failed to make canonical reference: %v", err) continue } createOpts = append(createOpts, client.WithMountFrom(canonicalRef)) } // send the layer lu, err := bs.Create(ctx, createOpts...) switch err := err.(type) { case nil: // noop case distribution.ErrBlobMounted: progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name()) err.Descriptor.MediaType = schema2.MediaTypeLayer pd.pushState.Lock() pd.pushState.confirmedV2 = true pd.pushState.remoteLayers[diffID] = err.Descriptor pd.pushState.Unlock() // Cache mapping from this layer's DiffID to the blobsum if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{ Digest: err.Descriptor.Digest, SourceRepository: pd.repoInfo.FullName(), }); err != nil { return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} } return err.Descriptor, nil default: logrus.Infof("failed to mount layer %s (%s) from %s: %v", diffID, mountCandidate.Digest, mountCandidate.SourceRepository, err) } if len(mountCandidate.SourceRepository) > 0 && (metadata.CheckV2MetadataHMAC(&mountCandidate, pd.hmacKey) || len(mountCandidate.HMAC) == 0) { cause := "blob mount failure" if err != nil { cause = fmt.Sprintf("an error: %v", err.Error()) } logrus.Debugf("removing association between layer %s and %s due to %s", mountCandidate.Digest, mountCandidate.SourceRepository, cause) pd.v2MetadataService.Remove(mountCandidate) } if lu != nil { // cancel previous upload cancelLayerUpload(ctx, mountCandidate.Digest, layerUpload) layerUpload = lu } } if maxExistenceChecks-len(pd.checkedDigests) > 0 { // do additional layer existence checks with other known digests if any descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, checkOtherRepositories, maxExistenceChecks-len(pd.checkedDigests), v2Metadata) if exists || err != nil { return descriptor, err } } logrus.Debugf("Pushing layer: %s", diffID) if layerUpload == nil { layerUpload, err = bs.Create(ctx) if err != nil { return distribution.Descriptor{}, retryOnError(err) } } defer layerUpload.Close() // upload the blob desc, err := pd.uploadUsingSession(ctx, progressOutput, diffID, layerUpload) if err != nil { return desc, err } return desc, nil }
func TestAddDeleteGet(t *testing.T) { jsonFile, err := ioutil.TempFile("", "tag-store-test") if err != nil { t.Fatalf("error creating temp file: %v", err) } _, err = jsonFile.Write([]byte(`{}`)) jsonFile.Close() defer os.RemoveAll(jsonFile.Name()) store, err := NewTagStore(jsonFile.Name()) if err != nil { t.Fatalf("error creating tag store: %v", err) } testImageID1 := image.ID("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9c") testImageID2 := image.ID("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9d") testImageID3 := image.ID("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9e") // Try adding a reference with no tag or digest nameOnly, err := reference.WithName("username/repo") if err != nil { t.Fatalf("could not parse reference: %v", err) } if err = store.AddTag(nameOnly, testImageID1, false); err != nil { t.Fatalf("error adding to store: %v", err) } // Add a few references ref1, err := reference.ParseNamed("username/repo1:latest") if err != nil { t.Fatalf("could not parse reference: %v", err) } if err = store.AddTag(ref1, testImageID1, false); err != nil { t.Fatalf("error adding to store: %v", err) } ref2, err := reference.ParseNamed("username/repo1:old") if err != nil { t.Fatalf("could not parse reference: %v", err) } if err = store.AddTag(ref2, testImageID2, false); err != nil { t.Fatalf("error adding to store: %v", err) } ref3, err := reference.ParseNamed("username/repo1:alias") if err != nil { t.Fatalf("could not parse reference: %v", err) } if err = store.AddTag(ref3, testImageID1, false); err != nil { t.Fatalf("error adding to store: %v", err) } ref4, err := reference.ParseNamed("username/repo2:latest") if err != nil { t.Fatalf("could not parse reference: %v", err) } if err = store.AddTag(ref4, testImageID2, false); err != nil { t.Fatalf("error adding to store: %v", err) } ref5, err := reference.ParseNamed("username/repo3@sha256:58153dfb11794fad694460162bf0cb0a4fa710cfa3f60979c177d920813e267c") if err != nil { t.Fatalf("could not parse reference: %v", err) } if err = store.AddDigest(ref5.(reference.Canonical), testImageID2, false); err != nil { t.Fatalf("error adding to store: %v", err) } // Attempt to overwrite with force == false if err = store.AddTag(ref4, testImageID3, false); err == nil || !strings.HasPrefix(err.Error(), "Conflict:") { t.Fatalf("did not get expected error on overwrite attempt - got %v", err) } // Repeat to overwrite with force == true if err = store.AddTag(ref4, testImageID3, true); err != nil { t.Fatalf("failed to force tag overwrite: %v", err) } // Check references so far id, err := store.Get(nameOnly) if err != nil { t.Fatalf("Get returned error: %v", err) } if id != testImageID1 { t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String()) } id, err = store.Get(ref1) if err != nil { t.Fatalf("Get returned error: %v", err) } if id != testImageID1 { t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String()) } id, err = store.Get(ref2) if err != nil { t.Fatalf("Get returned error: %v", err) } if id != testImageID2 { t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID2.String()) } id, err = store.Get(ref3) if err != nil { t.Fatalf("Get returned error: %v", err) } if id != testImageID1 { t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String()) } id, err = store.Get(ref4) if err != nil { t.Fatalf("Get returned error: %v", err) } if id != testImageID3 { t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID3.String()) } id, err = store.Get(ref5) if err != nil { t.Fatalf("Get returned error: %v", err) } if id != testImageID2 { t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID3.String()) } // Get should return ErrDoesNotExist for a nonexistent repo nonExistRepo, err := reference.ParseNamed("username/nonexistrepo:latest") if err != nil { t.Fatalf("could not parse reference: %v", err) } if _, err = store.Get(nonExistRepo); err != ErrDoesNotExist { t.Fatal("Expected ErrDoesNotExist from Get") } // Get should return ErrDoesNotExist for a nonexistent tag nonExistTag, err := reference.ParseNamed("username/repo1:nonexist") if err != nil { t.Fatalf("could not parse reference: %v", err) } if _, err = store.Get(nonExistTag); err != ErrDoesNotExist { t.Fatal("Expected ErrDoesNotExist from Get") } // Check References refs := store.References(testImageID1) sort.Sort(LexicalRefs(refs)) if len(refs) != 3 { t.Fatal("unexpected number of references") } if refs[0].String() != ref3.String() { t.Fatalf("unexpected reference: %v", refs[0].String()) } if refs[1].String() != ref1.String() { t.Fatalf("unexpected reference: %v", refs[1].String()) } if refs[2].String() != nameOnly.String()+":latest" { t.Fatalf("unexpected reference: %v", refs[2].String()) } // Check ReferencesByName repoName, err := reference.WithName("username/repo1") if err != nil { t.Fatalf("could not parse reference: %v", err) } associations := store.ReferencesByName(repoName) sort.Sort(LexicalAssociations(associations)) if len(associations) != 3 { t.Fatal("unexpected number of associations") } if associations[0].Ref.String() != ref3.String() { t.Fatalf("unexpected reference: %v", associations[0].Ref.String()) } if associations[0].ImageID != testImageID1 { t.Fatalf("unexpected reference: %v", associations[0].Ref.String()) } if associations[1].Ref.String() != ref1.String() { t.Fatalf("unexpected reference: %v", associations[1].Ref.String()) } if associations[1].ImageID != testImageID1 { t.Fatalf("unexpected reference: %v", associations[1].Ref.String()) } if associations[2].Ref.String() != ref2.String() { t.Fatalf("unexpected reference: %v", associations[2].Ref.String()) } if associations[2].ImageID != testImageID2 { t.Fatalf("unexpected reference: %v", associations[2].Ref.String()) } // Delete should return ErrDoesNotExist for a nonexistent repo if _, err = store.Delete(nonExistRepo); err != ErrDoesNotExist { t.Fatal("Expected ErrDoesNotExist from Delete") } // Delete should return ErrDoesNotExist for a nonexistent tag if _, err = store.Delete(nonExistTag); err != ErrDoesNotExist { t.Fatal("Expected ErrDoesNotExist from Delete") } // Delete a few references if deleted, err := store.Delete(ref1); err != nil || deleted != true { t.Fatal("Delete failed") } if _, err := store.Get(ref1); err != ErrDoesNotExist { t.Fatal("Expected ErrDoesNotExist from Get") } if deleted, err := store.Delete(ref5); err != nil || deleted != true { t.Fatal("Delete failed") } if _, err := store.Get(ref5); err != ErrDoesNotExist { t.Fatal("Expected ErrDoesNotExist from Get") } if deleted, err := store.Delete(nameOnly); err != nil || deleted != true { t.Fatal("Delete failed") } if _, err := store.Get(nameOnly); err != ErrDoesNotExist { t.Fatal("Expected ErrDoesNotExist from Get") } }
func (r *mockRepository) Named() reference.Named { named, _ := reference.WithName("test") return named }
func main() { logger := lager.NewLogger("http") var request CheckRequest err := json.NewDecoder(os.Stdin).Decode(&request) fatalIf("failed to read request", err) os.Setenv("AWS_ACCESS_KEY_ID", request.Source.AWSAccessKeyID) os.Setenv("AWS_SECRET_ACCESS_KEY", request.Source.AWSSecretAccessKey) // silence benign ecr-login errors/warnings seelog.UseLogger(seelog.Disabled) ecrUser, ecrPass, err := ecr.ECRHelper{ ClientFactory: ecrapi.DefaultClientFactory{}, }.Get(request.Source.Repository) if err == nil { request.Source.Username = ecrUser request.Source.Password = ecrPass } registryHost, repo := parseRepository(request.Source.Repository) if len(request.Source.RegistryMirror) > 0 { registryMirrorUrl, err := url.Parse(request.Source.RegistryMirror) fatalIf("failed to parse registry mirror URL", err) registryHost = registryMirrorUrl.Host } tag := request.Source.Tag if tag == "" { tag = "latest" } transport, registryURL := makeTransport(logger, request, registryHost, repo) client := &http.Client{ Transport: retryRoundTripper(logger, transport), } ub, err := v2.NewURLBuilderFromString(registryURL, false) fatalIf("failed to construct registry URL builder", err) namedRef, err := reference.WithName(repo) fatalIf("failed to construct named reference", err) var response CheckResponse taggedRef, err := reference.WithTag(namedRef, tag) fatalIf("failed to construct tagged reference", err) latestManifestURL, err := ub.BuildManifestURL(taggedRef) fatalIf("failed to build latest manifest URL", err) latestDigest, foundLatest := fetchDigest(client, latestManifestURL) if request.Version.Digest != "" { digestRef, err := reference.WithDigest(namedRef, digest.Digest(request.Version.Digest)) fatalIf("failed to build cursor manifest URL", err) cursorManifestURL, err := ub.BuildManifestURL(digestRef) fatalIf("failed to build manifest URL", err) cursorDigest, foundCursor := fetchDigest(client, cursorManifestURL) if foundCursor && cursorDigest != latestDigest { response = append(response, Version{cursorDigest}) } } if foundLatest { response = append(response, Version{latestDigest}) } json.NewEncoder(os.Stdout).Encode(response) }
func TestParseRepositoryInfo(t *testing.T) { withName := func(name string) reference.Named { named, err := reference.WithName(name) if err != nil { t.Fatalf("could not parse reference %s", name) } return named } expectedRepoInfos := map[string]RepositoryInfo{ "fooo/bar": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, RemoteName: withName("fooo/bar"), LocalName: withName("fooo/bar"), CanonicalName: withName("docker.io/fooo/bar"), Official: false, }, "library/ubuntu": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, RemoteName: withName("library/ubuntu"), LocalName: withName("ubuntu"), CanonicalName: withName("docker.io/library/ubuntu"), Official: true, }, "nonlibrary/ubuntu": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, RemoteName: withName("nonlibrary/ubuntu"), LocalName: withName("nonlibrary/ubuntu"), CanonicalName: withName("docker.io/nonlibrary/ubuntu"), Official: false, }, "ubuntu": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, RemoteName: withName("library/ubuntu"), LocalName: withName("ubuntu"), CanonicalName: withName("docker.io/library/ubuntu"), Official: true, }, "other/library": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, RemoteName: withName("other/library"), LocalName: withName("other/library"), CanonicalName: withName("docker.io/other/library"), Official: false, }, "127.0.0.1:8000/private/moonbase": { Index: ®istrytypes.IndexInfo{ Name: "127.0.0.1:8000", Official: false, }, RemoteName: withName("private/moonbase"), LocalName: withName("127.0.0.1:8000/private/moonbase"), CanonicalName: withName("127.0.0.1:8000/private/moonbase"), Official: false, }, "127.0.0.1:8000/privatebase": { Index: ®istrytypes.IndexInfo{ Name: "127.0.0.1:8000", Official: false, }, RemoteName: withName("privatebase"), LocalName: withName("127.0.0.1:8000/privatebase"), CanonicalName: withName("127.0.0.1:8000/privatebase"), Official: false, }, "localhost:8000/private/moonbase": { Index: ®istrytypes.IndexInfo{ Name: "localhost:8000", Official: false, }, RemoteName: withName("private/moonbase"), LocalName: withName("localhost:8000/private/moonbase"), CanonicalName: withName("localhost:8000/private/moonbase"), Official: false, }, "localhost:8000/privatebase": { Index: ®istrytypes.IndexInfo{ Name: "localhost:8000", Official: false, }, RemoteName: withName("privatebase"), LocalName: withName("localhost:8000/privatebase"), CanonicalName: withName("localhost:8000/privatebase"), Official: false, }, "example.com/private/moonbase": { Index: ®istrytypes.IndexInfo{ Name: "example.com", Official: false, }, RemoteName: withName("private/moonbase"), LocalName: withName("example.com/private/moonbase"), CanonicalName: withName("example.com/private/moonbase"), Official: false, }, "example.com/privatebase": { Index: ®istrytypes.IndexInfo{ Name: "example.com", Official: false, }, RemoteName: withName("privatebase"), LocalName: withName("example.com/privatebase"), CanonicalName: withName("example.com/privatebase"), Official: false, }, "example.com:8000/private/moonbase": { Index: ®istrytypes.IndexInfo{ Name: "example.com:8000", Official: false, }, RemoteName: withName("private/moonbase"), LocalName: withName("example.com:8000/private/moonbase"), CanonicalName: withName("example.com:8000/private/moonbase"), Official: false, }, "example.com:8000/privatebase": { Index: ®istrytypes.IndexInfo{ Name: "example.com:8000", Official: false, }, RemoteName: withName("privatebase"), LocalName: withName("example.com:8000/privatebase"), CanonicalName: withName("example.com:8000/privatebase"), Official: false, }, "localhost/private/moonbase": { Index: ®istrytypes.IndexInfo{ Name: "localhost", Official: false, }, RemoteName: withName("private/moonbase"), LocalName: withName("localhost/private/moonbase"), CanonicalName: withName("localhost/private/moonbase"), Official: false, }, "localhost/privatebase": { Index: ®istrytypes.IndexInfo{ Name: "localhost", Official: false, }, RemoteName: withName("privatebase"), LocalName: withName("localhost/privatebase"), CanonicalName: withName("localhost/privatebase"), Official: false, }, IndexName + "/public/moonbase": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, RemoteName: withName("public/moonbase"), LocalName: withName("public/moonbase"), CanonicalName: withName("docker.io/public/moonbase"), Official: false, }, "index." + IndexName + "/public/moonbase": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, RemoteName: withName("public/moonbase"), LocalName: withName("public/moonbase"), CanonicalName: withName("docker.io/public/moonbase"), Official: false, }, "ubuntu-12.04-base": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, RemoteName: withName("library/ubuntu-12.04-base"), LocalName: withName("ubuntu-12.04-base"), CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), Official: true, }, IndexName + "/ubuntu-12.04-base": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, RemoteName: withName("library/ubuntu-12.04-base"), LocalName: withName("ubuntu-12.04-base"), CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), Official: true, }, "index." + IndexName + "/ubuntu-12.04-base": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, RemoteName: withName("library/ubuntu-12.04-base"), LocalName: withName("ubuntu-12.04-base"), CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), Official: true, }, } for reposName, expectedRepoInfo := range expectedRepoInfos { named, err := reference.WithName(reposName) if err != nil { t.Error(err) } repoInfo, err := ParseRepositoryInfo(named) if err != nil { t.Error(err) } else { checkEqual(t, repoInfo.Index.Name, expectedRepoInfo.Index.Name, reposName) checkEqual(t, repoInfo.RemoteName.String(), expectedRepoInfo.RemoteName.String(), reposName) checkEqual(t, repoInfo.LocalName.String(), expectedRepoInfo.LocalName.String(), reposName) checkEqual(t, repoInfo.CanonicalName.String(), expectedRepoInfo.CanonicalName.String(), reposName) checkEqual(t, repoInfo.Index.Official, expectedRepoInfo.Index.Official, reposName) checkEqual(t, repoInfo.Official, expectedRepoInfo.Official, reposName) } } }
func migrateTags(root, driverName string, ts tagAdder, mappings map[string]image.ID) error { migrationFile := filepath.Join(root, migrationTagsFileName) if _, err := os.Lstat(migrationFile); !os.IsNotExist(err) { return err } type repositories struct { Repositories map[string]map[string]string } var repos repositories f, err := os.Open(filepath.Join(root, repositoriesFilePrefixLegacy+driverName)) if err != nil { if os.IsNotExist(err) { return nil } return err } defer f.Close() if err := json.NewDecoder(f).Decode(&repos); err != nil { return err } for name, repo := range repos.Repositories { for tag, id := range repo { if strongID, exists := mappings[id]; exists { ref, err := reference.WithName(name) if err != nil { logrus.Errorf("migrate tags: invalid name %q, %q", name, err) continue } if dgst, err := digest.ParseDigest(tag); err == nil { canonical, err := reference.WithDigest(ref, dgst) if err != nil { logrus.Errorf("migrate tags: invalid digest %q, %q", dgst, err) continue } if err := ts.AddDigest(canonical, strongID, false); err != nil { logrus.Errorf("can't migrate digest %q for %q, err: %q", ref.String(), strongID, err) } } else { tagRef, err := reference.WithTag(ref, tag) if err != nil { logrus.Errorf("migrate tags: invalid tag %q, %q", tag, err) continue } if err := ts.AddTag(tagRef, strongID, false); err != nil { logrus.Errorf("can't migrate tag %q for %q, err: %q", ref.String(), strongID, err) } } logrus.Infof("migrated tag %s:%s to point to %s", name, tag, strongID) } } } mf, err := os.Create(migrationFile) if err != nil { return err } mf.Close() return nil }
func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) error { diffID := pd.DiffID() pd.pushState.Lock() if _, ok := pd.pushState.remoteLayers[diffID]; ok { // it is already known that the push is not needed and // therefore doing a stat is unnecessary pd.pushState.Unlock() progress.Update(progressOutput, pd.ID(), "Layer already exists") return nil } pd.pushState.Unlock() // Do we have any metadata associated with this layer's DiffID? v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID) if err == nil { descriptor, exists, err := layerAlreadyExists(ctx, v2Metadata, pd.repoInfo, pd.repo, pd.pushState) if err != nil { progress.Update(progressOutput, pd.ID(), "Image push failed") return retryOnError(err) } if exists { progress.Update(progressOutput, pd.ID(), "Layer already exists") pd.pushState.Lock() pd.pushState.remoteLayers[diffID] = descriptor pd.pushState.Unlock() return nil } } logrus.Debugf("Pushing layer: %s", diffID) // if digest was empty or not saved, or if blob does not exist on the remote repository, // then push the blob. bs := pd.repo.Blobs(ctx) var mountFrom metadata.V2Metadata // Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload for _, metadata := range v2Metadata { sourceRepo, err := reference.ParseNamed(metadata.SourceRepository) if err != nil { continue } if pd.repoInfo.Hostname() == sourceRepo.Hostname() { logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, metadata.Digest, sourceRepo.FullName()) mountFrom = metadata break } } var createOpts []distribution.BlobCreateOption if mountFrom.SourceRepository != "" { namedRef, err := reference.WithName(mountFrom.SourceRepository) if err != nil { return err } // TODO (brianbland): We need to construct a reference where the Name is // only the full remote name, so clean this up when distribution has a // richer reference package remoteRef, err := distreference.WithName(namedRef.RemoteName()) if err != nil { return err } canonicalRef, err := distreference.WithDigest(remoteRef, mountFrom.Digest) if err != nil { return err } createOpts = append(createOpts, client.WithMountFrom(canonicalRef)) } // Send the layer layerUpload, err := bs.Create(ctx, createOpts...) switch err := err.(type) { case distribution.ErrBlobMounted: progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name()) err.Descriptor.MediaType = schema2.MediaTypeLayer pd.pushState.Lock() pd.pushState.confirmedV2 = true pd.pushState.remoteLayers[diffID] = err.Descriptor pd.pushState.Unlock() // Cache mapping from this layer's DiffID to the blobsum if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: mountFrom.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil { return xfer.DoNotRetry{Err: err} } return nil } if mountFrom.SourceRepository != "" { // unable to mount layer from this repository, so this source mapping is no longer valid logrus.Debugf("unassociating layer %s (%s) with %s", diffID, mountFrom.Digest, mountFrom.SourceRepository) pd.v2MetadataService.Remove(mountFrom) } if err != nil { return retryOnError(err) } defer layerUpload.Close() arch, err := pd.layer.TarStream() if err != nil { return xfer.DoNotRetry{Err: err} } // don't care if this fails; best effort size, _ := pd.layer.DiffSize() reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, arch), progressOutput, size, pd.ID(), "Pushing") compressedReader, compressionDone := compress(reader) defer func() { reader.Close() <-compressionDone }() digester := digest.Canonical.New() tee := io.TeeReader(compressedReader, digester.Hash()) nn, err := layerUpload.ReadFrom(tee) compressedReader.Close() if err != nil { return retryOnError(err) } pushDigest := digester.Digest() if _, err := layerUpload.Commit(ctx, distribution.Descriptor{Digest: pushDigest}); err != nil { return retryOnError(err) } logrus.Debugf("uploaded layer %s (%s), %d bytes", diffID, pushDigest, nn) progress.Update(progressOutput, pd.ID(), "Pushed") // Cache mapping from this layer's DiffID to the blobsum if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: pushDigest, SourceRepository: pd.repoInfo.FullName()}); err != nil { return xfer.DoNotRetry{Err: err} } pd.pushState.Lock() // If Commit succeded, that's an indication that the remote registry // speaks the v2 protocol. pd.pushState.confirmedV2 = true pd.pushState.remoteLayers[diffID] = distribution.Descriptor{ Digest: pushDigest, MediaType: schema2.MediaTypeLayer, Size: nn, } pd.pushState.Unlock() return nil }
func TestValidRemoteName(t *testing.T) { validRepositoryNames := []string{ // Sanity check. "docker/docker", // Allow 64-character non-hexadecimal names (hexadecimal names are forbidden). "thisisthesongthatneverendsitgoesonandonandonthisisthesongthatnev", // Allow embedded hyphens. "docker-rules/docker", // Allow multiple hyphens as well. "docker---rules/docker", //Username doc and image name docker being tested. "doc/docker", // single character names are now allowed. "d/docker", "jess/t", // Consecutive underscores. "dock__er/docker", } for _, repositoryName := range validRepositoryNames { repositoryRef, err := reference.WithName(repositoryName) if err != nil { t.Errorf("Repository name should be valid: %v. Error: %v", repositoryName, err) } if err := validateRemoteName(repositoryRef); err != nil { t.Errorf("Repository name should be valid: %v. Error: %v", repositoryName, err) } } invalidRepositoryNames := []string{ // Disallow capital letters. "docker/Docker", // Only allow one slash. "docker///docker", // Disallow 64-character hexadecimal. "1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", // Disallow leading and trailing hyphens in namespace. "-docker/docker", "docker-/docker", "-docker-/docker", // Don't allow underscores everywhere (as opposed to hyphens). "____/____", "_docker/_docker", // Disallow consecutive periods. "dock..er/docker", "dock_.er/docker", "dock-.er/docker", // No repository. "docker/", //namespace too long "this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255/docker", } for _, repositoryName := range invalidRepositoryNames { repositoryRef, err := reference.ParseNamed(repositoryName) if err != nil { continue } if err := validateRemoteName(repositoryRef); err == nil { t.Errorf("Repository name should be invalid: %v", repositoryName) } } }