func TestMirrorEndpointLookup(t *testing.T) {
containsMirror := func(endpoints []APIEndpoint) bool {
for _, pe := range endpoints {
if pe.URL.Host == "my.mirror" {
return true
}
}
return false
}
s := DefaultService{config: makeServiceConfig([]string{"my.mirror"}, nil)}
imageName, err := reference.WithName(IndexName + "/test/image")
if err != nil {
t.Error(err)
}
pushAPIEndpoints, err := s.LookupPushEndpoints(imageName.Hostname())
if err != nil {
t.Fatal(err)
}
if containsMirror(pushAPIEndpoints) {
t.Fatal("Push endpoint should not contain mirror")
}
pullAPIEndpoints, err := s.LookupPullEndpoints(imageName.Hostname())
if err != nil {
t.Fatal(err)
}
if !containsMirror(pullAPIEndpoints) {
t.Fatal("Pull endpoint should contain mirror")
}
}
// Add adds an image to the image cache
func (ic *ICache) Add(imageConfig *metadata.ImageConfig) {
defer trace.End(trace.Begin(""))
ic.m.Lock()
defer ic.m.Unlock()
// Normalize the name stored in imageConfig using Docker's reference code
ref, err := reference.WithName(imageConfig.Name)
if err != nil {
log.Errorf("Tried to create reference from %s: %s", imageConfig.Name, err.Error())
return
}
imageID := prefixImageID(imageConfig.ImageID)
ic.iDIndex.Add(imageConfig.ImageID)
ic.cacheByID[imageID] = imageConfig
for _, tag := range imageConfig.Tags {
ref, err = reference.WithTag(ref, tag)
if err != nil {
log.Errorf("Tried to create tagged reference from %s and tag %s: %s", imageConfig.Name, tag, err.Error())
return
}
ic.cacheByName[imageConfig.Reference] = imageConfig
}
}
func (l *tarexporter) legacyLoad(tmpDir string, outStream io.Writer, progressOutput progress.Output) error {
legacyLoadedMap := make(map[string]image.ID)
dirs, err := ioutil.ReadDir(tmpDir)
if err != nil {
return err
}
// every dir represents an image
for _, d := range dirs {
if d.IsDir() {
if err := l.legacyLoadImage(d.Name(), tmpDir, legacyLoadedMap, progressOutput); err != nil {
return err
}
}
}
// load tags from repositories file
repositoriesPath, err := safePath(tmpDir, legacyRepositoriesFileName)
if err != nil {
return err
}
repositoriesFile, err := os.Open(repositoriesPath)
if err != nil {
if !os.IsNotExist(err) {
return err
}
return repositoriesFile.Close()
}
defer repositoriesFile.Close()
repositories := make(map[string]map[string]string)
if err := json.NewDecoder(repositoriesFile).Decode(&repositories); err != nil {
return err
}
for name, tagMap := range repositories {
for tag, oldID := range tagMap {
imgID, ok := legacyLoadedMap[oldID]
if !ok {
return fmt.Errorf("invalid target ID: %v", oldID)
}
named, err := reference.WithName(name)
if err != nil {
return err
}
ref, err := reference.WithTag(named, tag)
if err != nil {
return err
}
l.setLoadedTag(ref, imgID, outStream)
}
}
return nil
}
// TagImage creates the tag specified by newTag, pointing to the image named
// imageName (alternatively, imageName can also be an image ID).
func (daemon *Daemon) TagImage(imageName, repository, tag string) error {
imageID, err := daemon.GetImageID(imageName)
if err != nil {
return err
}
newTag, err := reference.WithName(repository)
if err != nil {
return err
}
if tag != "" {
if newTag, err = reference.WithTag(newTag, tag); err != nil {
return err
}
}
return daemon.TagImageWithReference(imageID, newTag)
}
// AddImage adds an image to the image cache
func (ic *ICache) AddImage(imageConfig *metadata.ImageConfig) {
ic.m.Lock()
defer ic.m.Unlock()
var imageID string
// Don't assume the image id in image has "sha256:<id> as format. We store it in
// this fomat to make it easier to lookup by digest
if strings.HasPrefix(imageConfig.ImageID, "sha") {
imageID = imageConfig.ImageID
} else {
imageID = "sha256:" + imageConfig.ImageID
}
// add image id to truncindex
ic.idIndex.Add(imageConfig.ImageID)
ic.cacheByID[imageID] = imageConfig
// Normalize the name stored in imageConfig using Docker's reference code
ref, err := reference.WithName(imageConfig.Name)
if err != nil {
log.Errorf("Tried to create reference from %s: %s", imageConfig.Name, err.Error())
return
}
for _, tag := range imageConfig.Tags {
ref, err = reference.WithTag(ref, tag)
if err != nil {
log.Errorf("Tried to create tagged reference from %s and tag %s: %s", imageConfig.Name, tag, err.Error())
return
}
if tagged, ok := ref.(reference.NamedTagged); ok {
taggedName := fmt.Sprintf("%s:%s", tagged.Name(), tagged.Tag())
ic.cacheByName[taggedName] = imageConfig
} else {
ic.cacheByName[ref.Name()] = imageConfig
}
}
}
func handlerPutTag(w http.ResponseWriter, r *http.Request) {
if !requiresAuth(w, r) {
return
}
vars := mux.Vars(r)
repositoryName, err := reference.WithName(vars["repository"])
if err != nil {
apiError(w, "Could not parse repository", 400)
return
}
tagName := vars["tag"]
tags, exists := testRepositories[repositoryName.String()]
if !exists {
tags = make(map[string]string)
testRepositories[repositoryName.String()] = tags
}
tagValue := ""
readJSON(r, tagValue)
tags[tagName] = tagValue
writeResponse(w, true, 200)
}
func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) {
if !requiresAuth(w, r) {
return
}
repositoryName, err := reference.WithName(mux.Vars(r)["repository"])
if err != nil {
apiError(w, "Could not parse repository", 400)
return
}
tags, exists := testRepositories[repositoryName.String()]
if !exists {
apiError(w, "Repository not found", 404)
return
}
if r.Method == "DELETE" {
delete(testRepositories, repositoryName.String())
writeResponse(w, true, 200)
return
}
writeResponse(w, tags, 200)
}
func (s *router) postImagesTag(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
if err := httputils.ParseForm(r); err != nil {
return err
}
repo := r.Form.Get("repo")
tag := r.Form.Get("tag")
newTag, err := reference.WithName(repo)
if err != nil {
return err
}
if tag != "" {
if newTag, err = reference.WithTag(newTag, tag); err != nil {
return err
}
}
if err := s.daemon.TagImage(newTag, vars["name"]); err != nil {
return err
}
w.WriteHeader(http.StatusCreated)
return nil
}
func handlerGetTag(w http.ResponseWriter, r *http.Request) {
if !requiresAuth(w, r) {
return
}
vars := mux.Vars(r)
repositoryName, err := reference.WithName(vars["repository"])
if err != nil {
apiError(w, "Could not parse repository", 400)
return
}
tagName := vars["tag"]
tags, exists := testRepositories[repositoryName.String()]
if !exists {
apiError(w, "Repository not found", 404)
return
}
tag, exists := tags[tagName]
if !exists {
apiError(w, "Tag not found", 404)
return
}
writeResponse(w, tag, 200)
}
// parseRemoteRef parses the remote reference into a reference.Named
// returning the tag associated with the reference. In the case the
// given reference string includes both digest and tag, the returned
// reference will have the digest without the tag, but the tag will
// be returned.
func parseRemoteRef(remote string) (reference.Named, string, error) {
// Parse remote reference, supporting remotes with name and tag
// NOTE: Using distribution reference to handle references
// containing both a name and digest
remoteRef, err := distreference.ParseNamed(remote)
if err != nil {
return nil, "", err
}
var tag string
if t, ok := remoteRef.(distreference.Tagged); ok {
tag = t.Tag()
}
// Convert distribution reference to docker reference
// TODO: remove when docker reference changes reconciled upstream
ref, err := reference.WithName(remoteRef.Name())
if err != nil {
return nil, "", err
}
if d, ok := remoteRef.(distreference.Digested); ok {
ref, err = reference.WithDigest(ref, d.Digest())
if err != nil {
return nil, "", err
}
} else if tag != "" {
ref, err = reference.WithTag(ref, tag)
if err != nil {
return nil, "", err
}
} else {
ref = reference.WithDefaultTag(ref)
}
return ref, tag, nil
}
// Commit creates a new filesystem image from the current state of a container.
// The image can optionally be tagged into a repository.
func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (string, error) {
container, err := daemon.GetContainer(name)
if err != nil {
return "", err
}
// It is not possible to commit a running container on Windows
if runtime.GOOS == "windows" && container.IsRunning() {
return "", fmt.Errorf("Windows does not support commit of a running container")
}
if c.Pause && !container.IsPaused() {
daemon.containerPause(container)
defer daemon.containerUnpause(container)
}
newConfig, err := dockerfile.BuildFromConfig(c.Config, c.Changes)
if err != nil {
return "", err
}
if c.MergeConfigs {
if err := merge(newConfig, container.Config); err != nil {
return "", err
}
}
rwTar, err := daemon.exportContainerRw(container)
if err != nil {
return "", err
}
defer func() {
if rwTar != nil {
rwTar.Close()
}
}()
var history []image.History
rootFS := image.NewRootFS()
osVersion := ""
var osFeatures []string
if container.ImageID != "" {
img, err := daemon.imageStore.Get(container.ImageID)
if err != nil {
return "", err
}
history = img.History
rootFS = img.RootFS
osVersion = img.OSVersion
osFeatures = img.OSFeatures
}
l, err := daemon.layerStore.Register(rwTar, rootFS.ChainID())
if err != nil {
return "", err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
h := image.History{
Author: c.Author,
Created: time.Now().UTC(),
CreatedBy: strings.Join(container.Config.Cmd, " "),
Comment: c.Comment,
EmptyLayer: true,
}
if diffID := l.DiffID(); layer.DigestSHA256EmptyTar != diffID {
h.EmptyLayer = false
rootFS.Append(diffID)
}
history = append(history, h)
config, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Config: newConfig,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Container: container.ID,
ContainerConfig: *container.Config,
Author: c.Author,
Created: h.Created,
},
RootFS: rootFS,
History: history,
OSFeatures: osFeatures,
OSVersion: osVersion,
})
if err != nil {
return "", err
}
id, err := daemon.imageStore.Create(config)
if err != nil {
return "", err
}
if container.ImageID != "" {
if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil {
return "", err
}
}
if c.Repo != "" {
newTag, err := reference.WithName(c.Repo) // todo: should move this to API layer
if err != nil {
return "", err
}
if c.Tag != "" {
if newTag, err = reference.WithTag(newTag, c.Tag); err != nil {
return "", err
}
}
if err := daemon.TagImageWithReference(id, newTag); err != nil {
return "", err
}
}
attributes := map[string]string{
"comment": c.Comment,
}
daemon.LogContainerEventWithAttributes(container, "commit", attributes)
return id.String(), nil
}
func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) {
if fs, ok := pd.layer.(distribution.Describable); ok {
if d := fs.Descriptor(); len(d.URLs) > 0 {
progress.Update(progressOutput, pd.ID(), "Skipped foreign layer")
return d, nil
}
}
diffID := pd.DiffID()
pd.pushState.Lock()
if descriptor, ok := pd.pushState.remoteLayers[diffID]; ok {
// it is already known that the push is not needed and
// therefore doing a stat is unnecessary
pd.pushState.Unlock()
progress.Update(progressOutput, pd.ID(), "Layer already exists")
return descriptor, nil
}
pd.pushState.Unlock()
// Do we have any metadata associated with this layer's DiffID?
v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID)
if err == nil {
descriptor, exists, err := layerAlreadyExists(ctx, v2Metadata, pd.repoInfo, pd.repo, pd.pushState)
if err != nil {
progress.Update(progressOutput, pd.ID(), "Image push failed")
return distribution.Descriptor{}, retryOnError(err)
}
if exists {
progress.Update(progressOutput, pd.ID(), "Layer already exists")
pd.pushState.Lock()
pd.pushState.remoteLayers[diffID] = descriptor
pd.pushState.Unlock()
return descriptor, nil
}
}
logrus.Debugf("Pushing layer: %s", diffID)
// if digest was empty or not saved, or if blob does not exist on the remote repository,
// then push the blob.
bs := pd.repo.Blobs(ctx)
var layerUpload distribution.BlobWriter
mountAttemptsRemaining := 3
// Attempt to find another repository in the same registry to mount the layer
// from to avoid an unnecessary upload.
// Note: metadata is stored from oldest to newest, so we iterate through this
// slice in reverse to maximize our chances of the blob still existing in the
// remote repository.
for i := len(v2Metadata) - 1; i >= 0 && mountAttemptsRemaining > 0; i-- {
mountFrom := v2Metadata[i]
sourceRepo, err := reference.ParseNamed(mountFrom.SourceRepository)
if err != nil {
continue
}
if pd.repoInfo.Hostname() != sourceRepo.Hostname() {
// don't mount blobs from another registry
continue
}
namedRef, err := reference.WithName(mountFrom.SourceRepository)
if err != nil {
continue
}
// TODO (brianbland): We need to construct a reference where the Name is
// only the full remote name, so clean this up when distribution has a
// richer reference package
remoteRef, err := distreference.WithName(namedRef.RemoteName())
if err != nil {
continue
}
canonicalRef, err := distreference.WithDigest(remoteRef, mountFrom.Digest)
if err != nil {
continue
}
logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountFrom.Digest, sourceRepo.FullName())
layerUpload, err = bs.Create(ctx, client.WithMountFrom(canonicalRef))
switch err := err.(type) {
case distribution.ErrBlobMounted:
progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name())
err.Descriptor.MediaType = schema2.MediaTypeLayer
pd.pushState.Lock()
pd.pushState.confirmedV2 = true
pd.pushState.remoteLayers[diffID] = err.Descriptor
pd.pushState.Unlock()
// Cache mapping from this layer's DiffID to the blobsum
if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: mountFrom.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
return distribution.Descriptor{}, xfer.DoNotRetry{Err: err}
}
return err.Descriptor, nil
case nil:
// blob upload session created successfully, so begin the upload
mountAttemptsRemaining = 0
default:
// unable to mount layer from this repository, so this source mapping is no longer valid
logrus.Debugf("unassociating layer %s (%s) with %s", diffID, mountFrom.Digest, mountFrom.SourceRepository)
pd.v2MetadataService.Remove(mountFrom)
mountAttemptsRemaining--
}
}
if layerUpload == nil {
layerUpload, err = bs.Create(ctx)
if err != nil {
return distribution.Descriptor{}, retryOnError(err)
}
}
defer layerUpload.Close()
arch, err := pd.layer.TarStream()
if err != nil {
return distribution.Descriptor{}, xfer.DoNotRetry{Err: err}
}
// don't care if this fails; best effort
size, _ := pd.layer.DiffSize()
reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, arch), progressOutput, size, pd.ID(), "Pushing")
compressedReader, compressionDone := compress(reader)
defer func() {
reader.Close()
<-compressionDone
}()
digester := digest.Canonical.New()
tee := io.TeeReader(compressedReader, digester.Hash())
nn, err := layerUpload.ReadFrom(tee)
compressedReader.Close()
if err != nil {
return distribution.Descriptor{}, retryOnError(err)
}
pushDigest := digester.Digest()
if _, err := layerUpload.Commit(ctx, distribution.Descriptor{Digest: pushDigest}); err != nil {
return distribution.Descriptor{}, retryOnError(err)
}
logrus.Debugf("uploaded layer %s (%s), %d bytes", diffID, pushDigest, nn)
progress.Update(progressOutput, pd.ID(), "Pushed")
// Cache mapping from this layer's DiffID to the blobsum
if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: pushDigest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
return distribution.Descriptor{}, xfer.DoNotRetry{Err: err}
}
pd.pushState.Lock()
// If Commit succeeded, that's an indication that the remote registry
// speaks the v2 protocol.
pd.pushState.confirmedV2 = true
descriptor := distribution.Descriptor{
Digest: pushDigest,
MediaType: schema2.MediaTypeLayer,
Size: nn,
}
pd.pushState.remoteLayers[diffID] = descriptor
pd.pushState.Unlock()
return descriptor, nil
}
func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) {
if fs, ok := pd.layer.(distribution.Describable); ok {
if d := fs.Descriptor(); len(d.URLs) > 0 {
progress.Update(progressOutput, pd.ID(), "Skipped foreign layer")
return d, nil
}
}
diffID := pd.DiffID()
pd.pushState.Lock()
if descriptor, ok := pd.pushState.remoteLayers[diffID]; ok {
// it is already known that the push is not needed and
// therefore doing a stat is unnecessary
pd.pushState.Unlock()
progress.Update(progressOutput, pd.ID(), "Layer already exists")
return descriptor, nil
}
pd.pushState.Unlock()
maxMountAttempts, maxExistenceChecks, checkOtherRepositories := getMaxMountAndExistenceCheckAttempts(pd.layer)
// Do we have any metadata associated with this layer's DiffID?
v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID)
if err == nil {
// check for blob existence in the target repository if we have a mapping with it
descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, false, 1, v2Metadata)
if exists || err != nil {
return descriptor, err
}
}
// if digest was empty or not saved, or if blob does not exist on the remote repository,
// then push the blob.
bs := pd.repo.Blobs(ctx)
var layerUpload distribution.BlobWriter
// Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload
candidates := getRepositoryMountCandidates(pd.repoInfo, pd.hmacKey, maxMountAttempts, v2Metadata)
for _, mountCandidate := range candidates {
logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountCandidate.Digest, mountCandidate.SourceRepository)
createOpts := []distribution.BlobCreateOption{}
if len(mountCandidate.SourceRepository) > 0 {
namedRef, err := reference.WithName(mountCandidate.SourceRepository)
if err != nil {
logrus.Errorf("failed to parse source repository reference %v: %v", namedRef.String(), err)
pd.v2MetadataService.Remove(mountCandidate)
continue
}
// TODO (brianbland): We need to construct a reference where the Name is
// only the full remote name, so clean this up when distribution has a
// richer reference package
remoteRef, err := distreference.WithName(namedRef.RemoteName())
if err != nil {
logrus.Errorf("failed to make remote reference out of %q: %v", namedRef.RemoteName(), namedRef.RemoteName())
continue
}
canonicalRef, err := distreference.WithDigest(distreference.TrimNamed(remoteRef), mountCandidate.Digest)
if err != nil {
logrus.Errorf("failed to make canonical reference: %v", err)
continue
}
createOpts = append(createOpts, client.WithMountFrom(canonicalRef))
}
// send the layer
lu, err := bs.Create(ctx, createOpts...)
switch err := err.(type) {
case nil:
// noop
case distribution.ErrBlobMounted:
progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name())
err.Descriptor.MediaType = schema2.MediaTypeLayer
pd.pushState.Lock()
pd.pushState.confirmedV2 = true
pd.pushState.remoteLayers[diffID] = err.Descriptor
pd.pushState.Unlock()
// Cache mapping from this layer's DiffID to the blobsum
if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{
Digest: err.Descriptor.Digest,
SourceRepository: pd.repoInfo.FullName(),
}); err != nil {
return distribution.Descriptor{}, xfer.DoNotRetry{Err: err}
}
return err.Descriptor, nil
default:
logrus.Infof("failed to mount layer %s (%s) from %s: %v", diffID, mountCandidate.Digest, mountCandidate.SourceRepository, err)
}
if len(mountCandidate.SourceRepository) > 0 &&
(metadata.CheckV2MetadataHMAC(&mountCandidate, pd.hmacKey) ||
len(mountCandidate.HMAC) == 0) {
cause := "blob mount failure"
if err != nil {
cause = fmt.Sprintf("an error: %v", err.Error())
}
logrus.Debugf("removing association between layer %s and %s due to %s", mountCandidate.Digest, mountCandidate.SourceRepository, cause)
pd.v2MetadataService.Remove(mountCandidate)
}
if lu != nil {
// cancel previous upload
cancelLayerUpload(ctx, mountCandidate.Digest, layerUpload)
layerUpload = lu
}
}
if maxExistenceChecks-len(pd.checkedDigests) > 0 {
// do additional layer existence checks with other known digests if any
descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, checkOtherRepositories, maxExistenceChecks-len(pd.checkedDigests), v2Metadata)
if exists || err != nil {
return descriptor, err
}
}
logrus.Debugf("Pushing layer: %s", diffID)
if layerUpload == nil {
layerUpload, err = bs.Create(ctx)
if err != nil {
return distribution.Descriptor{}, retryOnError(err)
}
}
defer layerUpload.Close()
// upload the blob
desc, err := pd.uploadUsingSession(ctx, progressOutput, diffID, layerUpload)
if err != nil {
return desc, err
}
return desc, nil
}
// Update adds new images to the cache
func (c *ImageCache) Update(client *client.PortLayer) error {
c.m.Lock()
defer c.m.Unlock()
CacheNotUpdated = true
log.Debugf("Updating image cache...")
host, err := guest.UUID()
if host == "" {
host, err = os.Hostname()
}
if err != nil {
return fmt.Errorf("Unexpected error getting hostname: %s", err)
}
// attempt to create the image store if it doesn't exist
store := &models.ImageStore{Name: host}
_, err = client.Storage.CreateImageStore(
storage.NewCreateImageStoreParams().WithBody(store),
)
if err != nil {
if _, ok := err.(*storage.CreateImageStoreConflict); ok {
log.Debugf("Store already exists")
} else {
log.Debugf("Creating a store failed: %#v", err)
return err
}
}
params := storage.NewListImagesParams().WithStoreName(host)
layers, err := client.Storage.ListImages(params)
if err != nil {
return fmt.Errorf("Failed to retrieve image list from portlayer: %s", err)
}
for _, layer := range layers.Payload {
imageConfig := &metadata.ImageConfig{}
if err := json.Unmarshal([]byte(layer.Metadata["metaData"]), imageConfig); err != nil {
derr.NewErrorWithStatusCode(fmt.Errorf("Failed to unmarshal image config: %s", err),
http.StatusInternalServerError)
}
if imageConfig.ImageID != "" {
var imageID string
// Don't assume the image id in image has "sha256:<id> as format. We store it in
// this fomat to make it easier to lookup by digest
if strings.HasPrefix(imageConfig.ImageID, "sha") {
imageID = imageConfig.ImageID
} else {
imageID = "sha256:" + imageConfig.ImageID
}
c.cacheByID[imageID] = imageConfig
// Normalize the name stored in imageConfig using Docker's reference code
ref, err := reference.WithName(imageConfig.Name)
if err != nil {
log.Errorf("Tried to create reference from %s: %s", imageConfig.Name, err.Error())
continue
}
for id := range imageConfig.Tags {
tag := imageConfig.Tags[id]
ref, err = reference.WithTag(ref, tag)
if err != nil {
log.Errorf("Tried to create tagged reference from %s and tag %s: %s", imageConfig.Name, tag, err.Error())
continue
}
if tagged, ok := ref.(reference.NamedTagged); ok {
taggedName := fmt.Sprintf("%s:%s", tagged.Name(), tagged.Tag())
c.cacheByName[taggedName] = imageConfig
} else {
c.cacheByName[ref.Name()] = imageConfig
}
}
}
}
CacheNotUpdated = false
return nil
}
func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) error {
diffID := pd.DiffID()
pd.pushState.Lock()
if _, ok := pd.pushState.remoteLayers[diffID]; ok {
// it is already known that the push is not needed and
// therefore doing a stat is unnecessary
pd.pushState.Unlock()
progress.Update(progressOutput, pd.ID(), "Layer already exists")
return nil
}
pd.pushState.Unlock()
// Do we have any metadata associated with this layer's DiffID?
v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID)
if err == nil {
descriptor, exists, err := layerAlreadyExists(ctx, v2Metadata, pd.repoInfo, pd.repo, pd.pushState)
if err != nil {
progress.Update(progressOutput, pd.ID(), "Image push failed")
return retryOnError(err)
}
if exists {
progress.Update(progressOutput, pd.ID(), "Layer already exists")
pd.pushState.Lock()
pd.pushState.remoteLayers[diffID] = descriptor
pd.pushState.Unlock()
return nil
}
}
logrus.Debugf("Pushing layer: %s", diffID)
// if digest was empty or not saved, or if blob does not exist on the remote repository,
// then push the blob.
bs := pd.repo.Blobs(ctx)
var mountFrom metadata.V2Metadata
// Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload
for _, metadata := range v2Metadata {
sourceRepo, err := reference.ParseNamed(metadata.SourceRepository)
if err != nil {
continue
}
if pd.repoInfo.Hostname() == sourceRepo.Hostname() {
logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, metadata.Digest, sourceRepo.FullName())
mountFrom = metadata
break
}
}
var createOpts []distribution.BlobCreateOption
if mountFrom.SourceRepository != "" {
namedRef, err := reference.WithName(mountFrom.SourceRepository)
if err != nil {
return err
}
// TODO (brianbland): We need to construct a reference where the Name is
// only the full remote name, so clean this up when distribution has a
// richer reference package
remoteRef, err := distreference.WithName(namedRef.RemoteName())
if err != nil {
return err
}
canonicalRef, err := distreference.WithDigest(remoteRef, mountFrom.Digest)
if err != nil {
return err
}
createOpts = append(createOpts, client.WithMountFrom(canonicalRef))
}
// Send the layer
layerUpload, err := bs.Create(ctx, createOpts...)
switch err := err.(type) {
case distribution.ErrBlobMounted:
progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name())
err.Descriptor.MediaType = schema2.MediaTypeLayer
pd.pushState.Lock()
pd.pushState.confirmedV2 = true
pd.pushState.remoteLayers[diffID] = err.Descriptor
pd.pushState.Unlock()
// Cache mapping from this layer's DiffID to the blobsum
if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: mountFrom.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
return xfer.DoNotRetry{Err: err}
}
return nil
}
if mountFrom.SourceRepository != "" {
// unable to mount layer from this repository, so this source mapping is no longer valid
logrus.Debugf("unassociating layer %s (%s) with %s", diffID, mountFrom.Digest, mountFrom.SourceRepository)
pd.v2MetadataService.Remove(mountFrom)
}
if err != nil {
return retryOnError(err)
}
defer layerUpload.Close()
arch, err := pd.layer.TarStream()
if err != nil {
return xfer.DoNotRetry{Err: err}
}
// don't care if this fails; best effort
size, _ := pd.layer.DiffSize()
reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, arch), progressOutput, size, pd.ID(), "Pushing")
compressedReader, compressionDone := compress(reader)
defer func() {
reader.Close()
<-compressionDone
}()
digester := digest.Canonical.New()
tee := io.TeeReader(compressedReader, digester.Hash())
nn, err := layerUpload.ReadFrom(tee)
compressedReader.Close()
if err != nil {
return retryOnError(err)
}
pushDigest := digester.Digest()
if _, err := layerUpload.Commit(ctx, distribution.Descriptor{Digest: pushDigest}); err != nil {
return retryOnError(err)
}
logrus.Debugf("uploaded layer %s (%s), %d bytes", diffID, pushDigest, nn)
progress.Update(progressOutput, pd.ID(), "Pushed")
// Cache mapping from this layer's DiffID to the blobsum
if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: pushDigest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
return xfer.DoNotRetry{Err: err}
}
pd.pushState.Lock()
// If Commit succeded, that's an indication that the remote registry
// speaks the v2 protocol.
pd.pushState.confirmedV2 = true
pd.pushState.remoteLayers[diffID] = distribution.Descriptor{
Digest: pushDigest,
MediaType: schema2.MediaTypeLayer,
Size: nn,
}
pd.pushState.Unlock()
return nil
}
func migrateRefs(root, driverName string, rs refAdder, mappings map[string]image.ID) error {
migrationFile := filepath.Join(root, migrationTagsFileName)
if _, err := os.Lstat(migrationFile); !os.IsNotExist(err) {
return err
}
type repositories struct {
Repositories map[string]map[string]string
}
var repos repositories
f, err := os.Open(filepath.Join(root, repositoriesFilePrefixLegacy+driverName))
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&repos); err != nil {
return err
}
for name, repo := range repos.Repositories {
for tag, id := range repo {
if strongID, exists := mappings[id]; exists {
ref, err := reference.WithName(name)
if err != nil {
logrus.Errorf("migrate tags: invalid name %q, %q", name, err)
continue
}
if dgst, err := digest.ParseDigest(tag); err == nil {
canonical, err := reference.WithDigest(ref, dgst)
if err != nil {
logrus.Errorf("migrate tags: invalid digest %q, %q", dgst, err)
continue
}
if err := rs.AddDigest(canonical, strongID, false); err != nil {
logrus.Errorf("can't migrate digest %q for %q, err: %q", ref.String(), strongID, err)
}
} else {
tagRef, err := reference.WithTag(ref, tag)
if err != nil {
logrus.Errorf("migrate tags: invalid tag %q, %q", tag, err)
continue
}
if err := rs.AddTag(tagRef, strongID, false); err != nil {
logrus.Errorf("can't migrate tag %q for %q, err: %q", ref.String(), strongID, err)
}
}
logrus.Infof("migrated tag %s:%s to point to %s", name, tag, strongID)
}
}
}
mf, err := os.Create(migrationFile)
if err != nil {
return err
}
mf.Close()
return nil
}
func TestParseRepositoryInfo(t *testing.T) {
type staticRepositoryInfo struct {
Index *registrytypes.IndexInfo
RemoteName string
CanonicalName string
LocalName string
Official bool
}
expectedRepoInfos := map[string]staticRepositoryInfo{
"fooo/bar": {
Index: ®istrytypes.IndexInfo{
Name: IndexName,
Official: true,
},
RemoteName: "fooo/bar",
LocalName: "fooo/bar",
CanonicalName: "docker.io/fooo/bar",
Official: false,
},
"library/ubuntu": {
Index: ®istrytypes.IndexInfo{
Name: IndexName,
Official: true,
},
RemoteName: "library/ubuntu",
LocalName: "ubuntu",
CanonicalName: "docker.io/library/ubuntu",
Official: true,
},
"nonlibrary/ubuntu": {
Index: ®istrytypes.IndexInfo{
Name: IndexName,
Official: true,
},
RemoteName: "nonlibrary/ubuntu",
LocalName: "nonlibrary/ubuntu",
CanonicalName: "docker.io/nonlibrary/ubuntu",
Official: false,
},
"ubuntu": {
Index: ®istrytypes.IndexInfo{
Name: IndexName,
Official: true,
},
RemoteName: "library/ubuntu",
LocalName: "ubuntu",
CanonicalName: "docker.io/library/ubuntu",
Official: true,
},
"other/library": {
Index: ®istrytypes.IndexInfo{
Name: IndexName,
Official: true,
},
RemoteName: "other/library",
LocalName: "other/library",
CanonicalName: "docker.io/other/library",
Official: false,
},
"127.0.0.1:8000/private/moonbase": {
Index: ®istrytypes.IndexInfo{
Name: "127.0.0.1:8000",
Official: false,
},
RemoteName: "private/moonbase",
LocalName: "127.0.0.1:8000/private/moonbase",
CanonicalName: "127.0.0.1:8000/private/moonbase",
Official: false,
},
"127.0.0.1:8000/privatebase": {
Index: ®istrytypes.IndexInfo{
Name: "127.0.0.1:8000",
Official: false,
},
RemoteName: "privatebase",
LocalName: "127.0.0.1:8000/privatebase",
CanonicalName: "127.0.0.1:8000/privatebase",
Official: false,
},
"localhost:8000/private/moonbase": {
Index: ®istrytypes.IndexInfo{
Name: "localhost:8000",
Official: false,
},
RemoteName: "private/moonbase",
LocalName: "localhost:8000/private/moonbase",
CanonicalName: "localhost:8000/private/moonbase",
Official: false,
},
"localhost:8000/privatebase": {
Index: ®istrytypes.IndexInfo{
Name: "localhost:8000",
Official: false,
},
RemoteName: "privatebase",
LocalName: "localhost:8000/privatebase",
CanonicalName: "localhost:8000/privatebase",
Official: false,
},
"example.com/private/moonbase": {
Index: ®istrytypes.IndexInfo{
Name: "example.com",
Official: false,
},
RemoteName: "private/moonbase",
LocalName: "example.com/private/moonbase",
CanonicalName: "example.com/private/moonbase",
Official: false,
},
"example.com/privatebase": {
Index: ®istrytypes.IndexInfo{
Name: "example.com",
Official: false,
},
RemoteName: "privatebase",
LocalName: "example.com/privatebase",
CanonicalName: "example.com/privatebase",
Official: false,
},
"example.com:8000/private/moonbase": {
Index: ®istrytypes.IndexInfo{
Name: "example.com:8000",
Official: false,
},
RemoteName: "private/moonbase",
LocalName: "example.com:8000/private/moonbase",
CanonicalName: "example.com:8000/private/moonbase",
Official: false,
},
"example.com:8000/privatebase": {
Index: ®istrytypes.IndexInfo{
Name: "example.com:8000",
Official: false,
},
RemoteName: "privatebase",
LocalName: "example.com:8000/privatebase",
CanonicalName: "example.com:8000/privatebase",
Official: false,
},
"localhost/private/moonbase": {
Index: ®istrytypes.IndexInfo{
Name: "localhost",
Official: false,
},
RemoteName: "private/moonbase",
LocalName: "localhost/private/moonbase",
CanonicalName: "localhost/private/moonbase",
Official: false,
},
"localhost/privatebase": {
Index: ®istrytypes.IndexInfo{
Name: "localhost",
Official: false,
},
RemoteName: "privatebase",
LocalName: "localhost/privatebase",
CanonicalName: "localhost/privatebase",
Official: false,
},
IndexName + "/public/moonbase": {
Index: ®istrytypes.IndexInfo{
Name: IndexName,
Official: true,
},
RemoteName: "public/moonbase",
LocalName: "public/moonbase",
CanonicalName: "docker.io/public/moonbase",
Official: false,
},
"index." + IndexName + "/public/moonbase": {
Index: ®istrytypes.IndexInfo{
Name: IndexName,
Official: true,
},
RemoteName: "public/moonbase",
LocalName: "public/moonbase",
CanonicalName: "docker.io/public/moonbase",
Official: false,
},
"ubuntu-12.04-base": {
Index: ®istrytypes.IndexInfo{
Name: IndexName,
Official: true,
},
RemoteName: "library/ubuntu-12.04-base",
LocalName: "ubuntu-12.04-base",
CanonicalName: "docker.io/library/ubuntu-12.04-base",
Official: true,
},
IndexName + "/ubuntu-12.04-base": {
Index: ®istrytypes.IndexInfo{
Name: IndexName,
Official: true,
},
RemoteName: "library/ubuntu-12.04-base",
LocalName: "ubuntu-12.04-base",
CanonicalName: "docker.io/library/ubuntu-12.04-base",
Official: true,
},
"index." + IndexName + "/ubuntu-12.04-base": {
Index: ®istrytypes.IndexInfo{
Name: IndexName,
Official: true,
},
RemoteName: "library/ubuntu-12.04-base",
LocalName: "ubuntu-12.04-base",
CanonicalName: "docker.io/library/ubuntu-12.04-base",
Official: true,
},
}
for reposName, expectedRepoInfo := range expectedRepoInfos {
named, err := reference.WithName(reposName)
if err != nil {
t.Error(err)
}
repoInfo, err := ParseRepositoryInfo(named)
if err != nil {
t.Error(err)
} else {
checkEqual(t, repoInfo.Index.Name, expectedRepoInfo.Index.Name, reposName)
checkEqual(t, repoInfo.RemoteName(), expectedRepoInfo.RemoteName, reposName)
checkEqual(t, repoInfo.Name(), expectedRepoInfo.LocalName, reposName)
checkEqual(t, repoInfo.FullName(), expectedRepoInfo.CanonicalName, reposName)
checkEqual(t, repoInfo.Index.Official, expectedRepoInfo.Index.Official, reposName)
checkEqual(t, repoInfo.Official, expectedRepoInfo.Official, reposName)
}
}
}