func TestCancelledDownload(t *testing.T) {
ldm := NewLayerDownloadManager(&mockLayerStore{make(map[layer.ChainID]*mockLayer)}, maxDownloadConcurrency)
progressChan := make(chan progress.Progress)
progressDone := make(chan struct{})
go func() {
for range progressChan {
}
close(progressDone)
}()
ctx, cancel := context.WithCancel(context.Background())
go func() {
<-time.After(time.Millisecond)
cancel()
}()
descriptors := downloadDescriptors(nil)
_, _, err := ldm.Download(ctx, *image.NewRootFS(), descriptors, progress.ChanOutput(progressChan))
if err != context.Canceled {
t.Fatal("expected download to be cancelled")
}
close(progressChan)
<-progressDone
}
func (mf *v2ManifestFetcher) pullSchema1(ctx context.Context, ref reference.Named, unverifiedManifest *schema1.SignedManifest) (img *image.Image, manifestDigest digest.Digest, err error) {
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref)
if err != nil {
return nil, "", err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return nil, "", err
}
// Image history converted to the new format
var history []image.History
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
var throwAway struct {
ThrowAway bool `json:"throwaway,omitempty"`
}
if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
return nil, "", err
}
h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
if err != nil {
return nil, "", err
}
history = append(history, h)
}
rootFS := image.NewRootFS()
configRaw, err := makeRawConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), rootFS, history)
config, err := json.Marshal(configRaw)
if err != nil {
return nil, "", err
}
img, err = image.NewFromJSON(config)
if err != nil {
return nil, "", err
}
manifestDigest = digest.FromBytes(unverifiedManifest.Canonical)
return img, manifestDigest, nil
}
func (ic *imageCache) restoreCachedImage(parent, target *image.Image, cfg *containertypes.Config) (image.ID, error) {
var history []image.History
rootFS := image.NewRootFS()
lenHistory := 0
if parent != nil {
history = parent.History
rootFS = parent.RootFS
lenHistory = len(parent.History)
}
history = append(history, target.History[lenHistory])
if layer := getLayerForHistoryIndex(target, lenHistory); layer != "" {
rootFS.Append(layer)
}
config, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Config: cfg,
Architecture: target.Architecture,
OS: target.OS,
Author: target.Author,
Created: history[len(history)-1].Created,
},
RootFS: rootFS,
History: history,
OSFeatures: target.OSFeatures,
OSVersion: target.OSVersion,
})
if err != nil {
return "", errors.Wrap(err, "failed to marshal image config")
}
imgID, err := ic.daemon.imageStore.Create(config)
if err != nil {
return "", errors.Wrap(err, "failed to create cache image")
}
if parent != nil {
if err := ic.daemon.imageStore.SetParent(imgID, parent.ID()); err != nil {
return "", errors.Wrapf(err, "failed to set parent for %v to %v", target.ID(), parent.ID())
}
}
return imgID, nil
}
func (mf *v1ManifestFetcher) pullImageJSON(imgID, endpoint string, token []string) (*image.Image, error) {
imgJSON, _, err := mf.session.GetRemoteImageJSON(imgID, endpoint)
if err != nil {
return nil, err
}
h, err := v1.HistoryFromConfig(imgJSON, false)
if err != nil {
return nil, err
}
configRaw, err := makeRawConfigFromV1Config(imgJSON, image.NewRootFS(), []image.History{h})
if err != nil {
return nil, err
}
config, err := json.Marshal(configRaw)
if err != nil {
return nil, err
}
img, err := image.NewFromJSON(config)
if err != nil {
return nil, err
}
return img, nil
}
// Commit creates a new filesystem image from the current state of a container.
// The image can optionally be tagged into a repository.
func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (string, error) {
container, err := daemon.Get(name)
if err != nil {
return "", err
}
// It is not possible to commit a running container on Windows
if runtime.GOOS == "windows" && container.IsRunning() {
return "", fmt.Errorf("Windows does not support commit of a running container")
}
if c.Pause && !container.isPaused() {
daemon.containerPause(container)
defer daemon.containerUnpause(container)
}
if c.MergeConfigs {
if err := runconfig.Merge(c.Config, container.Config); err != nil {
return "", err
}
}
rwTar, err := daemon.exportContainerRw(container)
if err != nil {
return "", err
}
defer func() {
if rwTar != nil {
rwTar.Close()
}
}()
var history []image.History
rootFS := image.NewRootFS()
if container.ImageID != "" {
img, err := daemon.imageStore.Get(container.ImageID)
if err != nil {
return "", err
}
history = img.History
rootFS = img.RootFS
}
l, err := daemon.layerStore.Register(rwTar, rootFS.ChainID())
if err != nil {
return "", err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
h := image.History{
Author: c.Author,
Created: time.Now().UTC(),
CreatedBy: strings.Join(container.Config.Cmd.Slice(), " "),
Comment: c.Comment,
EmptyLayer: true,
}
if diffID := l.DiffID(); layer.DigestSHA256EmptyTar != diffID {
h.EmptyLayer = false
rootFS.Append(diffID)
}
history = append(history, h)
config, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Config: c.Config,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Container: container.ID,
ContainerConfig: *container.Config,
Author: c.Author,
Created: h.Created,
},
RootFS: rootFS,
History: history,
})
if err != nil {
return "", err
}
id, err := daemon.imageStore.Create(config)
if err != nil {
return "", err
}
if container.ImageID != "" {
if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil {
return "", err
}
}
if c.Repo != "" {
newTag, err := reference.WithName(c.Repo) // todo: should move this to API layer
if err != nil {
return "", err
}
if c.Tag != "" {
if newTag, err = reference.WithTag(newTag, c.Tag); err != nil {
return "", err
}
}
if err := daemon.TagImage(newTag, id.String(), true); err != nil {
return "", err
}
}
daemon.LogContainerEvent(container, "commit")
return id.String(), nil
}
func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest) (imageID image.ID, manifestDigest digest.Digest, err error) {
manifestDigest, err = schema2ManifestDigest(ref, mfst)
if err != nil {
return "", "", err
}
target := mfst.Target()
imageID = image.ID(target.Digest)
if _, err := p.config.ImageStore.Get(imageID); err == nil {
// If the image already exists locally, no need to pull
// anything.
return imageID, manifestDigest, nil
}
var descriptors []xfer.DownloadDescriptor
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for _, d := range mfst.Layers {
layerDescriptor := &v2LayerDescriptor{
digest: d.Digest,
repo: p.repo,
repoInfo: p.repoInfo,
V2MetadataService: p.V2MetadataService,
src: d,
}
descriptors = append(descriptors, layerDescriptor)
}
configChan := make(chan []byte, 1)
errChan := make(chan error, 1)
var cancel func()
ctx, cancel = context.WithCancel(ctx)
// Pull the image config
go func() {
configJSON, err := p.pullSchema2ImageConfig(ctx, target.Digest)
if err != nil {
errChan <- ImageConfigPullError{Err: err}
cancel()
return
}
configChan <- configJSON
}()
var (
configJSON []byte // raw serialized image config
unmarshalledConfig image.Image // deserialized image config
downloadRootFS image.RootFS // rootFS to use for registering layers.
)
// https://github.com/docker/docker/issues/24766 - Err on the side of caution,
// explicitly blocking images intended for linux from the Windows daemon
if runtime.GOOS == "windows" && unmarshalledConfig.OS == "linux" {
return "", "", fmt.Errorf("image operating system %q cannot be used on this platform", unmarshalledConfig.OS)
}
downloadRootFS = *image.NewRootFS()
rootFS, release, err := p.config.DownloadManager.Download(ctx, downloadRootFS, descriptors, p.config.ProgressOutput)
if err != nil {
if configJSON != nil {
// Already received the config
return "", "", err
}
select {
case err = <-errChan:
return "", "", err
default:
cancel()
select {
case <-configChan:
case <-errChan:
}
return "", "", err
}
}
defer release()
if configJSON == nil {
configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
if err != nil {
return "", "", err
}
}
// The DiffIDs returned in rootFS MUST match those in the config.
// Otherwise the image config could be referencing layers that aren't
// included in the manifest.
if len(rootFS.DiffIDs) != len(unmarshalledConfig.RootFS.DiffIDs) {
return "", "", errRootFSMismatch
}
for i := range rootFS.DiffIDs {
if rootFS.DiffIDs[i] != unmarshalledConfig.RootFS.DiffIDs[i] {
return "", "", errRootFSMismatch
}
}
imageID, err = p.config.ImageStore.Create(configJSON)
if err != nil {
return "", "", err
}
return imageID, manifestDigest, nil
}
// Commit creates a new filesystem image from the current state of a container.
// The image can optionally be tagged into a repository.
func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (string, error) {
start := time.Now()
container, err := daemon.GetContainer(name)
if err != nil {
return "", err
}
// It is not possible to commit a running container on Windows and on Solaris.
if (runtime.GOOS == "windows" || runtime.GOOS == "solaris") && container.IsRunning() {
return "", fmt.Errorf("%+v does not support commit of a running container", runtime.GOOS)
}
if c.Pause && !container.IsPaused() {
daemon.containerPause(container)
defer daemon.containerUnpause(container)
}
newConfig, err := dockerfile.BuildFromConfig(c.Config, c.Changes)
if err != nil {
return "", err
}
if c.MergeConfigs {
if err := merge(newConfig, container.Config); err != nil {
return "", err
}
}
rwTar, err := daemon.exportContainerRw(container)
if err != nil {
return "", err
}
defer func() {
if rwTar != nil {
rwTar.Close()
}
}()
var history []image.History
rootFS := image.NewRootFS()
osVersion := ""
var osFeatures []string
if container.ImageID != "" {
img, err := daemon.imageStore.Get(container.ImageID)
if err != nil {
return "", err
}
history = img.History
rootFS = img.RootFS
osVersion = img.OSVersion
osFeatures = img.OSFeatures
}
l, err := daemon.layerStore.Register(rwTar, rootFS.ChainID())
if err != nil {
return "", err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
h := image.History{
Author: c.Author,
Created: time.Now().UTC(),
CreatedBy: strings.Join(container.Config.Cmd, " "),
Comment: c.Comment,
EmptyLayer: true,
}
if diffID := l.DiffID(); layer.DigestSHA256EmptyTar != diffID {
h.EmptyLayer = false
rootFS.Append(diffID)
}
history = append(history, h)
config, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Config: newConfig,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Container: container.ID,
ContainerConfig: *container.Config,
Author: c.Author,
Created: h.Created,
},
RootFS: rootFS,
History: history,
OSFeatures: osFeatures,
OSVersion: osVersion,
})
if err != nil {
return "", err
}
id, err := daemon.imageStore.Create(config)
if err != nil {
return "", err
}
if container.ImageID != "" {
if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil {
return "", err
}
}
imageRef := ""
if c.Repo != "" {
newTag, err := reference.WithName(c.Repo) // todo: should move this to API layer
if err != nil {
return "", err
}
if c.Tag != "" {
if newTag, err = reference.WithTag(newTag, c.Tag); err != nil {
return "", err
}
}
if err := daemon.TagImageWithReference(id, newTag); err != nil {
return "", err
}
imageRef = newTag.String()
}
attributes := map[string]string{
"comment": c.Comment,
"imageID": id.String(),
"imageRef": imageRef,
}
daemon.LogContainerEventWithAttributes(container, "commit", attributes)
containerActions.WithValues("commit").UpdateSince(start)
return id.String(), nil
}
// SquashImage creates a new image with the diff of the specified image and the specified parent.
// This new image contains only the layers from it's parent + 1 extra layer which contains the diff of all the layers in between.
// The existing image(s) is not destroyed.
// If no parent is specified, a new image with the diff of all the specified image's layers merged into a new layer that has no parents.
func (daemon *Daemon) SquashImage(id, parent string) (string, error) {
img, err := daemon.imageStore.Get(image.ID(id))
if err != nil {
return "", err
}
var parentImg *image.Image
var parentChainID layer.ChainID
if len(parent) != 0 {
parentImg, err = daemon.imageStore.Get(image.ID(parent))
if err != nil {
return "", errors.Wrap(err, "error getting specified parent layer")
}
parentChainID = parentImg.RootFS.ChainID()
} else {
rootFS := image.NewRootFS()
parentImg = &image.Image{RootFS: rootFS}
}
l, err := daemon.layerStore.Get(img.RootFS.ChainID())
if err != nil {
return "", errors.Wrap(err, "error getting image layer")
}
defer daemon.layerStore.Release(l)
ts, err := l.TarStreamFrom(parentChainID)
if err != nil {
return "", errors.Wrapf(err, "error getting tar stream to parent")
}
defer ts.Close()
newL, err := daemon.layerStore.Register(ts, parentChainID)
if err != nil {
return "", errors.Wrap(err, "error registering layer")
}
defer daemon.layerStore.Release(newL)
var newImage image.Image
newImage = *img
newImage.RootFS = nil
var rootFS image.RootFS
rootFS = *parentImg.RootFS
rootFS.DiffIDs = append(rootFS.DiffIDs, newL.DiffID())
newImage.RootFS = &rootFS
for i, hi := range newImage.History {
if i >= len(parentImg.History) {
hi.EmptyLayer = true
}
newImage.History[i] = hi
}
now := time.Now()
var historyComment string
if len(parent) > 0 {
historyComment = fmt.Sprintf("merge %s to %s", id, parent)
} else {
historyComment = fmt.Sprintf("create new from %s", id)
}
newImage.History = append(newImage.History, image.History{
Created: now,
Comment: historyComment,
})
newImage.Created = now
b, err := json.Marshal(&newImage)
if err != nil {
return "", errors.Wrap(err, "error marshalling image config")
}
newImgID, err := daemon.imageStore.Create(b)
if err != nil {
return "", errors.Wrap(err, "error creating new image after squash")
}
return string(newImgID), nil
}
func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest) (imageID image.ID, manifestDigest digest.Digest, err error) {
manifestDigest, err = schema2ManifestDigest(ref, mfst)
if err != nil {
return "", "", err
}
target := mfst.Target()
imageID = image.ID(target.Digest)
if _, err := p.config.ImageStore.Get(imageID); err == nil {
// If the image already exists locally, no need to pull
// anything.
return imageID, manifestDigest, nil
}
configChan := make(chan []byte, 1)
errChan := make(chan error, 1)
var cancel func()
ctx, cancel = context.WithCancel(ctx)
// Pull the image config
go func() {
configJSON, err := p.pullSchema2ImageConfig(ctx, target.Digest)
if err != nil {
errChan <- err
cancel()
return
}
configChan <- configJSON
}()
var descriptors []xfer.DownloadDescriptor
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for _, d := range mfst.References() {
layerDescriptor := &v2LayerDescriptor{
digest: d.Digest,
repo: p.repo,
blobSumService: p.blobSumService,
}
descriptors = append(descriptors, layerDescriptor)
}
var (
configJSON []byte // raw serialized image config
unmarshalledConfig image.Image // deserialized image config
downloadRootFS image.RootFS // rootFS to use for registering layers.
)
if runtime.GOOS == "windows" {
configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
if err != nil {
return "", "", err
}
if unmarshalledConfig.RootFS == nil {
return "", "", errors.New("image config has no rootfs section")
}
downloadRootFS = *unmarshalledConfig.RootFS
downloadRootFS.DiffIDs = []layer.DiffID{}
} else {
downloadRootFS = *image.NewRootFS()
}
rootFS, release, err := p.config.DownloadManager.Download(ctx, downloadRootFS, descriptors, p.config.ProgressOutput)
if err != nil {
if configJSON != nil {
// Already received the config
return "", "", err
}
select {
case err = <-errChan:
return "", "", err
default:
cancel()
select {
case <-configChan:
case <-errChan:
}
return "", "", err
}
}
defer release()
if configJSON == nil {
configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
if err != nil {
return "", "", err
}
}
// The DiffIDs returned in rootFS MUST match those in the config.
// Otherwise the image config could be referencing layers that aren't
// included in the manifest.
if len(rootFS.DiffIDs) != len(unmarshalledConfig.RootFS.DiffIDs) {
return "", "", errRootFSMismatch
}
for i := range rootFS.DiffIDs {
if rootFS.DiffIDs[i] != unmarshalledConfig.RootFS.DiffIDs[i] {
return "", "", errRootFSMismatch
}
}
imageID, err = p.config.ImageStore.Create(configJSON)
if err != nil {
return "", "", err
}
return imageID, manifestDigest, nil
}
func (p *v1Puller) pullImage(ctx context.Context, v1ID, endpoint string, localNameRef reference.Named, layersDownloaded *bool) (err error) {
var history []string
history, err = p.session.GetRemoteHistory(v1ID, endpoint)
if err != nil {
return err
}
if len(history) < 1 {
return fmt.Errorf("empty history for image %s", v1ID)
}
progress.Update(p.config.ProgressOutput, stringid.TruncateID(v1ID), "Pulling dependent layers")
var (
descriptors []xfer.DownloadDescriptor
newHistory []image.History
imgJSON []byte
imgSize int64
)
// Iterate over layers, in order from bottom-most to top-most. Download
// config for all layers and create descriptors.
for i := len(history) - 1; i >= 0; i-- {
v1LayerID := history[i]
imgJSON, imgSize, err = p.downloadLayerConfig(v1LayerID, endpoint)
if err != nil {
return err
}
// Create a new-style config from the legacy configs
h, err := v1.HistoryFromConfig(imgJSON, false)
if err != nil {
return err
}
newHistory = append(newHistory, h)
layerDescriptor := &v1LayerDescriptor{
v1LayerID: v1LayerID,
indexName: p.repoInfo.Index.Name,
endpoint: endpoint,
v1IDService: p.v1IDService,
layersDownloaded: layersDownloaded,
layerSize: imgSize,
session: p.session,
}
descriptors = append(descriptors, layerDescriptor)
}
rootFS := image.NewRootFS()
resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, descriptors, p.config.ProgressOutput)
if err != nil {
return err
}
defer release()
config, err := v1.MakeConfigFromV1Config(imgJSON, &resultRootFS, newHistory)
if err != nil {
return err
}
imageID, err := p.config.ImageStore.Create(config)
if err != nil {
return err
}
if err := p.config.ReferenceStore.AddTag(localNameRef, imageID, true); err != nil {
return err
}
return nil
}
// CreateImageConfig constructs the image metadata from layers that compose the image
func CreateImageConfig(images []*ImageWithMeta, manifest *Manifest) error {
if len(images) == 0 {
return nil
}
imageLayer := images[0] // the layer that represents the actual image
image := docker.V1Image{}
rootFS := docker.NewRootFS()
history := make([]docker.History, 0, len(images))
diffIDs := make(map[string]string)
var size int64
// step through layers to get command history and diffID from oldest to newest
for i := len(images) - 1; i >= 0; i-- {
layer := images[i]
if err := json.Unmarshal([]byte(layer.meta), &image); err != nil {
return fmt.Errorf("Failed to unmarshall layer history: %s", err)
}
h := docker.History{
Created: image.Created,
Author: image.Author,
CreatedBy: strings.Join(image.ContainerConfig.Cmd, " "),
Comment: image.Comment,
}
history = append(history, h)
rootFS.DiffIDs = append(rootFS.DiffIDs, dockerLayer.DiffID(layer.diffID))
diffIDs[layer.diffID] = layer.ID
size += layer.size
}
// result is constructed without unused fields
result := docker.Image{
V1Image: docker.V1Image{
Comment: image.Comment,
Created: image.Created,
Container: image.Container,
ContainerConfig: image.ContainerConfig,
DockerVersion: image.DockerVersion,
Author: image.Author,
Config: image.Config,
Architecture: image.Architecture,
OS: image.OS,
},
RootFS: rootFS,
History: history,
}
bytes, err := result.MarshalJSON()
if err != nil {
return fmt.Errorf("Failed to marshall image metadata: %s", err)
}
// calculate image ID
sum := fmt.Sprintf("%x", sha256.Sum256(bytes))
log.Infof("Image ID: sha256:%s", sum)
// prepare metadata
result.V1Image.Parent = image.Parent
result.Size = size
result.V1Image.ID = imageLayer.ID
metaData := metadata.ImageConfig{
V1Image: result.V1Image,
ImageID: sum,
// TODO: this will change when issue 1186 is
// implemented -- only populate the digests when pulled by digest
Digests: []string{manifest.Digest},
Tags: []string{options.tag},
Name: manifest.Name,
DiffIDs: diffIDs,
History: history,
}
blob, err := json.Marshal(metaData)
if err != nil {
return fmt.Errorf("Failed to marshal image metadata: %s", err)
}
// store metadata
imageLayer.meta = string(blob)
return nil
}
func (p *v1Puller) pullImage(out io.Writer, v1ID, endpoint string, localNameRef reference.Named) (layersDownloaded bool, err error) {
var history []string
history, err = p.session.GetRemoteHistory(v1ID, endpoint)
if err != nil {
return false, err
}
if len(history) < 1 {
return false, fmt.Errorf("empty history for image %s", v1ID)
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(v1ID), "Pulling dependent layers", nil))
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
var (
referencedLayers []layer.Layer
parentID layer.ChainID
newHistory []image.History
img *image.V1Image
imgJSON []byte
imgSize int64
)
defer func() {
for _, l := range referencedLayers {
layer.ReleaseAndLog(p.config.LayerStore, l)
}
}()
layersDownloaded = false
// Iterate over layers from top-most to bottom-most, checking if any
// already exist on disk.
var i int
for i = 0; i != len(history); i++ {
v1LayerID := history[i]
// Do we have a mapping for this particular v1 ID on this
// registry?
if layerID, err := p.v1IDService.Get(v1LayerID, p.repoInfo.Index.Name); err == nil {
// Does the layer actually exist
if l, err := p.config.LayerStore.Get(layerID); err == nil {
for j := i; j >= 0; j-- {
logrus.Debugf("Layer already exists: %s", history[j])
out.Write(p.sf.FormatProgress(stringid.TruncateID(history[j]), "Already exists", nil))
}
referencedLayers = append(referencedLayers, l)
parentID = layerID
break
}
}
}
needsDownload := i
// Iterate over layers, in order from bottom-most to top-most. Download
// config for all layers, and download actual layer data if needed.
for i = len(history) - 1; i >= 0; i-- {
v1LayerID := history[i]
imgJSON, imgSize, err = p.downloadLayerConfig(out, v1LayerID, endpoint)
if err != nil {
return layersDownloaded, err
}
img = &image.V1Image{}
if err := json.Unmarshal(imgJSON, img); err != nil {
return layersDownloaded, err
}
if i < needsDownload {
l, err := p.downloadLayer(out, v1LayerID, endpoint, parentID, imgSize, &layersDownloaded)
// Note: This needs to be done even in the error case to avoid
// stale references to the layer.
if l != nil {
referencedLayers = append(referencedLayers, l)
}
if err != nil {
return layersDownloaded, err
}
parentID = l.ChainID()
}
// Create a new-style config from the legacy configs
h, err := v1.HistoryFromConfig(imgJSON, false)
if err != nil {
return layersDownloaded, err
}
newHistory = append(newHistory, h)
}
rootFS := image.NewRootFS()
l := referencedLayers[len(referencedLayers)-1]
for l != nil {
rootFS.DiffIDs = append([]layer.DiffID{l.DiffID()}, rootFS.DiffIDs...)
l = l.Parent()
}
config, err := v1.MakeConfigFromV1Config(imgJSON, rootFS, newHistory)
if err != nil {
return layersDownloaded, err
}
imageID, err := p.config.ImageStore.Create(config)
if err != nil {
return layersDownloaded, err
}
if err := p.config.TagStore.Add(localNameRef, imageID, true); err != nil {
return layersDownloaded, err
}
return layersDownloaded, nil
}
func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) {
tagOrDigest := ""
if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
tagOrDigest = tagged.Tag()
} else if digested, isCanonical := ref.(reference.Canonical); isCanonical {
tagOrDigest = digested.Digest().String()
} else {
return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String())
}
logrus.Debugf("Pulling ref from V2 registry: %q", tagOrDigest)
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return false, err
}
unverifiedManifest, err := manSvc.GetByTag(tagOrDigest)
if err != nil {
return false, err
}
if unverifiedManifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
}
// If GetByTag succeeded, we can be confident that the registry on
// the other side speaks the v2 protocol.
p.confirmedV2 = true
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifyManifest(unverifiedManifest, ref)
if err != nil {
return false, err
}
rootFS := image.NewRootFS()
if err := detectBaseLayer(p.config.ImageStore, verifiedManifest, rootFS); err != nil {
return false, err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return false, err
}
progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+p.repo.Name())
var descriptors []xfer.DownloadDescriptor
// Image history converted to the new format
var history []image.History
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
blobSum := verifiedManifest.FSLayers[i].BlobSum
var throwAway struct {
ThrowAway bool `json:"throwaway,omitempty"`
}
if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
return false, err
}
h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
if err != nil {
return false, err
}
history = append(history, h)
if throwAway.ThrowAway {
continue
}
layerDescriptor := &v2LayerDescriptor{
digest: blobSum,
repo: p.repo,
blobSumService: p.blobSumService,
}
descriptors = append(descriptors, layerDescriptor)
}
resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, descriptors, p.config.ProgressOutput)
if err != nil {
return false, err
}
defer release()
config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), &resultRootFS, history)
if err != nil {
return false, err
}
imageID, err := p.config.ImageStore.Create(config)
if err != nil {
return false, err
}
manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo)
if err != nil {
return false, err
}
if manifestDigest != "" {
progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
}
oldTagImageID, err := p.config.ReferenceStore.Get(ref)
if err == nil && oldTagImageID == imageID {
return false, nil
}
if canonical, ok := ref.(reference.Canonical); ok {
if err = p.config.ReferenceStore.AddDigest(canonical, imageID, true); err != nil {
return false, err
}
} else if err = p.config.ReferenceStore.AddTag(ref, imageID, true); err != nil {
return false, err
}
return true, nil
}
func (p *v2Puller) pullV2Tag(out io.Writer, ref reference.Named) (tagUpdated bool, err error) {
tagOrDigest := ""
if tagged, isTagged := ref.(reference.Tagged); isTagged {
tagOrDigest = tagged.Tag()
} else if digested, isDigested := ref.(reference.Digested); isDigested {
tagOrDigest = digested.Digest().String()
} else {
return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String())
}
logrus.Debugf("Pulling ref from V2 registry: %q", tagOrDigest)
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return false, err
}
unverifiedManifest, err := manSvc.GetByTag(tagOrDigest)
if err != nil {
return false, err
}
if unverifiedManifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
}
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifyManifest(unverifiedManifest, ref)
if err != nil {
return false, err
}
rootFS := image.NewRootFS()
if err := detectBaseLayer(p.config.ImageStore, verifiedManifest, rootFS); err != nil {
return false, err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return false, err
}
out.Write(p.sf.FormatStatus(tagOrDigest, "Pulling from %s", p.repo.Name()))
var downloads []*downloadInfo
defer func() {
for _, d := range downloads {
p.config.Pool.removeWithError(d.poolKey, err)
if d.tmpFile != nil {
d.tmpFile.Close()
if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
}
}
}
}()
// Image history converted to the new format
var history []image.History
poolKey := "v2layer:"
notFoundLocally := false
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
blobSum := verifiedManifest.FSLayers[i].BlobSum
poolKey += blobSum.String()
var throwAway struct {
ThrowAway bool `json:"throwaway,omitempty"`
}
if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
return false, err
}
h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
if err != nil {
return false, err
}
history = append(history, h)
if throwAway.ThrowAway {
continue
}
// Do we have a layer on disk corresponding to the set of
// blobsums up to this point?
if !notFoundLocally {
notFoundLocally = true
diffID, err := p.blobSumService.GetDiffID(blobSum)
if err == nil {
rootFS.Append(diffID)
if l, err := p.config.LayerStore.Get(rootFS.ChainID()); err == nil {
notFoundLocally = false
logrus.Debugf("Layer already exists: %s", blobSum.String())
out.Write(p.sf.FormatProgress(stringid.TruncateID(blobSum.String()), "Already exists", nil))
defer layer.ReleaseAndLog(p.config.LayerStore, l)
continue
} else {
rootFS.DiffIDs = rootFS.DiffIDs[:len(rootFS.DiffIDs)-1]
}
}
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(blobSum.String()), "Pulling fs layer", nil))
tmpFile, err := ioutil.TempFile("", "GetImageBlob")
if err != nil {
return false, err
}
d := &downloadInfo{
poolKey: poolKey,
digest: blobSum,
tmpFile: tmpFile,
// TODO: seems like this chan buffer solved hanging problem in go1.5,
// this can indicate some deeper problem that somehow we never take
// error from channel in loop below
err: make(chan error, 1),
}
downloads = append(downloads, d)
broadcaster, found := p.config.Pool.add(d.poolKey)
broadcaster.Add(out)
d.broadcaster = broadcaster
if found {
d.err <- nil
} else {
go p.download(d)
}
}
for _, d := range downloads {
if err := <-d.err; err != nil {
return false, err
}
if d.layer == nil {
// Wait for a different pull to download and extract
// this layer.
err = d.broadcaster.Wait()
if err != nil {
return false, err
}
diffID, err := p.blobSumService.GetDiffID(d.digest)
if err != nil {
return false, err
}
rootFS.Append(diffID)
l, err := p.config.LayerStore.Get(rootFS.ChainID())
if err != nil {
return false, err
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
continue
}
d.tmpFile.Seek(0, 0)
reader := progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: d.broadcaster,
Formatter: p.sf,
Size: d.size,
NewLines: false,
ID: stringid.TruncateID(d.digest.String()),
Action: "Extracting",
})
inflatedLayerData, err := archive.DecompressStream(reader)
if err != nil {
return false, fmt.Errorf("could not get decompression stream: %v", err)
}
l, err := p.config.LayerStore.Register(inflatedLayerData, rootFS.ChainID())
if err != nil {
return false, fmt.Errorf("failed to register layer: %v", err)
}
logrus.Debugf("layer %s registered successfully", l.DiffID())
rootFS.Append(l.DiffID())
// Cache mapping from this layer's DiffID to the blobsum
if err := p.blobSumService.Add(l.DiffID(), d.digest); err != nil {
return false, err
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.digest.String()), "Pull complete", nil))
d.broadcaster.Close()
tagUpdated = true
}
config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), rootFS, history)
if err != nil {
return false, err
}
imageID, err := p.config.ImageStore.Create(config)
if err != nil {
return false, err
}
manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName.Name())
if err != nil {
return false, err
}
// Check for new tag if no layers downloaded
var oldTagImageID image.ID
if !tagUpdated {
oldTagImageID, err = p.config.TagStore.Get(ref)
if err != nil || oldTagImageID != imageID {
tagUpdated = true
}
}
if tagUpdated {
if canonical, ok := ref.(reference.Canonical); ok {
if err = p.config.TagStore.AddDigest(canonical, imageID, true); err != nil {
return false, err
}
} else if err = p.config.TagStore.AddTag(ref, imageID, true); err != nil {
return false, err
}
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
}
return tagUpdated, nil
}
func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest) (id digest.Digest, manifestDigest digest.Digest, err error) {
manifestDigest, err = schema2ManifestDigest(ref, mfst)
if err != nil {
return "", "", err
}
target := mfst.Target()
if _, err := p.config.ImageStore.Get(target.Digest); err == nil {
// If the image already exists locally, no need to pull
// anything.
return target.Digest, manifestDigest, nil
}
var descriptors []xfer.DownloadDescriptor
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for _, d := range mfst.Layers {
layerDescriptor := &v2LayerDescriptor{
digest: d.Digest,
repo: p.repo,
repoInfo: p.repoInfo,
V2MetadataService: p.V2MetadataService,
src: d,
}
descriptors = append(descriptors, layerDescriptor)
}
configChan := make(chan []byte, 1)
errChan := make(chan error, 1)
var cancel func()
ctx, cancel = context.WithCancel(ctx)
// Pull the image config
go func() {
configJSON, err := p.pullSchema2Config(ctx, target.Digest)
if err != nil {
errChan <- ImageConfigPullError{Err: err}
cancel()
return
}
configChan <- configJSON
}()
var (
configJSON []byte // raw serialized image config
downloadedRootFS *image.RootFS // rootFS from registered layers
configRootFS *image.RootFS // rootFS from configuration
)
// https://github.com/docker/docker/issues/24766 - Err on the side of caution,
// explicitly blocking images intended for linux from the Windows daemon. On
// Windows, we do this before the attempt to download, effectively serialising
// the download slightly slowing it down. We have to do it this way, as
// chances are the download of layers itself would fail due to file names
// which aren't suitable for NTFS. At some point in the future, if a similar
// check to block Windows images being pulled on Linux is implemented, it
// may be necessary to perform the same type of serialisation.
if runtime.GOOS == "windows" {
configJSON, configRootFS, err = receiveConfig(p.config.ImageStore, configChan, errChan)
if err != nil {
return "", "", err
}
if configRootFS == nil {
return "", "", errRootFSInvalid
}
}
if p.config.DownloadManager != nil {
downloadRootFS := *image.NewRootFS()
rootFS, release, err := p.config.DownloadManager.Download(ctx, downloadRootFS, descriptors, p.config.ProgressOutput)
if err != nil {
if configJSON != nil {
// Already received the config
return "", "", err
}
select {
case err = <-errChan:
return "", "", err
default:
cancel()
select {
case <-configChan:
case <-errChan:
}
return "", "", err
}
}
if release != nil {
defer release()
}
downloadedRootFS = &rootFS
}
if configJSON == nil {
configJSON, configRootFS, err = receiveConfig(p.config.ImageStore, configChan, errChan)
if err != nil {
return "", "", err
}
if configRootFS == nil {
return "", "", errRootFSInvalid
}
}
if downloadedRootFS != nil {
// The DiffIDs returned in rootFS MUST match those in the config.
// Otherwise the image config could be referencing layers that aren't
// included in the manifest.
if len(downloadedRootFS.DiffIDs) != len(configRootFS.DiffIDs) {
return "", "", errRootFSMismatch
}
for i := range downloadedRootFS.DiffIDs {
if downloadedRootFS.DiffIDs[i] != configRootFS.DiffIDs[i] {
return "", "", errRootFSMismatch
}
}
}
imageID, err := p.config.ImageStore.Put(configJSON)
if err != nil {
return "", "", err
}
return imageID, manifestDigest, nil
}
func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[string]image.ID, progressOutput progress.Output) error {
if _, loaded := loadedMap[oldID]; loaded {
return nil
}
configPath, err := safePath(sourceDir, filepath.Join(oldID, legacyConfigFileName))
if err != nil {
return err
}
imageJSON, err := ioutil.ReadFile(configPath)
if err != nil {
logrus.Debugf("Error reading json: %v", err)
return err
}
var img struct{ Parent string }
if err := json.Unmarshal(imageJSON, &img); err != nil {
return err
}
var parentID image.ID
if img.Parent != "" {
for {
var loaded bool
if parentID, loaded = loadedMap[img.Parent]; !loaded {
if err := l.legacyLoadImage(img.Parent, sourceDir, loadedMap, progressOutput); err != nil {
return err
}
} else {
break
}
}
}
// todo: try to connect with migrate code
rootFS := image.NewRootFS()
var history []image.History
if parentID != "" {
parentImg, err := l.is.Get(parentID)
if err != nil {
return err
}
rootFS = parentImg.RootFS
history = parentImg.History
}
layerPath, err := safePath(sourceDir, filepath.Join(oldID, legacyLayerFileName))
if err != nil {
return err
}
newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, progressOutput)
if err != nil {
return err
}
rootFS.Append(newLayer.DiffID())
h, err := v1.HistoryFromConfig(imageJSON, false)
if err != nil {
return err
}
history = append(history, h)
config, err := v1.MakeConfigFromV1Config(imageJSON, rootFS, history)
if err != nil {
return err
}
imgID, err := l.is.Create(config)
if err != nil {
return err
}
metadata, err := l.ls.Release(newLayer)
layer.LogReleaseMetadata(metadata)
if err != nil {
return err
}
if parentID != "" {
if err := l.is.SetParent(imgID, parentID); err != nil {
return err
}
}
loadedMap[oldID] = imgID
return nil
}
func restoreCustomImage(is image.Store, ls layer.Store, rs reference.Store) error {
type graphDriverStore interface {
GraphDriver() graphdriver.Driver
}
gds, ok := ls.(graphDriverStore)
if !ok {
return nil
}
driver := gds.GraphDriver()
wd, ok := driver.(*windows.Driver)
if !ok {
return nil
}
imageInfos, err := wd.GetCustomImageInfos()
if err != nil {
return err
}
// Convert imageData to valid image configuration
for i := range imageInfos {
name := strings.ToLower(imageInfos[i].Name)
type registrar interface {
RegisterDiffID(graphID string, size int64) (layer.Layer, error)
}
r, ok := ls.(registrar)
if !ok {
return errors.New("Layerstore doesn't support RegisterDiffID")
}
if _, err := r.RegisterDiffID(imageInfos[i].ID, imageInfos[i].Size); err != nil {
return err
}
// layer is intentionally not released
rootFS := image.NewRootFS()
rootFS.BaseLayer = filepath.Base(imageInfos[i].Path)
// Create history for base layer
config, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Created: imageInfos[i].CreatedTime,
},
RootFS: rootFS,
History: []image.History{},
})
named, err := reference.ParseNamed(name)
if err != nil {
return err
}
ref, err := reference.WithTag(named, imageInfos[i].Version)
if err != nil {
return err
}
id, err := is.Create(config)
if err != nil {
return err
}
if err := rs.AddTag(ref, id, true); err != nil {
return err
}
logrus.Debugf("Registered base layer %s as %s", ref, id)
}
return nil
}
func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) (err error) {
defer func() {
if err != nil {
logrus.Errorf("migration failed for %v, err: %v", id, err)
}
}()
jsonFile := filepath.Join(root, graphDirName, id, "json")
imageJSON, err := ioutil.ReadFile(jsonFile)
if err != nil {
return err
}
var parent struct {
Parent string
ParentID digest.Digest `json:"parent_id"`
}
if err := json.Unmarshal(imageJSON, &parent); err != nil {
return err
}
if parent.Parent == "" && parent.ParentID != "" { // v1.9
parent.Parent = parent.ParentID.Hex()
}
// compatibilityID for parent
parentCompatibilityID, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, "parent"))
if err == nil && len(parentCompatibilityID) > 0 {
parent.Parent = string(parentCompatibilityID)
}
var parentID image.ID
if parent.Parent != "" {
var exists bool
if parentID, exists = mappings[parent.Parent]; !exists {
if err := migrateImage(parent.Parent, root, ls, is, ms, mappings); err != nil {
// todo: fail or allow broken chains?
return err
}
parentID = mappings[parent.Parent]
}
}
rootFS := image.NewRootFS()
var history []image.History
if parentID != "" {
parentImg, err := is.Get(parentID)
if err != nil {
return err
}
rootFS = parentImg.RootFS
history = parentImg.History
}
layer, err := ls.RegisterByGraphID(id, rootFS.ChainID(), filepath.Join(filepath.Join(root, graphDirName, id, tarDataFileName)))
if err != nil {
return err
}
logrus.Infof("migrated layer %s to %s", id, layer.DiffID())
h, err := imagev1.HistoryFromConfig(imageJSON, false)
if err != nil {
return err
}
history = append(history, h)
rootFS.Append(layer.DiffID())
config, err := imagev1.MakeConfigFromV1Config(imageJSON, rootFS, history)
if err != nil {
return err
}
strongID, err := is.Create(config)
if err != nil {
return err
}
logrus.Infof("migrated image %s to %s", id, strongID)
if parentID != "" {
if err := is.SetParent(strongID, parentID); err != nil {
return err
}
}
checksum, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, "checksum"))
if err == nil { // best effort
dgst, err := digest.ParseDigest(string(checksum))
if err == nil {
blobSumService := metadata.NewBlobSumService(ms)
blobSumService.Add(layer.DiffID(), dgst)
}
}
_, err = ls.Release(layer)
if err != nil {
return err
}
mappings[id] = strongID
return
}
func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverifiedManifest *schema1.SignedManifest) (imageID image.ID, manifestDigest digest.Digest, err error) {
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref)
if err != nil {
return "", "", err
}
rootFS := image.NewRootFS()
if err := detectBaseLayer(p.config.ImageStore, verifiedManifest, rootFS); err != nil {
return "", "", err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return "", "", err
}
var descriptors []xfer.DownloadDescriptor
// Image history converted to the new format
var history []image.History
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
blobSum := verifiedManifest.FSLayers[i].BlobSum
var throwAway struct {
ThrowAway bool `json:"throwaway,omitempty"`
}
if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
return "", "", err
}
h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
if err != nil {
return "", "", err
}
history = append(history, h)
if throwAway.ThrowAway {
continue
}
layerDescriptor := &v2LayerDescriptor{
digest: blobSum,
repo: p.repo,
blobSumService: p.blobSumService,
}
descriptors = append(descriptors, layerDescriptor)
}
resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, descriptors, p.config.ProgressOutput)
if err != nil {
return "", "", err
}
defer release()
config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), &resultRootFS, history)
if err != nil {
return "", "", err
}
imageID, err = p.config.ImageStore.Create(config)
if err != nil {
return "", "", err
}
manifestDigest = digest.FromBytes(unverifiedManifest.Canonical)
return imageID, manifestDigest, nil
}
func TestSuccessfulDownload(t *testing.T) {
// TODO Windows: Fix this unit text
if runtime.GOOS == "windows" {
t.Skip("Needs fixing on Windows")
}
layerStore := &mockLayerStore{make(map[layer.ChainID]*mockLayer)}
ldm := NewLayerDownloadManager(layerStore, maxDownloadConcurrency)
progressChan := make(chan progress.Progress)
progressDone := make(chan struct{})
receivedProgress := make(map[string]progress.Progress)
go func() {
for p := range progressChan {
receivedProgress[p.ID] = p
}
close(progressDone)
}()
var currentDownloads int32
descriptors := downloadDescriptors(¤tDownloads)
firstDescriptor := descriptors[0].(*mockDownloadDescriptor)
// Pre-register the first layer to simulate an already-existing layer
l, err := layerStore.Register(firstDescriptor.mockTarStream(), "")
if err != nil {
t.Fatal(err)
}
firstDescriptor.diffID = l.DiffID()
rootFS, releaseFunc, err := ldm.Download(context.Background(), *image.NewRootFS(), descriptors, progress.ChanOutput(progressChan))
if err != nil {
t.Fatalf("download error: %v", err)
}
releaseFunc()
close(progressChan)
<-progressDone
if len(rootFS.DiffIDs) != len(descriptors) {
t.Fatal("got wrong number of diffIDs in rootfs")
}
for i, d := range descriptors {
descriptor := d.(*mockDownloadDescriptor)
if descriptor.diffID != "" {
if receivedProgress[d.ID()].Action != "Already exists" {
t.Fatalf("did not get 'Already exists' message for %v", d.ID())
}
} else if receivedProgress[d.ID()].Action != "Pull complete" {
t.Fatalf("did not get 'Pull complete' message for %v", d.ID())
}
if rootFS.DiffIDs[i] != descriptor.expectedDiffID {
t.Fatalf("rootFS item %d has the wrong diffID (expected: %v got: %v)", i, descriptor.expectedDiffID, rootFS.DiffIDs[i])
}
if descriptor.diffID == "" && descriptor.registeredDiffID != rootFS.DiffIDs[i] {
t.Fatal("diffID mismatch between rootFS and Registered callback")
}
}
}
func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) (err error) {
defer func() {
if err != nil {
logrus.Errorf("migration failed for %v, err: %v", id, err)
}
}()
parent, err := getParent(filepath.Join(root, graphDirName, id))
if err != nil {
return err
}
var parentID image.ID
if parent != "" {
var exists bool
if parentID, exists = mappings[parent]; !exists {
if err := migrateImage(parent, root, ls, is, ms, mappings); err != nil {
// todo: fail or allow broken chains?
return err
}
parentID = mappings[parent]
}
}
rootFS := image.NewRootFS()
var history []image.History
if parentID != "" {
parentImg, err := is.Get(parentID)
if err != nil {
return err
}
rootFS = parentImg.RootFS
history = parentImg.History
}
diffIDData, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationDiffIDFileName))
if err != nil {
return err
}
diffID, err := digest.ParseDigest(string(diffIDData))
if err != nil {
return err
}
sizeStr, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationSizeFileName))
if err != nil {
return err
}
size, err := strconv.ParseInt(string(sizeStr), 10, 64)
if err != nil {
return err
}
layer, err := ls.RegisterByGraphID(id, rootFS.ChainID(), layer.DiffID(diffID), filepath.Join(root, graphDirName, id, migrationTarDataFileName), size)
if err != nil {
return err
}
logrus.Infof("migrated layer %s to %s", id, layer.DiffID())
jsonFile := filepath.Join(root, graphDirName, id, "json")
imageJSON, err := ioutil.ReadFile(jsonFile)
if err != nil {
return err
}
h, err := imagev1.HistoryFromConfig(imageJSON, false)
if err != nil {
return err
}
history = append(history, h)
rootFS.Append(layer.DiffID())
config, err := imagev1.MakeConfigFromV1Config(imageJSON, rootFS, history)
if err != nil {
return err
}
strongID, err := is.Create(config)
if err != nil {
return err
}
logrus.Infof("migrated image %s to %s", id, strongID)
if parentID != "" {
if err := is.SetParent(strongID, parentID); err != nil {
return err
}
}
checksum, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, "checksum"))
if err == nil { // best effort
dgst, err := digest.ParseDigest(string(checksum))
if err == nil {
V2MetadataService := metadata.NewV2MetadataService(ms)
V2MetadataService.Add(layer.DiffID(), metadata.V2Metadata{Digest: dgst})
}
}
_, err = ls.Release(layer)
if err != nil {
return err
}
mappings[id] = strongID
return
}
// CreateImageConfig constructs the image metadata from layers that compose the image
func (ic *ImageC) CreateImageConfig(images []*ImageWithMeta) (metadata.ImageConfig, error) {
imageLayer := images[0] // the layer that represents the actual image
// if we already have an imageID associated with this layerID, we don't need
// to calculate imageID and can just grab the image config from the cache
id := cache.RepositoryCache().GetImageID(imageLayer.ID)
if image, err := cache.ImageCache().Get(id); err == nil {
return *image, nil
}
manifest := ic.ImageManifest
image := docker.V1Image{}
rootFS := docker.NewRootFS()
history := make([]docker.History, 0, len(images))
diffIDs := make(map[string]string)
var size int64
// step through layers to get command history and diffID from oldest to newest
for i := len(images) - 1; i >= 0; i-- {
layer := images[i]
if err := json.Unmarshal([]byte(layer.Meta), &image); err != nil {
return metadata.ImageConfig{}, fmt.Errorf("Failed to unmarshall layer history: %s", err)
}
h := docker.History{
Created: image.Created,
Author: image.Author,
CreatedBy: strings.Join(image.ContainerConfig.Cmd, " "),
Comment: image.Comment,
}
history = append(history, h)
rootFS.DiffIDs = append(rootFS.DiffIDs, dockerLayer.DiffID(layer.DiffID))
diffIDs[layer.DiffID] = layer.ID
size += layer.Size
}
// result is constructed without unused fields
result := docker.Image{
V1Image: docker.V1Image{
Comment: image.Comment,
Created: image.Created,
Container: image.Container,
ContainerConfig: image.ContainerConfig,
DockerVersion: image.DockerVersion,
Author: image.Author,
Config: image.Config,
Architecture: image.Architecture,
OS: image.OS,
},
RootFS: rootFS,
History: history,
}
bytes, err := result.MarshalJSON()
if err != nil {
return metadata.ImageConfig{}, fmt.Errorf("Failed to marshall image metadata: %s", err)
}
// calculate image ID
sum := fmt.Sprintf("%x", sha256.Sum256(bytes))
log.Infof("Image ID: sha256:%s", sum)
// prepare metadata
result.V1Image.Parent = image.Parent
result.Size = size
result.V1Image.ID = imageLayer.ID
imageConfig := metadata.ImageConfig{
V1Image: result.V1Image,
ImageID: sum,
// TODO: this will change when issue 1186 is
// implemented -- only populate the digests when pulled by digest
Digests: []string{manifest.Digest},
Tags: []string{ic.Tag},
Name: manifest.Name,
DiffIDs: diffIDs,
History: history,
Reference: ic.Reference,
}
return imageConfig, nil
}
