func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, foreignSrc distribution.Descriptor, progressOutput progress.Output) (layer.Layer, error) {
rawTar, err := os.Open(filename)
if err != nil {
logrus.Debugf("Error reading embedded tar: %v", err)
return nil, err
}
defer rawTar.Close()
var r io.Reader
if progressOutput != nil {
fileInfo, err := rawTar.Stat()
if err != nil {
logrus.Debugf("Error statting file: %v", err)
return nil, err
}
r = progress.NewProgressReader(rawTar, progressOutput, fileInfo.Size(), stringid.TruncateID(id), "Loading layer")
} else {
r = rawTar
}
inflatedLayerData, err := archive.DecompressStream(r)
if err != nil {
return nil, err
}
defer inflatedLayerData.Close()
if ds, ok := l.ls.(layer.DescribableStore); ok {
return ds.RegisterWithDescriptor(inflatedLayerData, rootFS.ChainID(), foreignSrc)
}
return l.ls.Register(inflatedLayerData, rootFS.ChainID())
}
func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, progressOutput progress.Output) (layer.Layer, error) {
rawTar, err := os.Open(filename)
if err != nil {
logrus.Debugf("Error reading embedded tar: %v", err)
return nil, err
}
defer rawTar.Close()
inflatedLayerData, err := archive.DecompressStream(rawTar)
if err != nil {
return nil, err
}
defer inflatedLayerData.Close()
if progressOutput != nil {
fileInfo, err := os.Stat(filename)
if err != nil {
logrus.Debugf("Error statting file: %v", err)
return nil, err
}
progressReader := progress.NewProgressReader(inflatedLayerData, progressOutput, fileInfo.Size(), stringid.TruncateID(id), "Loading layer")
return l.ls.Register(progressReader, rootFS.ChainID())
}
return l.ls.Register(inflatedLayerData, rootFS.ChainID())
}
func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, foreignSrc distribution.Descriptor, progressOutput progress.Output) (layer.Layer, error) {
// We use system.OpenSequential to use sequential file access on Windows, avoiding
// depleting the standby list. On Linux, this equates to a regular os.Open.
rawTar, err := system.OpenSequential(filename)
if err != nil {
logrus.Debugf("Error reading embedded tar: %v", err)
return nil, err
}
defer rawTar.Close()
var r io.Reader
if progressOutput != nil {
fileInfo, err := rawTar.Stat()
if err != nil {
logrus.Debugf("Error statting file: %v", err)
return nil, err
}
r = progress.NewProgressReader(rawTar, progressOutput, fileInfo.Size(), stringid.TruncateID(id), "Loading layer")
} else {
r = rawTar
}
inflatedLayerData, err := archive.DecompressStream(r)
if err != nil {
return nil, err
}
defer inflatedLayerData.Close()
if ds, ok := l.ls.(layer.DescribableStore); ok {
return ds.RegisterWithDescriptor(inflatedLayerData, rootFS.ChainID(), foreignSrc)
}
return l.ls.Register(inflatedLayerData, rootFS.ChainID())
}
func (dm *downloadManager) Download(ctx context.Context, initialRootFS image.RootFS, layers []xfer.DownloadDescriptor, progressOutput progress.Output) (image.RootFS, func(), error) {
for _, l := range layers {
b, err := dm.blobStore.New()
if err != nil {
return initialRootFS, nil, err
}
defer b.Close()
rc, _, err := l.Download(ctx, progressOutput)
if err != nil {
return initialRootFS, nil, errors.Wrap(err, "failed to download")
}
defer rc.Close()
r := io.TeeReader(rc, b)
inflatedLayerData, err := archive.DecompressStream(r)
if err != nil {
return initialRootFS, nil, err
}
digester := digest.Canonical.New()
if _, err := archive.ApplyLayer(dm.tmpDir, io.TeeReader(inflatedLayerData, digester.Hash())); err != nil {
return initialRootFS, nil, err
}
initialRootFS.Append(layer.DiffID(digester.Digest()))
d, err := b.Commit()
if err != nil {
return initialRootFS, nil, err
}
dm.blobs = append(dm.blobs, d)
}
return initialRootFS, nil, nil
}
func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS) (layer.Layer, error) {
rawTar, err := os.Open(filename)
if err != nil {
logrus.Debugf("Error reading embedded tar: %v", err)
return nil, err
}
inflatedLayerData, err := archive.DecompressStream(rawTar)
if err != nil {
return nil, err
}
defer rawTar.Close()
defer inflatedLayerData.Close()
return l.ls.Register(inflatedLayerData, rootFS.ChainID())
}
func setupBaseLayer(history []schema1.History, rootFS image.RootFS) error {
var v1Config map[string]*json.RawMessage
if err := json.Unmarshal([]byte(history[len(history)-1].V1Compatibility), &v1Config); err != nil {
return err
}
baseID, err := json.Marshal(rootFS.BaseLayerID())
if err != nil {
return err
}
v1Config["parent"] = (*json.RawMessage)(&baseID)
configJSON, err := json.Marshal(v1Config)
if err != nil {
return err
}
history[len(history)-1].V1Compatibility = string(configJSON)
return nil
}
func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
v1img := &image.V1Image{}
if err := json.Unmarshal([]byte(m.History[len(m.History)-1].V1Compatibility), v1img); err != nil {
return err
}
if v1img.Parent == "" {
return fmt.Errorf("Last layer %q does not have a base layer reference", v1img.ID)
}
// There must be an image that already references the baselayer.
for _, img := range is.Map() {
if img.RootFS.Type == image.TypeLayersWithBase && img.RootFS.BaseLayerID() == v1img.Parent {
rootFS.BaseLayer = img.RootFS.BaseLayer
rootFS.Type = image.TypeLayersWithBase
return nil
}
}
return fmt.Errorf("Invalid base layer %q", v1img.Parent)
}
func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest) (imageID image.ID, manifestDigest digest.Digest, err error) {
manifestDigest, err = schema2ManifestDigest(ref, mfst)
if err != nil {
return "", "", err
}
target := mfst.Target()
imageID = image.ID(target.Digest)
if _, err := p.config.ImageStore.Get(imageID); err == nil {
// If the image already exists locally, no need to pull
// anything.
return imageID, manifestDigest, nil
}
configChan := make(chan []byte, 1)
errChan := make(chan error, 1)
var cancel func()
ctx, cancel = context.WithCancel(ctx)
// Pull the image config
go func() {
configJSON, err := p.pullSchema2ImageConfig(ctx, target.Digest)
if err != nil {
errChan <- err
cancel()
return
}
configChan <- configJSON
}()
var descriptors []xfer.DownloadDescriptor
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for _, d := range mfst.References() {
layerDescriptor := &v2LayerDescriptor{
digest: d.Digest,
repo: p.repo,
blobSumService: p.blobSumService,
}
descriptors = append(descriptors, layerDescriptor)
}
var (
configJSON []byte // raw serialized image config
unmarshalledConfig image.Image // deserialized image config
downloadRootFS image.RootFS // rootFS to use for registering layers.
)
if runtime.GOOS == "windows" {
configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
if err != nil {
return "", "", err
}
if unmarshalledConfig.RootFS == nil {
return "", "", errors.New("image config has no rootfs section")
}
downloadRootFS = *unmarshalledConfig.RootFS
downloadRootFS.DiffIDs = []layer.DiffID{}
} else {
downloadRootFS = *image.NewRootFS()
}
rootFS, release, err := p.config.DownloadManager.Download(ctx, downloadRootFS, descriptors, p.config.ProgressOutput)
if err != nil {
if configJSON != nil {
// Already received the config
return "", "", err
}
select {
case err = <-errChan:
return "", "", err
default:
cancel()
select {
case <-configChan:
case <-errChan:
}
return "", "", err
}
}
defer release()
if configJSON == nil {
configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
if err != nil {
return "", "", err
}
}
// The DiffIDs returned in rootFS MUST match those in the config.
// Otherwise the image config could be referencing layers that aren't
// included in the manifest.
if len(rootFS.DiffIDs) != len(unmarshalledConfig.RootFS.DiffIDs) {
return "", "", errRootFSMismatch
}
for i := range rootFS.DiffIDs {
if rootFS.DiffIDs[i] != unmarshalledConfig.RootFS.DiffIDs[i] {
return "", "", errRootFSMismatch
}
}
imageID, err = p.config.ImageStore.Create(configJSON)
if err != nil {
return "", "", err
}
return imageID, manifestDigest, nil
}
func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool) error {
var (
sf = streamformatter.NewJSONStreamFormatter()
progressOutput progress.Output
)
if !quiet {
progressOutput = sf.NewProgressOutput(outStream, false)
outStream = &streamformatter.StdoutFormatter{Writer: outStream, StreamFormatter: streamformatter.NewJSONStreamFormatter()}
}
tmpDir, err := ioutil.TempDir("", "docker-import-")
if err != nil {
return err
}
defer os.RemoveAll(tmpDir)
if err := chrootarchive.Untar(inTar, tmpDir, nil); err != nil {
return err
}
// read manifest, if no file then load in legacy mode
manifestPath, err := safePath(tmpDir, manifestFileName)
if err != nil {
return err
}
manifestFile, err := os.Open(manifestPath)
if err != nil {
if os.IsNotExist(err) {
return l.legacyLoad(tmpDir, outStream, progressOutput)
}
return manifestFile.Close()
}
defer manifestFile.Close()
var manifest []manifestItem
if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil {
return err
}
var parentLinks []parentLink
for _, m := range manifest {
configPath, err := safePath(tmpDir, m.Config)
if err != nil {
return err
}
config, err := ioutil.ReadFile(configPath)
if err != nil {
return err
}
img, err := image.NewFromJSON(config)
if err != nil {
return err
}
var rootFS image.RootFS
rootFS = *img.RootFS
rootFS.DiffIDs = nil
if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual {
return fmt.Errorf("invalid manifest, layers length mismatch: expected %q, got %q", expected, actual)
}
for i, diffID := range img.RootFS.DiffIDs {
layerPath, err := safePath(tmpDir, m.Layers[i])
if err != nil {
return err
}
r := rootFS
r.Append(diffID)
newLayer, err := l.ls.Get(r.ChainID())
if err != nil {
newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), progressOutput)
if err != nil {
return err
}
}
defer layer.ReleaseAndLog(l.ls, newLayer)
if expected, actual := diffID, newLayer.DiffID(); expected != actual {
return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
}
rootFS.Append(diffID)
}
imgID, err := l.is.Create(config)
if err != nil {
return err
}
for _, repoTag := range m.RepoTags {
named, err := reference.ParseNamed(repoTag)
if err != nil {
return err
}
ref, ok := named.(reference.NamedTagged)
if !ok {
return fmt.Errorf("invalid tag %q", repoTag)
}
l.setLoadedTag(ref, imgID, outStream)
}
parentLinks = append(parentLinks, parentLink{imgID, m.Parent})
l.loggerImgEvent.LogImageEvent(imgID.String(), imgID.String(), "load")
}
for _, p := range validatedParentLinks(parentLinks) {
if p.parentID != "" {
if err := l.setParentID(p.id, p.parentID); err != nil {
return err
}
}
}
return nil
}
func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer) error {
tmpDir, err := ioutil.TempDir("", "docker-import-")
if err != nil {
return err
}
defer os.RemoveAll(tmpDir)
if err := chrootarchive.Untar(inTar, tmpDir, nil); err != nil {
return err
}
// read manifest, if no file then load in legacy mode
manifestPath, err := safePath(tmpDir, manifestFileName)
if err != nil {
return err
}
manifestFile, err := os.Open(manifestPath)
if err != nil {
if os.IsNotExist(err) {
return l.legacyLoad(tmpDir, outStream)
}
return manifestFile.Close()
}
defer manifestFile.Close()
var manifest []manifestItem
if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil {
return err
}
for _, m := range manifest {
configPath, err := safePath(tmpDir, m.Config)
if err != nil {
return err
}
config, err := ioutil.ReadFile(configPath)
if err != nil {
return err
}
img, err := image.NewFromJSON(config)
if err != nil {
return err
}
var rootFS image.RootFS
rootFS = *img.RootFS
rootFS.DiffIDs = nil
if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual {
return fmt.Errorf("invalid manifest, layers length mismatch: expected %q, got %q", expected, actual)
}
for i, diffID := range img.RootFS.DiffIDs {
layerPath, err := safePath(tmpDir, m.Layers[i])
if err != nil {
return err
}
newLayer, err := l.loadLayer(layerPath, rootFS)
if err != nil {
return err
}
defer layer.ReleaseAndLog(l.ls, newLayer)
if expected, actual := diffID, newLayer.DiffID(); expected != actual {
return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
}
rootFS.Append(diffID)
}
imgID, err := l.is.Create(config)
if err != nil {
return err
}
for _, repoTag := range m.RepoTags {
named, err := reference.ParseNamed(repoTag)
if err != nil {
return err
}
ref, ok := named.(reference.NamedTagged)
if !ok {
return fmt.Errorf("invalid tag %q", repoTag)
}
l.setLoadedTag(ref, imgID, outStream)
}
}
return nil
}
// SquashImage creates a new image with the diff of the specified image and the specified parent.
// This new image contains only the layers from it's parent + 1 extra layer which contains the diff of all the layers in between.
// The existing image(s) is not destroyed.
// If no parent is specified, a new image with the diff of all the specified image's layers merged into a new layer that has no parents.
func (daemon *Daemon) SquashImage(id, parent string) (string, error) {
img, err := daemon.imageStore.Get(image.ID(id))
if err != nil {
return "", err
}
var parentImg *image.Image
var parentChainID layer.ChainID
if len(parent) != 0 {
parentImg, err = daemon.imageStore.Get(image.ID(parent))
if err != nil {
return "", errors.Wrap(err, "error getting specified parent layer")
}
parentChainID = parentImg.RootFS.ChainID()
} else {
rootFS := image.NewRootFS()
parentImg = &image.Image{RootFS: rootFS}
}
l, err := daemon.layerStore.Get(img.RootFS.ChainID())
if err != nil {
return "", errors.Wrap(err, "error getting image layer")
}
defer daemon.layerStore.Release(l)
ts, err := l.TarStreamFrom(parentChainID)
if err != nil {
return "", errors.Wrapf(err, "error getting tar stream to parent")
}
defer ts.Close()
newL, err := daemon.layerStore.Register(ts, parentChainID)
if err != nil {
return "", errors.Wrap(err, "error registering layer")
}
defer daemon.layerStore.Release(newL)
var newImage image.Image
newImage = *img
newImage.RootFS = nil
var rootFS image.RootFS
rootFS = *parentImg.RootFS
rootFS.DiffIDs = append(rootFS.DiffIDs, newL.DiffID())
newImage.RootFS = &rootFS
for i, hi := range newImage.History {
if i >= len(parentImg.History) {
hi.EmptyLayer = true
}
newImage.History[i] = hi
}
now := time.Now()
var historyComment string
if len(parent) > 0 {
historyComment = fmt.Sprintf("merge %s to %s", id, parent)
} else {
historyComment = fmt.Sprintf("create new from %s", id)
}
newImage.History = append(newImage.History, image.History{
Created: now,
Comment: historyComment,
})
newImage.Created = now
b, err := json.Marshal(&newImage)
if err != nil {
return "", errors.Wrap(err, "error marshalling image config")
}
newImgID, err := daemon.imageStore.Create(b)
if err != nil {
return "", errors.Wrap(err, "error creating new image after squash")
}
return string(newImgID), nil
}