func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) error {
if _, exists := s.savedLayers[legacyImg.ID]; exists {
return nil
}
outDir := filepath.Join(s.outDir, legacyImg.ID)
if err := os.Mkdir(outDir, 0755); err != nil {
return err
}
// todo: why is this version file here?
if err := ioutil.WriteFile(filepath.Join(outDir, legacyVersionFileName), []byte("1.0"), 0644); err != nil {
return err
}
imageConfig, err := json.Marshal(legacyImg)
if err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(outDir, legacyConfigFileName), imageConfig, 0644); err != nil {
return err
}
// serialize filesystem
tarFile, err := os.Create(filepath.Join(outDir, legacyLayerFileName))
if err != nil {
return err
}
defer tarFile.Close()
l, err := s.ls.Get(id)
if err != nil {
return err
}
defer layer.ReleaseAndLog(s.ls, l)
arch, err := l.TarStream()
if err != nil {
return err
}
defer arch.Close()
if _, err := io.Copy(tarFile, arch); err != nil {
return err
}
for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} {
// todo: maybe save layer created timestamp?
if err := os.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil {
return err
}
}
s.savedLayers[legacyImg.ID] = struct{}{}
return nil
}
func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool) error {
var (
sf = streamformatter.NewJSONStreamFormatter()
progressOutput progress.Output
)
if !quiet {
progressOutput = sf.NewProgressOutput(outStream, false)
outStream = &streamformatter.StdoutFormatter{Writer: outStream, StreamFormatter: streamformatter.NewJSONStreamFormatter()}
}
tmpDir, err := ioutil.TempDir("", "docker-import-")
if err != nil {
return err
}
defer os.RemoveAll(tmpDir)
if err := chrootarchive.Untar(inTar, tmpDir, nil); err != nil {
return err
}
// read manifest, if no file then load in legacy mode
manifestPath, err := safePath(tmpDir, manifestFileName)
if err != nil {
return err
}
manifestFile, err := os.Open(manifestPath)
if err != nil {
if os.IsNotExist(err) {
return l.legacyLoad(tmpDir, outStream, progressOutput)
}
return manifestFile.Close()
}
defer manifestFile.Close()
var manifest []manifestItem
if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil {
return err
}
var parentLinks []parentLink
for _, m := range manifest {
configPath, err := safePath(tmpDir, m.Config)
if err != nil {
return err
}
config, err := ioutil.ReadFile(configPath)
if err != nil {
return err
}
img, err := image.NewFromJSON(config)
if err != nil {
return err
}
var rootFS image.RootFS
rootFS = *img.RootFS
rootFS.DiffIDs = nil
if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual {
return fmt.Errorf("invalid manifest, layers length mismatch: expected %q, got %q", expected, actual)
}
for i, diffID := range img.RootFS.DiffIDs {
layerPath, err := safePath(tmpDir, m.Layers[i])
if err != nil {
return err
}
r := rootFS
r.Append(diffID)
newLayer, err := l.ls.Get(r.ChainID())
if err != nil {
newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), progressOutput)
if err != nil {
return err
}
}
defer layer.ReleaseAndLog(l.ls, newLayer)
if expected, actual := diffID, newLayer.DiffID(); expected != actual {
return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
}
rootFS.Append(diffID)
}
imgID, err := l.is.Create(config)
if err != nil {
return err
}
for _, repoTag := range m.RepoTags {
named, err := reference.ParseNamed(repoTag)
if err != nil {
return err
}
ref, ok := named.(reference.NamedTagged)
if !ok {
return fmt.Errorf("invalid tag %q", repoTag)
}
l.setLoadedTag(ref, imgID, outStream)
}
parentLinks = append(parentLinks, parentLink{imgID, m.Parent})
l.loggerImgEvent.LogImageEvent(imgID.String(), imgID.String(), "load")
}
for _, p := range validatedParentLinks(parentLinks) {
if p.parentID != "" {
if err := l.setParentID(p.id, p.parentID); err != nil {
return err
}
}
}
return nil
}
// Commit creates a new filesystem image from the current state of a container.
// The image can optionally be tagged into a repository.
func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (string, error) {
container, err := daemon.Get(name)
if err != nil {
return "", err
}
// It is not possible to commit a running container on Windows
if runtime.GOOS == "windows" && container.IsRunning() {
return "", fmt.Errorf("Windows does not support commit of a running container")
}
if c.Pause && !container.isPaused() {
daemon.containerPause(container)
defer daemon.containerUnpause(container)
}
if c.MergeConfigs {
if err := runconfig.Merge(c.Config, container.Config); err != nil {
return "", err
}
}
rwTar, err := daemon.exportContainerRw(container)
if err != nil {
return "", err
}
defer func() {
if rwTar != nil {
rwTar.Close()
}
}()
var history []image.History
rootFS := image.NewRootFS()
if container.ImageID != "" {
img, err := daemon.imageStore.Get(container.ImageID)
if err != nil {
return "", err
}
history = img.History
rootFS = img.RootFS
}
l, err := daemon.layerStore.Register(rwTar, rootFS.ChainID())
if err != nil {
return "", err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
h := image.History{
Author: c.Author,
Created: time.Now().UTC(),
CreatedBy: strings.Join(container.Config.Cmd.Slice(), " "),
Comment: c.Comment,
EmptyLayer: true,
}
if diffID := l.DiffID(); layer.DigestSHA256EmptyTar != diffID {
h.EmptyLayer = false
rootFS.Append(diffID)
}
history = append(history, h)
config, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Config: c.Config,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Container: container.ID,
ContainerConfig: *container.Config,
Author: c.Author,
Created: h.Created,
},
RootFS: rootFS,
History: history,
})
if err != nil {
return "", err
}
id, err := daemon.imageStore.Create(config)
if err != nil {
return "", err
}
if container.ImageID != "" {
if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil {
return "", err
}
}
if c.Repo != "" {
newTag, err := reference.WithName(c.Repo) // todo: should move this to API layer
if err != nil {
return "", err
}
if c.Tag != "" {
if newTag, err = reference.WithTag(newTag, c.Tag); err != nil {
return "", err
}
}
if err := daemon.TagImage(newTag, id.String(), true); err != nil {
return "", err
}
}
daemon.LogContainerEvent(container, "commit")
return id.String(), nil
}
// Images returns a filtered list of images. filterArgs is a JSON-encoded set
// of filter arguments which will be interpreted by api/types/filters.
// filter is a shell glob string applied to repository names. The argument
// named all controls whether all images in the graph are filtered, or just
// the heads.
func (daemon *Daemon) Images(filterArgs, filter string, all bool) ([]*types.Image, error) {
var (
allImages map[image.ID]*image.Image
err error
danglingOnly = false
)
imageFilters, err := filters.FromParam(filterArgs)
if err != nil {
return nil, err
}
if err := imageFilters.Validate(acceptedImageFilterTags); err != nil {
return nil, err
}
if imageFilters.Include("dangling") {
if imageFilters.ExactMatch("dangling", "true") {
danglingOnly = true
} else if !imageFilters.ExactMatch("dangling", "false") {
return nil, fmt.Errorf("Invalid filter 'dangling=%s'", imageFilters.Get("dangling"))
}
}
if danglingOnly {
allImages = daemon.imageStore.Heads()
} else {
allImages = daemon.imageStore.Map()
}
var beforeFilter, sinceFilter *image.Image
err = imageFilters.WalkValues("before", func(value string) error {
beforeFilter, err = daemon.GetImage(value)
return err
})
if err != nil {
return nil, err
}
err = imageFilters.WalkValues("since", func(value string) error {
sinceFilter, err = daemon.GetImage(value)
return err
})
if err != nil {
return nil, err
}
images := []*types.Image{}
var filterTagged bool
if filter != "" {
filterRef, err := reference.ParseNamed(filter)
if err == nil { // parse error means wildcard repo
if _, ok := filterRef.(reference.NamedTagged); ok {
filterTagged = true
}
}
}
for id, img := range allImages {
if beforeFilter != nil {
if img.Created.Equal(beforeFilter.Created) || img.Created.After(beforeFilter.Created) {
continue
}
}
if sinceFilter != nil {
if img.Created.Equal(sinceFilter.Created) || img.Created.Before(sinceFilter.Created) {
continue
}
}
if imageFilters.Include("label") {
// Very old image that do not have image.Config (or even labels)
if img.Config == nil {
continue
}
// We are now sure image.Config is not nil
if !imageFilters.MatchKVList("label", img.Config.Labels) {
continue
}
}
layerID := img.RootFS.ChainID()
var size int64
if layerID != "" {
l, err := daemon.layerStore.Get(layerID)
if err != nil {
return nil, err
}
size, err = l.Size()
layer.ReleaseAndLog(daemon.layerStore, l)
if err != nil {
return nil, err
}
}
newImage := newImage(img, size)
for _, ref := range daemon.referenceStore.References(id.Digest()) {
if filter != "" { // filter by tag/repo name
if filterTagged { // filter by tag, require full ref match
if ref.String() != filter {
continue
}
} else if matched, err := path.Match(filter, ref.Name()); !matched || err != nil { // name only match, FIXME: docs say exact
continue
}
}
if _, ok := ref.(reference.Canonical); ok {
newImage.RepoDigests = append(newImage.RepoDigests, ref.String())
}
if _, ok := ref.(reference.NamedTagged); ok {
newImage.RepoTags = append(newImage.RepoTags, ref.String())
}
}
if newImage.RepoDigests == nil && newImage.RepoTags == nil {
if all || len(daemon.imageStore.Children(id)) == 0 {
if imageFilters.Include("dangling") && !danglingOnly {
//dangling=false case, so dangling image is not needed
continue
}
if filter != "" { // skip images with no references if filtering by tag
continue
}
newImage.RepoDigests = []string{"<none>@<none>"}
newImage.RepoTags = []string{"<none>:<none>"}
} else {
continue
}
} else if danglingOnly && len(newImage.RepoTags) > 0 {
continue
}
images = append(images, newImage)
}
sort.Sort(sort.Reverse(byCreated(images)))
return images, nil
}
// makeDownloadFuncFromDownload returns a function that performs the layer
// registration when the layer data is coming from an existing download. It
// waits for sourceDownload and parentDownload to complete, and then
// reregisters the data from sourceDownload's top layer on top of
// parentDownload. This function does not log progress output because it would
// interfere with the progress reporting for sourceDownload, which has the same
// Key.
func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor DownloadDescriptor, sourceDownload *downloadTransfer, parentDownload *downloadTransfer) DoFunc {
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
d := &downloadTransfer{
Transfer: NewTransfer(),
layerStore: ldm.layerStore,
}
go func() {
defer func() {
close(progressChan)
}()
<-start
close(inactive)
select {
case <-d.Transfer.Context().Done():
d.err = errors.New("layer registration cancelled")
return
case <-parentDownload.Done():
}
l, err := parentDownload.result()
if err != nil {
d.err = err
return
}
parentLayer := l.ChainID()
// sourceDownload should have already finished if
// parentDownload finished, but wait for it explicitly
// to be sure.
select {
case <-d.Transfer.Context().Done():
d.err = errors.New("layer registration cancelled")
return
case <-sourceDownload.Done():
}
l, err = sourceDownload.result()
if err != nil {
d.err = err
return
}
layerReader, err := l.TarStream()
if err != nil {
d.err = err
return
}
defer layerReader.Close()
var src distribution.Descriptor
if fs, ok := l.(distribution.Describable); ok {
src = fs.Descriptor()
}
if ds, ok := d.layerStore.(layer.DescribableStore); ok {
d.layer, err = ds.RegisterWithDescriptor(layerReader, parentLayer, src)
} else {
d.layer, err = d.layerStore.Register(layerReader, parentLayer)
}
if err != nil {
d.err = fmt.Errorf("failed to register layer: %v", err)
return
}
withRegistered, hasRegistered := descriptor.(DownloadDescriptorWithRegistered)
if hasRegistered {
withRegistered.Registered(d.layer.DiffID())
}
// Doesn't actually need to be its own goroutine, but
// done like this so we can defer close(c).
go func() {
<-d.Transfer.Released()
if d.layer != nil {
layer.ReleaseAndLog(d.layerStore, d.layer)
}
}()
}()
return d
}
}
func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) (distribution.Descriptor, error) {
if _, exists := s.savedLayers[legacyImg.ID]; exists {
return distribution.Descriptor{}, nil
}
outDir := filepath.Join(s.outDir, legacyImg.ID)
if err := os.Mkdir(outDir, 0755); err != nil {
return distribution.Descriptor{}, err
}
// todo: why is this version file here?
if err := ioutil.WriteFile(filepath.Join(outDir, legacyVersionFileName), []byte("1.0"), 0644); err != nil {
return distribution.Descriptor{}, err
}
imageConfig, err := json.Marshal(legacyImg)
if err != nil {
return distribution.Descriptor{}, err
}
if err := ioutil.WriteFile(filepath.Join(outDir, legacyConfigFileName), imageConfig, 0644); err != nil {
return distribution.Descriptor{}, err
}
// serialize filesystem
layerPath := filepath.Join(outDir, legacyLayerFileName)
l, err := s.ls.Get(id)
if err != nil {
return distribution.Descriptor{}, err
}
defer layer.ReleaseAndLog(s.ls, l)
if oldPath, exists := s.diffIDPaths[l.DiffID()]; exists {
relPath, err := filepath.Rel(outDir, oldPath)
if err != nil {
return distribution.Descriptor{}, err
}
os.Symlink(relPath, layerPath)
} else {
tarFile, err := os.Create(layerPath)
if err != nil {
return distribution.Descriptor{}, err
}
defer tarFile.Close()
arch, err := l.TarStream()
if err != nil {
return distribution.Descriptor{}, err
}
defer arch.Close()
if _, err := io.Copy(tarFile, arch); err != nil {
return distribution.Descriptor{}, err
}
for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} {
// todo: maybe save layer created timestamp?
if err := system.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil {
return distribution.Descriptor{}, err
}
}
s.diffIDPaths[l.DiffID()] = layerPath
}
s.savedLayers[legacyImg.ID] = struct{}{}
var src distribution.Descriptor
if fs, ok := l.(distribution.Describable); ok {
src = fs.Descriptor()
}
return src, nil
}
func (p *v2Pusher) pushV2Tag(ctx context.Context, association reference.Association) error {
ref := association.Ref
logrus.Debugf("Pushing repository: %s", ref.String())
img, err := p.config.ImageStore.Get(association.ImageID)
if err != nil {
return fmt.Errorf("could not find image from tag %s: %v", ref.String(), err)
}
var l layer.Layer
topLayerID := img.RootFS.ChainID()
if topLayerID == "" {
l = layer.EmptyLayer
} else {
l, err = p.config.LayerStore.Get(topLayerID)
if err != nil {
return fmt.Errorf("failed to get top layer from image: %v", err)
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
}
var descriptors []xfer.UploadDescriptor
descriptorTemplate := v2PushDescriptor{
blobSumService: p.blobSumService,
repo: p.repo,
layersPushed: &p.layersPushed,
confirmedV2: &p.confirmedV2,
}
// Push empty layer if necessary
for _, h := range img.History {
if h.EmptyLayer {
descriptor := descriptorTemplate
descriptor.layer = layer.EmptyLayer
descriptors = []xfer.UploadDescriptor{&descriptor}
break
}
}
// Loop bounds condition is to avoid pushing the base layer on Windows.
for i := 0; i < len(img.RootFS.DiffIDs); i++ {
descriptor := descriptorTemplate
descriptor.layer = l
descriptors = append(descriptors, &descriptor)
l = l.Parent()
}
fsLayers, err := p.config.UploadManager.Upload(ctx, descriptors, p.config.ProgressOutput)
if err != nil {
return err
}
var tag string
if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
tag = tagged.Tag()
}
m, err := CreateV2Manifest(p.repo.Name(), tag, img, fsLayers)
if err != nil {
return err
}
logrus.Infof("Signed manifest for %s using daemon's key: %s", ref.String(), p.config.TrustKey.KeyID())
signed, err := schema1.Sign(m, p.config.TrustKey)
if err != nil {
return err
}
manifestDigest, manifestSize, err := digestFromManifest(signed, ref)
if err != nil {
return err
}
if manifestDigest != "" {
if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
// NOTE: do not change this format without first changing the trust client
// code. This information is used to determine what was pushed and should be signed.
progress.Messagef(p.config.ProgressOutput, "", "%s: digest: %s size: %d", tagged.Tag(), manifestDigest, manifestSize)
}
}
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return err
}
return manSvc.Put(signed)
}
// ImageHistory returns a slice of ImageHistory structures for the specified image
// name by walking the image lineage.
func (daemon *Daemon) ImageHistory(name string) ([]*types.ImageHistory, error) {
img, err := daemon.GetImage(name)
if err != nil {
return nil, err
}
history := []*types.ImageHistory{}
layerCounter := 0
rootFS := *img.RootFS
rootFS.DiffIDs = nil
for _, h := range img.History {
var layerSize int64
if !h.EmptyLayer {
if len(img.RootFS.DiffIDs) <= layerCounter {
return nil, errors.New("too many non-empty layers in History section")
}
rootFS.Append(img.RootFS.DiffIDs[layerCounter])
l, err := daemon.layerStore.Get(rootFS.ChainID())
if err != nil {
return nil, err
}
layerSize, err = l.DiffSize()
layer.ReleaseAndLog(daemon.layerStore, l)
if err != nil {
return nil, err
}
layerCounter++
}
history = append([]*types.ImageHistory{{
ID: "<missing>",
Created: h.Created.Unix(),
CreatedBy: h.CreatedBy,
Comment: h.Comment,
Size: layerSize,
}}, history...)
}
// Fill in image IDs and tags
histImg := img
id := img.ID()
for _, h := range history {
h.ID = id.String()
var tags []string
for _, r := range daemon.tagStore.References(id) {
if _, ok := r.(reference.NamedTagged); ok {
tags = append(tags, r.String())
}
}
h.Tags = tags
id = histImg.Parent
if id == "" {
break
}
histImg, err = daemon.GetImage(id.String())
if err != nil {
break
}
}
return history, nil
}
func (p *v1Puller) pullImage(out io.Writer, v1ID, endpoint string, localNameRef reference.Named) (layersDownloaded bool, err error) {
var history []string
history, err = p.session.GetRemoteHistory(v1ID, endpoint)
if err != nil {
return false, err
}
if len(history) < 1 {
return false, fmt.Errorf("empty history for image %s", v1ID)
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(v1ID), "Pulling dependent layers", nil))
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
var (
referencedLayers []layer.Layer
parentID layer.ChainID
newHistory []image.History
img *image.V1Image
imgJSON []byte
imgSize int64
)
defer func() {
for _, l := range referencedLayers {
layer.ReleaseAndLog(p.config.LayerStore, l)
}
}()
layersDownloaded = false
// Iterate over layers from top-most to bottom-most, checking if any
// already exist on disk.
var i int
for i = 0; i != len(history); i++ {
v1LayerID := history[i]
// Do we have a mapping for this particular v1 ID on this
// registry?
if layerID, err := p.v1IDService.Get(v1LayerID, p.repoInfo.Index.Name); err == nil {
// Does the layer actually exist
if l, err := p.config.LayerStore.Get(layerID); err == nil {
for j := i; j >= 0; j-- {
logrus.Debugf("Layer already exists: %s", history[j])
out.Write(p.sf.FormatProgress(stringid.TruncateID(history[j]), "Already exists", nil))
}
referencedLayers = append(referencedLayers, l)
parentID = layerID
break
}
}
}
needsDownload := i
// Iterate over layers, in order from bottom-most to top-most. Download
// config for all layers, and download actual layer data if needed.
for i = len(history) - 1; i >= 0; i-- {
v1LayerID := history[i]
imgJSON, imgSize, err = p.downloadLayerConfig(out, v1LayerID, endpoint)
if err != nil {
return layersDownloaded, err
}
img = &image.V1Image{}
if err := json.Unmarshal(imgJSON, img); err != nil {
return layersDownloaded, err
}
if i < needsDownload {
l, err := p.downloadLayer(out, v1LayerID, endpoint, parentID, imgSize, &layersDownloaded)
// Note: This needs to be done even in the error case to avoid
// stale references to the layer.
if l != nil {
referencedLayers = append(referencedLayers, l)
}
if err != nil {
return layersDownloaded, err
}
parentID = l.ChainID()
}
// Create a new-style config from the legacy configs
h, err := v1.HistoryFromConfig(imgJSON, false)
if err != nil {
return layersDownloaded, err
}
newHistory = append(newHistory, h)
}
rootFS := image.NewRootFS()
l := referencedLayers[len(referencedLayers)-1]
for l != nil {
rootFS.DiffIDs = append([]layer.DiffID{l.DiffID()}, rootFS.DiffIDs...)
l = l.Parent()
}
config, err := v1.MakeConfigFromV1Config(imgJSON, rootFS, newHistory)
if err != nil {
return layersDownloaded, err
}
imageID, err := p.config.ImageStore.Create(config)
if err != nil {
return layersDownloaded, err
}
if err := p.config.TagStore.Add(localNameRef, imageID, true); err != nil {
return layersDownloaded, err
}
return layersDownloaded, nil
}
func (p *v2Pusher) pushV2Tag(association tag.Association) error {
ref := association.Ref
logrus.Debugf("Pushing repository: %s", ref.String())
img, err := p.config.ImageStore.Get(association.ImageID)
if err != nil {
return fmt.Errorf("could not find image from tag %s: %v", ref.String(), err)
}
out := p.config.OutStream
var l layer.Layer
topLayerID := img.RootFS.ChainID()
if topLayerID == "" {
l = layer.EmptyLayer
} else {
l, err = p.config.LayerStore.Get(topLayerID)
if err != nil {
return fmt.Errorf("failed to get top layer from image: %v", err)
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
}
fsLayers := make(map[layer.DiffID]schema1.FSLayer)
// Push empty layer if necessary
for _, h := range img.History {
if h.EmptyLayer {
dgst, err := p.pushLayerIfNecessary(out, layer.EmptyLayer)
if err != nil {
return err
}
p.layersPushed[dgst] = true
fsLayers[layer.EmptyLayer.DiffID()] = schema1.FSLayer{BlobSum: dgst}
break
}
}
for i := 0; i < len(img.RootFS.DiffIDs); i++ {
dgst, err := p.pushLayerIfNecessary(out, l)
if err != nil {
return err
}
p.layersPushed[dgst] = true
fsLayers[l.DiffID()] = schema1.FSLayer{BlobSum: dgst}
l = l.Parent()
}
var tag string
if tagged, isTagged := ref.(reference.Tagged); isTagged {
tag = tagged.Tag()
}
m, err := CreateV2Manifest(p.repo.Name(), tag, img, fsLayers)
if err != nil {
return err
}
logrus.Infof("Signed manifest for %s using daemon's key: %s", ref.String(), p.config.TrustKey.KeyID())
signed, err := schema1.Sign(m, p.config.TrustKey)
if err != nil {
return err
}
manifestDigest, manifestSize, err := digestFromManifest(signed, p.repo.Name())
if err != nil {
return err
}
if manifestDigest != "" {
if tagged, isTagged := ref.(reference.Tagged); isTagged {
// NOTE: do not change this format without first changing the trust client
// code. This information is used to determine what was pushed and should be signed.
out.Write(p.sf.FormatStatus("", "%s: digest: %s size: %d", tagged.Tag(), manifestDigest, manifestSize))
}
}
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return err
}
return manSvc.Put(signed)
}
func (p *v2Puller) pullV2Tag(out io.Writer, ref reference.Named) (tagUpdated bool, err error) {
tagOrDigest := ""
if tagged, isTagged := ref.(reference.Tagged); isTagged {
tagOrDigest = tagged.Tag()
} else if digested, isDigested := ref.(reference.Digested); isDigested {
tagOrDigest = digested.Digest().String()
} else {
return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String())
}
logrus.Debugf("Pulling ref from V2 registry: %q", tagOrDigest)
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return false, err
}
unverifiedManifest, err := manSvc.GetByTag(tagOrDigest)
if err != nil {
return false, err
}
if unverifiedManifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
}
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifyManifest(unverifiedManifest, ref)
if err != nil {
return false, err
}
rootFS := image.NewRootFS()
if err := detectBaseLayer(p.config.ImageStore, verifiedManifest, rootFS); err != nil {
return false, err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return false, err
}
out.Write(p.sf.FormatStatus(tagOrDigest, "Pulling from %s", p.repo.Name()))
var downloads []*downloadInfo
defer func() {
for _, d := range downloads {
p.config.Pool.removeWithError(d.poolKey, err)
if d.tmpFile != nil {
d.tmpFile.Close()
if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
}
}
}
}()
// Image history converted to the new format
var history []image.History
poolKey := "v2layer:"
notFoundLocally := false
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
blobSum := verifiedManifest.FSLayers[i].BlobSum
poolKey += blobSum.String()
var throwAway struct {
ThrowAway bool `json:"throwaway,omitempty"`
}
if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
return false, err
}
h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
if err != nil {
return false, err
}
history = append(history, h)
if throwAway.ThrowAway {
continue
}
// Do we have a layer on disk corresponding to the set of
// blobsums up to this point?
if !notFoundLocally {
notFoundLocally = true
diffID, err := p.blobSumService.GetDiffID(blobSum)
if err == nil {
rootFS.Append(diffID)
if l, err := p.config.LayerStore.Get(rootFS.ChainID()); err == nil {
notFoundLocally = false
logrus.Debugf("Layer already exists: %s", blobSum.String())
out.Write(p.sf.FormatProgress(stringid.TruncateID(blobSum.String()), "Already exists", nil))
defer layer.ReleaseAndLog(p.config.LayerStore, l)
continue
} else {
rootFS.DiffIDs = rootFS.DiffIDs[:len(rootFS.DiffIDs)-1]
}
}
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(blobSum.String()), "Pulling fs layer", nil))
tmpFile, err := ioutil.TempFile("", "GetImageBlob")
if err != nil {
return false, err
}
d := &downloadInfo{
poolKey: poolKey,
digest: blobSum,
tmpFile: tmpFile,
// TODO: seems like this chan buffer solved hanging problem in go1.5,
// this can indicate some deeper problem that somehow we never take
// error from channel in loop below
err: make(chan error, 1),
}
downloads = append(downloads, d)
broadcaster, found := p.config.Pool.add(d.poolKey)
broadcaster.Add(out)
d.broadcaster = broadcaster
if found {
d.err <- nil
} else {
go p.download(d)
}
}
for _, d := range downloads {
if err := <-d.err; err != nil {
return false, err
}
if d.layer == nil {
// Wait for a different pull to download and extract
// this layer.
err = d.broadcaster.Wait()
if err != nil {
return false, err
}
diffID, err := p.blobSumService.GetDiffID(d.digest)
if err != nil {
return false, err
}
rootFS.Append(diffID)
l, err := p.config.LayerStore.Get(rootFS.ChainID())
if err != nil {
return false, err
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
continue
}
d.tmpFile.Seek(0, 0)
reader := progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: d.broadcaster,
Formatter: p.sf,
Size: d.size,
NewLines: false,
ID: stringid.TruncateID(d.digest.String()),
Action: "Extracting",
})
inflatedLayerData, err := archive.DecompressStream(reader)
if err != nil {
return false, fmt.Errorf("could not get decompression stream: %v", err)
}
l, err := p.config.LayerStore.Register(inflatedLayerData, rootFS.ChainID())
if err != nil {
return false, fmt.Errorf("failed to register layer: %v", err)
}
logrus.Debugf("layer %s registered successfully", l.DiffID())
rootFS.Append(l.DiffID())
// Cache mapping from this layer's DiffID to the blobsum
if err := p.blobSumService.Add(l.DiffID(), d.digest); err != nil {
return false, err
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.digest.String()), "Pull complete", nil))
d.broadcaster.Close()
tagUpdated = true
}
config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), rootFS, history)
if err != nil {
return false, err
}
imageID, err := p.config.ImageStore.Create(config)
if err != nil {
return false, err
}
manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName.Name())
if err != nil {
return false, err
}
// Check for new tag if no layers downloaded
var oldTagImageID image.ID
if !tagUpdated {
oldTagImageID, err = p.config.TagStore.Get(ref)
if err != nil || oldTagImageID != imageID {
tagUpdated = true
}
}
if tagUpdated {
if canonical, ok := ref.(reference.Canonical); ok {
if err = p.config.TagStore.AddDigest(canonical, imageID, true); err != nil {
return false, err
}
} else if err = p.config.TagStore.AddTag(ref, imageID, true); err != nil {
return false, err
}
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
}
return tagUpdated, nil
}
func (daemon *Daemon) getInspectData(container *container.Container, size bool) (*types.ContainerJSONBase, error) {
// make a copy to play with
hostConfig := *container.HostConfig
if children, err := daemon.children(container.Name); err == nil {
for linkAlias, child := range children {
hostConfig.Links = append(hostConfig.Links, fmt.Sprintf("%s:%s", child.Name, linkAlias))
}
}
// we need this trick to preserve empty log driver, so
// container will use daemon defaults even if daemon change them
if hostConfig.LogConfig.Type == "" {
hostConfig.LogConfig.Type = daemon.defaultLogConfig.Type
}
if len(hostConfig.LogConfig.Config) == 0 {
hostConfig.LogConfig.Config = daemon.defaultLogConfig.Config
}
containerState := &types.ContainerState{
Status: container.State.StateString(),
Running: container.State.Running,
Paused: container.State.Paused,
Restarting: container.State.Restarting,
OOMKilled: container.State.OOMKilled,
Dead: container.State.Dead,
Pid: container.State.Pid,
ExitCode: container.State.ExitCode,
Error: container.State.Error,
StartedAt: container.State.StartedAt.Format(time.RFC3339Nano),
FinishedAt: container.State.FinishedAt.Format(time.RFC3339Nano),
}
contJSONBase := &types.ContainerJSONBase{
ID: container.ID,
Created: container.Created.Format(time.RFC3339Nano),
Path: container.Path,
Args: container.Args,
State: containerState,
Image: container.ImageID.String(),
LogPath: container.LogPath,
Name: container.Name,
RestartCount: container.RestartCount,
Driver: container.Driver,
MountLabel: container.MountLabel,
ProcessLabel: container.ProcessLabel,
ExecIDs: container.GetExecIDs(),
HostConfig: &hostConfig,
}
var (
sizeRw int64
sizeRootFs int64
)
if size {
sizeRw, sizeRootFs = daemon.getSize(container)
contJSONBase.SizeRw = &sizeRw
contJSONBase.SizeRootFs = &sizeRootFs
}
// Now set any platform-specific fields
contJSONBase = setPlatformSpecificContainerFields(container, contJSONBase)
contJSONBase.GraphDriver.Name = container.Driver
image, err := daemon.imageStore.Get(container.ImageID)
if err != nil {
return nil, err
}
l, err := daemon.layerStore.Get(image.RootFS.ChainID())
if err != nil {
return nil, err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
graphDriverData, err := l.Metadata()
if err != nil {
return nil, err
}
contJSONBase.GraphDriver.Data = graphDriverData
return contJSONBase, nil
}
// ImportImage imports an image, getting the archived layer data either from
// inConfig (if src is "-"), or from a URI specified in src. Progress output is
// written to outStream. Repository and tag names can optionally be given in
// the repo and tag arguments, respectively.
func (daemon *Daemon) ImportImage(src string, newRef reference.Named, msg string, inConfig io.ReadCloser, outStream io.Writer, config *runconfig.Config) error {
var (
sf = streamformatter.NewJSONStreamFormatter()
archive io.ReadCloser
resp *http.Response
)
if src == "-" {
archive = inConfig
} else {
inConfig.Close()
u, err := url.Parse(src)
if err != nil {
return err
}
if u.Scheme == "" {
u.Scheme = "http"
u.Host = src
u.Path = ""
}
outStream.Write(sf.FormatStatus("", "Downloading from %s", u))
resp, err = httputils.Download(u.String())
if err != nil {
return err
}
progressOutput := sf.NewProgressOutput(outStream, true)
archive = progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Importing")
}
defer archive.Close()
if len(msg) == 0 {
msg = "Imported from " + src
}
// TODO: support windows baselayer?
l, err := daemon.layerStore.Register(archive, "")
if err != nil {
return err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
created := time.Now().UTC()
imgConfig, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Config: config,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Created: created,
Comment: msg,
},
RootFS: &image.RootFS{
Type: "layers",
DiffIDs: []layer.DiffID{l.DiffID()},
},
History: []image.History{{
Created: created,
Comment: msg,
}},
})
if err != nil {
return err
}
id, err := daemon.imageStore.Create(imgConfig)
if err != nil {
return err
}
// FIXME: connect with commit code and call refstore directly
if newRef != nil {
if err := daemon.TagImage(newRef, id.String()); err != nil {
return err
}
}
outStream.Write(sf.FormatStatus("", id.String()))
daemon.EventsService.Log("import", id.String(), "")
return nil
}
func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, imageID image.ID) error {
logrus.Debugf("Pushing repository: %s", ref.String())
img, err := p.config.ImageStore.Get(imageID)
if err != nil {
return fmt.Errorf("could not find image from tag %s: %v", ref.String(), err)
}
var l layer.Layer
topLayerID := img.RootFS.ChainID()
if topLayerID == "" {
l = layer.EmptyLayer
} else {
l, err = p.config.LayerStore.Get(topLayerID)
if err != nil {
return fmt.Errorf("failed to get top layer from image: %v", err)
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
}
var descriptors []xfer.UploadDescriptor
descriptorTemplate := v2PushDescriptor{
v2MetadataService: p.v2MetadataService,
repoInfo: p.repoInfo,
repo: p.repo,
pushState: &p.pushState,
}
// Loop bounds condition is to avoid pushing the base layer on Windows.
for i := 0; i < len(img.RootFS.DiffIDs); i++ {
descriptor := descriptorTemplate
descriptor.layer = l
descriptors = append(descriptors, &descriptor)
l = l.Parent()
}
if err := p.config.UploadManager.Upload(ctx, descriptors, p.config.ProgressOutput); err != nil {
return err
}
// Try schema2 first
builder := schema2.NewManifestBuilder(p.repo.Blobs(ctx), img.RawJSON())
manifest, err := manifestFromBuilder(ctx, builder, descriptors)
if err != nil {
return err
}
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return err
}
putOptions := []distribution.ManifestServiceOption{client.WithTag(ref.Tag())}
if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
logrus.Warnf("failed to upload schema2 manifest: %v - falling back to schema1", err)
manifestRef, err := distreference.WithTag(p.repo.Named(), ref.Tag())
if err != nil {
return err
}
builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, manifestRef, img.RawJSON())
manifest, err = manifestFromBuilder(ctx, builder, descriptors)
if err != nil {
return err
}
if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
return err
}
}
var canonicalManifest []byte
switch v := manifest.(type) {
case *schema1.SignedManifest:
canonicalManifest = v.Canonical
case *schema2.DeserializedManifest:
_, canonicalManifest, err = v.Payload()
if err != nil {
return err
}
}
manifestDigest := digest.FromBytes(canonicalManifest)
progress.Messagef(p.config.ProgressOutput, "", "%s: digest: %s size: %d", ref.Tag(), manifestDigest, len(canonicalManifest))
// Signal digest to the trust client so it can sign the
// push, if appropriate.
progress.Aux(p.config.ProgressOutput, PushResult{Tag: ref.Tag(), Digest: manifestDigest, Size: len(canonicalManifest)})
return nil
}
// Images returns a filtered list of images. filterArgs is a JSON-encoded set
// of filter arguments which will be interpreted by api/types/filters.
// filter is a shell glob string applied to repository names. The argument
// named all controls whether all images in the graph are filtered, or just
// the heads.
func (daemon *Daemon) Images(filterArgs, filter string, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) {
var (
allImages map[image.ID]*image.Image
err error
danglingOnly = false
)
imageFilters, err := filters.FromParam(filterArgs)
if err != nil {
return nil, err
}
if err := imageFilters.Validate(acceptedImageFilterTags); err != nil {
return nil, err
}
if imageFilters.Include("dangling") {
if imageFilters.ExactMatch("dangling", "true") {
danglingOnly = true
} else if !imageFilters.ExactMatch("dangling", "false") {
return nil, fmt.Errorf("Invalid filter 'dangling=%s'", imageFilters.Get("dangling"))
}
}
if danglingOnly {
allImages = daemon.imageStore.Heads()
} else {
allImages = daemon.imageStore.Map()
}
var beforeFilter, sinceFilter *image.Image
err = imageFilters.WalkValues("before", func(value string) error {
beforeFilter, err = daemon.GetImage(value)
return err
})
if err != nil {
return nil, err
}
err = imageFilters.WalkValues("since", func(value string) error {
sinceFilter, err = daemon.GetImage(value)
return err
})
if err != nil {
return nil, err
}
images := []*types.ImageSummary{}
var imagesMap map[*image.Image]*types.ImageSummary
var layerRefs map[layer.ChainID]int
var allLayers map[layer.ChainID]layer.Layer
var allContainers []*container.Container
var filterTagged bool
if filter != "" {
filterRef, err := reference.ParseNamed(filter)
if err == nil { // parse error means wildcard repo
if _, ok := filterRef.(reference.NamedTagged); ok {
filterTagged = true
}
}
}
for id, img := range allImages {
if beforeFilter != nil {
if img.Created.Equal(beforeFilter.Created) || img.Created.After(beforeFilter.Created) {
continue
}
}
if sinceFilter != nil {
if img.Created.Equal(sinceFilter.Created) || img.Created.Before(sinceFilter.Created) {
continue
}
}
if imageFilters.Include("label") {
// Very old image that do not have image.Config (or even labels)
if img.Config == nil {
continue
}
// We are now sure image.Config is not nil
if !imageFilters.MatchKVList("label", img.Config.Labels) {
continue
}
}
layerID := img.RootFS.ChainID()
var size int64
if layerID != "" {
l, err := daemon.layerStore.Get(layerID)
if err != nil {
return nil, err
}
size, err = l.Size()
layer.ReleaseAndLog(daemon.layerStore, l)
if err != nil {
return nil, err
}
}
newImage := newImage(img, size)
for _, ref := range daemon.referenceStore.References(id.Digest()) {
if filter != "" { // filter by tag/repo name
if filterTagged { // filter by tag, require full ref match
if ref.String() != filter {
continue
}
} else if matched, err := path.Match(filter, ref.Name()); !matched || err != nil { // name only match, FIXME: docs say exact
continue
}
}
if _, ok := ref.(reference.Canonical); ok {
newImage.RepoDigests = append(newImage.RepoDigests, ref.String())
}
if _, ok := ref.(reference.NamedTagged); ok {
newImage.RepoTags = append(newImage.RepoTags, ref.String())
}
}
if newImage.RepoDigests == nil && newImage.RepoTags == nil {
if all || len(daemon.imageStore.Children(id)) == 0 {
if imageFilters.Include("dangling") && !danglingOnly {
//dangling=false case, so dangling image is not needed
continue
}
if filter != "" { // skip images with no references if filtering by tag
continue
}
newImage.RepoDigests = []string{"<none>@<none>"}
newImage.RepoTags = []string{"<none>:<none>"}
} else {
continue
}
} else if danglingOnly && len(newImage.RepoTags) > 0 {
continue
}
if withExtraAttrs {
// lazyly init variables
if imagesMap == nil {
allContainers = daemon.List()
allLayers = daemon.layerStore.Map()
imagesMap = make(map[*image.Image]*types.ImageSummary)
layerRefs = make(map[layer.ChainID]int)
}
// Get container count
newImage.Containers = 0
for _, c := range allContainers {
if c.ImageID == id {
newImage.Containers++
}
}
// count layer references
rootFS := *img.RootFS
rootFS.DiffIDs = nil
for _, id := range img.RootFS.DiffIDs {
rootFS.Append(id)
chid := rootFS.ChainID()
layerRefs[chid]++
if _, ok := allLayers[chid]; !ok {
return nil, fmt.Errorf("layer %v was not found (corruption?)", chid)
}
}
imagesMap[img] = newImage
}
images = append(images, newImage)
}
if withExtraAttrs {
// Get Shared and Unique sizes
for img, newImage := range imagesMap {
rootFS := *img.RootFS
rootFS.DiffIDs = nil
newImage.Size = 0
newImage.SharedSize = 0
for _, id := range img.RootFS.DiffIDs {
rootFS.Append(id)
chid := rootFS.ChainID()
diffSize, err := allLayers[chid].DiffSize()
if err != nil {
return nil, err
}
if layerRefs[chid] > 1 {
newImage.SharedSize += diffSize
} else {
newImage.Size += diffSize
}
}
}
}
sort.Sort(sort.Reverse(byCreated(images)))
return images, nil
}
// LookupImage looks up an image by name and returns it as an ImageInspect
// structure.
func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) {
img, err := daemon.GetImage(name)
if err != nil {
return nil, fmt.Errorf("No such image: %s", name)
}
refs := daemon.tagStore.References(img.ID())
repoTags := []string{}
repoDigests := []string{}
for _, ref := range refs {
switch ref.(type) {
case reference.Tagged:
repoTags = append(repoTags, ref.String())
case reference.Digested:
repoDigests = append(repoDigests, ref.String())
}
}
var size int64
var layerMetadata map[string]string
layerID := img.RootFS.ChainID()
if layerID != "" {
l, err := daemon.layerStore.Get(layerID)
if err != nil {
return nil, err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
size, err = l.Size()
if err != nil {
return nil, err
}
layerMetadata, err = l.Metadata()
if err != nil {
return nil, err
}
}
imageInspect := &types.ImageInspect{
ID: img.ID().String(),
RepoTags: repoTags,
RepoDigests: repoDigests,
Parent: img.Parent.String(),
Comment: img.Comment,
Created: img.Created.Format(time.RFC3339Nano),
Container: img.Container,
ContainerConfig: &img.ContainerConfig,
DockerVersion: img.DockerVersion,
Author: img.Author,
Config: img.Config,
Architecture: img.Architecture,
Os: img.OS,
Size: size,
VirtualSize: size, // TODO: field unused, deprecate
}
imageInspect.GraphDriver.Name = daemon.driver.String()
imageInspect.GraphDriver.Data = layerMetadata
return imageInspect, nil
}
// Commit creates a new filesystem image from the current state of a container.
// The image can optionally be tagged into a repository.
func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (string, error) {
start := time.Now()
container, err := daemon.GetContainer(name)
if err != nil {
return "", err
}
// It is not possible to commit a running container on Windows and on Solaris.
if (runtime.GOOS == "windows" || runtime.GOOS == "solaris") && container.IsRunning() {
return "", fmt.Errorf("%+v does not support commit of a running container", runtime.GOOS)
}
if c.Pause && !container.IsPaused() {
daemon.containerPause(container)
defer daemon.containerUnpause(container)
}
newConfig, err := dockerfile.BuildFromConfig(c.Config, c.Changes)
if err != nil {
return "", err
}
if c.MergeConfigs {
if err := merge(newConfig, container.Config); err != nil {
return "", err
}
}
rwTar, err := daemon.exportContainerRw(container)
if err != nil {
return "", err
}
defer func() {
if rwTar != nil {
rwTar.Close()
}
}()
var history []image.History
rootFS := image.NewRootFS()
osVersion := ""
var osFeatures []string
if container.ImageID != "" {
img, err := daemon.imageStore.Get(container.ImageID)
if err != nil {
return "", err
}
history = img.History
rootFS = img.RootFS
osVersion = img.OSVersion
osFeatures = img.OSFeatures
}
l, err := daemon.layerStore.Register(rwTar, rootFS.ChainID())
if err != nil {
return "", err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
h := image.History{
Author: c.Author,
Created: time.Now().UTC(),
CreatedBy: strings.Join(container.Config.Cmd, " "),
Comment: c.Comment,
EmptyLayer: true,
}
if diffID := l.DiffID(); layer.DigestSHA256EmptyTar != diffID {
h.EmptyLayer = false
rootFS.Append(diffID)
}
history = append(history, h)
config, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Config: newConfig,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Container: container.ID,
ContainerConfig: *container.Config,
Author: c.Author,
Created: h.Created,
},
RootFS: rootFS,
History: history,
OSFeatures: osFeatures,
OSVersion: osVersion,
})
if err != nil {
return "", err
}
id, err := daemon.imageStore.Create(config)
if err != nil {
return "", err
}
if container.ImageID != "" {
if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil {
return "", err
}
}
imageRef := ""
if c.Repo != "" {
newTag, err := reference.WithName(c.Repo) // todo: should move this to API layer
if err != nil {
return "", err
}
if c.Tag != "" {
if newTag, err = reference.WithTag(newTag, c.Tag); err != nil {
return "", err
}
}
if err := daemon.TagImageWithReference(id, newTag); err != nil {
return "", err
}
imageRef = newTag.String()
}
attributes := map[string]string{
"comment": c.Comment,
"imageID": id.String(),
"imageRef": imageRef,
}
daemon.LogContainerEventWithAttributes(container, "commit", attributes)
containerActions.WithValues("commit").UpdateSince(start)
return id.String(), nil
}
// makeDownloadFunc returns a function that performs the layer download and
// registration. If parentDownload is non-nil, it waits for that download to
// complete before the registration step, and registers the downloaded data
// on top of parentDownload's resulting layer. Otherwise, it registers the
// layer on top of the ChainID given by parentLayer.
func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor, parentLayer layer.ChainID, parentDownload *downloadTransfer) DoFunc {
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
d := &downloadTransfer{
Transfer: NewTransfer(),
layerStore: ldm.layerStore,
}
go func() {
defer func() {
close(progressChan)
}()
progressOutput := progress.ChanOutput(progressChan)
select {
case <-start:
default:
progress.Update(progressOutput, descriptor.ID(), "Waiting")
<-start
}
if parentDownload != nil {
// Did the parent download already fail or get
// cancelled?
select {
case <-parentDownload.Done():
_, err := parentDownload.result()
if err != nil {
d.err = err
return
}
default:
}
}
var (
downloadReader io.ReadCloser
size int64
err error
retries int
)
defer descriptor.Close()
for {
downloadReader, size, err = descriptor.Download(d.Transfer.Context(), progressOutput)
if err == nil {
break
}
// If an error was returned because the context
// was cancelled, we shouldn't retry.
select {
case <-d.Transfer.Context().Done():
d.err = err
return
default:
}
retries++
if _, isDNR := err.(DoNotRetry); isDNR || retries == maxDownloadAttempts {
logrus.Errorf("Download failed: %v", err)
d.err = err
return
}
logrus.Errorf("Download failed, retrying: %v", err)
delay := retries * 5
ticker := time.NewTicker(time.Second)
selectLoop:
for {
progress.Updatef(progressOutput, descriptor.ID(), "Retrying in %d second%s", delay, (map[bool]string{true: "s"})[delay != 1])
select {
case <-ticker.C:
delay--
if delay == 0 {
ticker.Stop()
break selectLoop
}
case <-d.Transfer.Context().Done():
ticker.Stop()
d.err = errors.New("download cancelled during retry delay")
return
}
}
}
close(inactive)
if parentDownload != nil {
select {
case <-d.Transfer.Context().Done():
d.err = errors.New("layer registration cancelled")
downloadReader.Close()
return
case <-parentDownload.Done():
}
l, err := parentDownload.result()
if err != nil {
d.err = err
downloadReader.Close()
return
}
parentLayer = l.ChainID()
}
reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(d.Transfer.Context(), downloadReader), progressOutput, size, descriptor.ID(), "Extracting")
defer reader.Close()
inflatedLayerData, err := archive.DecompressStream(reader)
if err != nil {
d.err = fmt.Errorf("could not get decompression stream: %v", err)
return
}
var src distribution.Descriptor
if fs, ok := descriptor.(distribution.Describable); ok {
src = fs.Descriptor()
}
if ds, ok := d.layerStore.(layer.DescribableStore); ok {
d.layer, err = ds.RegisterWithDescriptor(inflatedLayerData, parentLayer, src)
} else {
d.layer, err = d.layerStore.Register(inflatedLayerData, parentLayer)
}
if err != nil {
select {
case <-d.Transfer.Context().Done():
d.err = errors.New("layer registration cancelled")
default:
d.err = fmt.Errorf("failed to register layer: %v", err)
}
return
}
progress.Update(progressOutput, descriptor.ID(), "Pull complete")
withRegistered, hasRegistered := descriptor.(DownloadDescriptorWithRegistered)
if hasRegistered {
withRegistered.Registered(d.layer.DiffID())
}
// Doesn't actually need to be its own goroutine, but
// done like this so we can defer close(c).
go func() {
<-d.Transfer.Released()
if d.layer != nil {
layer.ReleaseAndLog(d.layerStore, d.layer)
}
}()
}()
return d
}
}
func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer) error {
tmpDir, err := ioutil.TempDir("", "docker-import-")
if err != nil {
return err
}
defer os.RemoveAll(tmpDir)
if err := chrootarchive.Untar(inTar, tmpDir, nil); err != nil {
return err
}
// read manifest, if no file then load in legacy mode
manifestPath, err := safePath(tmpDir, manifestFileName)
if err != nil {
return err
}
manifestFile, err := os.Open(manifestPath)
if err != nil {
if os.IsNotExist(err) {
return l.legacyLoad(tmpDir, outStream)
}
return manifestFile.Close()
}
defer manifestFile.Close()
var manifest []manifestItem
if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil {
return err
}
for _, m := range manifest {
configPath, err := safePath(tmpDir, m.Config)
if err != nil {
return err
}
config, err := ioutil.ReadFile(configPath)
if err != nil {
return err
}
img, err := image.NewFromJSON(config)
if err != nil {
return err
}
var rootFS image.RootFS
rootFS = *img.RootFS
rootFS.DiffIDs = nil
if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual {
return fmt.Errorf("invalid manifest, layers length mismatch: expected %q, got %q", expected, actual)
}
for i, diffID := range img.RootFS.DiffIDs {
layerPath, err := safePath(tmpDir, m.Layers[i])
if err != nil {
return err
}
newLayer, err := l.loadLayer(layerPath, rootFS)
if err != nil {
return err
}
defer layer.ReleaseAndLog(l.ls, newLayer)
if expected, actual := diffID, newLayer.DiffID(); expected != actual {
return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
}
rootFS.Append(diffID)
}
imgID, err := l.is.Create(config)
if err != nil {
return err
}
for _, repoTag := range m.RepoTags {
named, err := reference.ParseNamed(repoTag)
if err != nil {
return err
}
ref, ok := named.(reference.NamedTagged)
if !ok {
return fmt.Errorf("invalid tag %q", repoTag)
}
l.setLoadedTag(ref, imgID, outStream)
}
}
return nil
}
// Download is a blocking function which ensures the requested layers are
// present in the layer store. It uses the string returned by the Key method to
// deduplicate downloads. If a given layer is not already known to present in
// the layer store, and the key is not used by an in-progress download, the
// Download method is called to get the layer tar data. Layers are then
// registered in the appropriate order. The caller must call the returned
// release function once it is is done with the returned RootFS object.
func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS image.RootFS, layers []DownloadDescriptor, progressOutput progress.Output) (image.RootFS, func(), error) {
var (
topLayer layer.Layer
topDownload *downloadTransfer
watcher *Watcher
missingLayer bool
transferKey = ""
downloadsByKey = make(map[string]*downloadTransfer)
)
rootFS := initialRootFS
for _, descriptor := range layers {
key := descriptor.Key()
transferKey += key
if !missingLayer {
missingLayer = true
diffID, err := descriptor.DiffID()
if err == nil {
getRootFS := rootFS
getRootFS.Append(diffID)
l, err := ldm.layerStore.Get(getRootFS.ChainID())
if err == nil {
// Layer already exists.
logrus.Debugf("Layer already exists: %s", descriptor.ID())
progress.Update(progressOutput, descriptor.ID(), "Already exists")
if topLayer != nil {
layer.ReleaseAndLog(ldm.layerStore, topLayer)
}
topLayer = l
missingLayer = false
rootFS.Append(diffID)
continue
}
}
}
// Does this layer have the same data as a previous layer in
// the stack? If so, avoid downloading it more than once.
var topDownloadUncasted Transfer
if existingDownload, ok := downloadsByKey[key]; ok {
xferFunc := ldm.makeDownloadFuncFromDownload(descriptor, existingDownload, topDownload)
defer topDownload.Transfer.Release(watcher)
topDownloadUncasted, watcher = ldm.tm.Transfer(transferKey, xferFunc, progressOutput)
topDownload = topDownloadUncasted.(*downloadTransfer)
continue
}
// Layer is not known to exist - download and register it.
progress.Update(progressOutput, descriptor.ID(), "Pulling fs layer")
var xferFunc DoFunc
if topDownload != nil {
xferFunc = ldm.makeDownloadFunc(descriptor, "", topDownload)
defer topDownload.Transfer.Release(watcher)
} else {
xferFunc = ldm.makeDownloadFunc(descriptor, rootFS.ChainID(), nil)
}
topDownloadUncasted, watcher = ldm.tm.Transfer(transferKey, xferFunc, progressOutput)
topDownload = topDownloadUncasted.(*downloadTransfer)
downloadsByKey[key] = topDownload
}
if topDownload == nil {
return rootFS, func() {
if topLayer != nil {
layer.ReleaseAndLog(ldm.layerStore, topLayer)
}
}, nil
}
// Won't be using the list built up so far - will generate it
// from downloaded layers instead.
rootFS.DiffIDs = []layer.DiffID{}
defer func() {
if topLayer != nil {
layer.ReleaseAndLog(ldm.layerStore, topLayer)
}
}()
select {
case <-ctx.Done():
topDownload.Transfer.Release(watcher)
return rootFS, func() {}, ctx.Err()
case <-topDownload.Done():
break
}
l, err := topDownload.result()
if err != nil {
topDownload.Transfer.Release(watcher)
return rootFS, func() {}, err
}
// Must do this exactly len(layers) times, so we don't include the
// base layer on Windows.
for range layers {
if l == nil {
topDownload.Transfer.Release(watcher)
return rootFS, func() {}, errors.New("internal error: too few parent layers")
}
rootFS.DiffIDs = append([]layer.DiffID{l.DiffID()}, rootFS.DiffIDs...)
l = l.Parent()
}
return rootFS, func() { topDownload.Transfer.Release(watcher) }, err
}
// Images returns a filtered list of images. filterArgs is a JSON-encoded set
// of filter arguments which will be interpreted by pkg/parsers/filters.
// filter is a shell glob string applied to repository names. The argument
// named all controls whether all images in the graph are filtered, or just
// the heads.
func (daemon *Daemon) Images(filterArgs, filter string, all bool) ([]*types.Image, error) {
var (
allImages map[image.ID]*image.Image
err error
danglingOnly = false
)
imageFilters, err := filters.FromParam(filterArgs)
if err != nil {
return nil, err
}
for name := range imageFilters {
if _, ok := acceptedImageFilterTags[name]; !ok {
return nil, fmt.Errorf("Invalid filter '%s'", name)
}
}
if i, ok := imageFilters["dangling"]; ok {
for _, value := range i {
if v := strings.ToLower(value); v == "true" {
danglingOnly = true
} else if v != "false" {
return nil, fmt.Errorf("Invalid filter 'dangling=%s'", v)
}
}
}
if danglingOnly {
allImages = daemon.imageStore.Heads()
} else {
allImages = daemon.imageStore.Map()
}
images := []*types.Image{}
var filterTagged bool
if filter != "" {
filterRef, err := reference.Parse(filter)
if err == nil { // parse error means wildcard repo
if _, ok := filterRef.(reference.Tagged); ok {
filterTagged = true
}
}
}
for id, img := range allImages {
if _, ok := imageFilters["label"]; ok {
if img.Config == nil {
// Very old image that do not have image.Config (or even labels)
continue
}
// We are now sure image.Config is not nil
if !imageFilters.MatchKVList("label", img.Config.Labels) {
continue
}
}
layerID := img.RootFS.ChainID()
var size int64
if layerID != "" {
l, err := daemon.layerStore.Get(layerID)
if err != nil {
return nil, err
}
size, err = l.Size()
layer.ReleaseAndLog(daemon.layerStore, l)
if err != nil {
return nil, err
}
}
newImage := newImage(img, size)
for _, ref := range daemon.tagStore.References(id) {
if filter != "" { // filter by tag/repo name
if filterTagged { // filter by tag, require full ref match
if ref.String() != filter {
continue
}
} else if matched, err := path.Match(filter, ref.Name()); !matched || err != nil { // name only match, FIXME: docs say exact
continue
}
}
if _, ok := ref.(reference.Digested); ok {
newImage.RepoDigests = append(newImage.RepoDigests, ref.String())
}
if _, ok := ref.(reference.Tagged); ok {
newImage.RepoTags = append(newImage.RepoTags, ref.String())
}
}
if newImage.RepoDigests == nil && newImage.RepoTags == nil {
if all || len(daemon.imageStore.Children(id)) == 0 {
if filter != "" { // skip images with no references if filtering by tag
continue
}
newImage.RepoDigests = []string{"<none>@<none>"}
newImage.RepoTags = []string{"<none>:<none>"}
} else {
continue
}
} else if danglingOnly {
continue
}
images = append(images, newImage)
}
sort.Sort(sort.Reverse(byCreated(images)))
return images, nil
}
// ImportImage imports an image, getting the archived layer data either from
// inConfig (if src is "-"), or from a URI specified in src. Progress output is
// written to outStream. Repository and tag names can optionally be given in
// the repo and tag arguments, respectively.
func (daemon *Daemon) ImportImage(src string, repository, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error {
var (
sf = streamformatter.NewJSONStreamFormatter()
rc io.ReadCloser
resp *http.Response
newRef reference.Named
)
if repository != "" {
var err error
newRef, err = reference.ParseNamed(repository)
if err != nil {
return err
}
if _, isCanonical := newRef.(reference.Canonical); isCanonical {
return errors.New("cannot import digest reference")
}
if tag != "" {
newRef, err = reference.WithTag(newRef, tag)
if err != nil {
return err
}
}
}
config, err := dockerfile.BuildFromConfig(&container.Config{}, changes)
if err != nil {
return err
}
if src == "-" {
rc = inConfig
} else {
inConfig.Close()
u, err := url.Parse(src)
if err != nil {
return err
}
if u.Scheme == "" {
u.Scheme = "http"
u.Host = src
u.Path = ""
}
outStream.Write(sf.FormatStatus("", "Downloading from %s", u))
resp, err = httputils.Download(u.String())
if err != nil {
return err
}
progressOutput := sf.NewProgressOutput(outStream, true)
rc = progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Importing")
}
defer rc.Close()
if len(msg) == 0 {
msg = "Imported from " + src
}
inflatedLayerData, err := archive.DecompressStream(rc)
if err != nil {
return err
}
// TODO: support windows baselayer?
l, err := daemon.layerStore.Register(inflatedLayerData, "")
if err != nil {
return err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
created := time.Now().UTC()
imgConfig, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Config: config,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Created: created,
Comment: msg,
},
RootFS: &image.RootFS{
Type: "layers",
DiffIDs: []layer.DiffID{l.DiffID()},
},
History: []image.History{{
Created: created,
Comment: msg,
}},
})
if err != nil {
return err
}
id, err := daemon.imageStore.Create(imgConfig)
if err != nil {
return err
}
// FIXME: connect with commit code and call refstore directly
if newRef != nil {
if err := daemon.TagImageWithReference(id, newRef); err != nil {
return err
}
}
daemon.LogImageEvent(id.String(), id.String(), "import")
outStream.Write(sf.FormatStatus("", id.String()))
return nil
}
func (l *storeLayer) Release() {
if l.ls != nil {
layer.ReleaseAndLog(l.ls, l.Layer)
}
}