func (p *v2Pusher) getImageTags(askedTag string) ([]string, error) {
logrus.Debugf("Checking %q against %#v", askedTag, p.localRepo)
if len(askedTag) > 0 {
if _, ok := p.localRepo[askedTag]; !ok || utils.DigestReference(askedTag) {
return nil, fmt.Errorf("Tag does not exist for %s", askedTag)
}
return []string{askedTag}, nil
}
var tags []string
for tag := range p.localRepo {
if !utils.DigestReference(tag) {
tags = append(tags, tag)
}
}
return tags, nil
}
func (s *TagStore) getImageTags(localRepo map[string]string, askedTag string) ([]string, error) {
log.Debugf("Checking %s against %#v", askedTag, localRepo)
if len(askedTag) > 0 {
if _, ok := localRepo[askedTag]; !ok || utils.DigestReference(askedTag) {
return nil, fmt.Errorf("Tag does not exist: %s", askedTag)
}
return []string{askedTag}, nil
}
var tags []string
for tag := range localRepo {
if !utils.DigestReference(tag) {
tags = append(tags, tag)
}
}
return tags, nil
}
func (p *v1Puller) Pull(tag string) (fallback bool, err error) {
if utils.DigestReference(tag) {
// Allowing fallback, because HTTPS v1 is before HTTP v2
return true, registry.ErrNoSupport{errors.New("Cannot pull by digest with v1 registry")}
}
tlsConfig, err := p.registryService.TLSConfig(p.repoInfo.Index.Name)
if err != nil {
return false, err
}
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
tr := transport.NewTransport(
// TODO(tiborvass): was ReceiveTimeout
registry.NewTransport(tlsConfig),
registry.DockerHeaders(p.config.MetaHeaders)...,
)
client := registry.HTTPClient(tr)
v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
if err != nil {
logrus.Debugf("Could not get v1 endpoint: %v", err)
return true, err
}
p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
if err != nil {
// TODO(dmcgowan): Check if should fallback
logrus.Debugf("Fallback from error: %s", err)
return true, err
}
if err := p.pullRepository(tag); err != nil {
// TODO(dmcgowan): Check if should fallback
return false, err
}
return false, nil
}
// Retrieve the all the images to be uploaded in the correct order
func (p *v1Pusher) getImageList(requestedTag string) ([]string, map[string][]string, error) {
var (
imageList []string
imagesSeen = make(map[string]bool)
tagsByImage = make(map[string][]string)
)
for tag, id := range p.localRepo {
if requestedTag != "" && requestedTag != tag {
// Include only the requested tag.
continue
}
if utils.DigestReference(tag) {
// Ignore digest references.
continue
}
var imageListForThisTag []string
tagsByImage[id] = append(tagsByImage[id], tag)
for img, err := p.graph.Get(id); img != nil; img, err = p.graph.GetParent(img) {
if err != nil {
return nil, nil, err
}
if imagesSeen[img.ID] {
// This image is already on the list, we can ignore it and all its parents
break
}
imagesSeen[img.ID] = true
imageListForThisTag = append(imageListForThisTag, img.ID)
}
// reverse the image list for this tag (so the "most"-parent image is first)
for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 {
imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i]
}
// append to main image list
imageList = append(imageList, imageListForThisTag...)
}
if len(imageList) == 0 {
return nil, nil, fmt.Errorf("No images found for the requested repository / tag")
}
logrus.Debugf("Image list: %v", imageList)
logrus.Debugf("Tags by image: %v", tagsByImage)
return imageList, tagsByImage, nil
}
// Lookup looks up an image by name in a TagStore and returns it as an
// ImageInspect structure.
func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) {
image, err := s.LookupImage(name)
if err != nil || image == nil {
return nil, fmt.Errorf("No such image: %s", name)
}
var repoTags = make([]string, 0)
var repoDigests = make([]string, 0)
s.Lock()
for repoName, repository := range s.Repositories {
for ref, id := range repository {
if id == image.ID {
imgRef := utils.ImageReference(repoName, ref)
if utils.DigestReference(ref) {
repoDigests = append(repoDigests, imgRef)
} else {
repoTags = append(repoTags, imgRef)
}
}
}
}
s.Unlock()
imageInspect := &types.ImageInspect{
ID: image.ID,
RepoTags: repoTags,
RepoDigests: repoDigests,
Parent: image.Parent,
Comment: image.Comment,
Created: image.Created.Format(time.RFC3339Nano),
Container: image.Container,
ContainerConfig: &image.ContainerConfig,
DockerVersion: image.DockerVersion,
Author: image.Author,
Config: image.Config,
Architecture: image.Architecture,
Os: image.OS,
Size: image.Size,
VirtualSize: s.graph.GetParentsSize(image) + image.Size,
}
imageInspect.GraphDriver.Name = s.graph.driver.String()
graphDriverData, err := s.graph.driver.GetMetadata(image.ID)
if err != nil {
return nil, err
}
imageInspect.GraphDriver.Data = graphDriverData
return imageInspect, nil
}
func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (verified bool, err error) {
logrus.Debugf("Pulling tag from V2 registry: %q", tag)
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return false, err
}
manifest, err := manSvc.GetByTag(tag)
if err != nil {
return false, err
}
verified, err = p.validateManifest(manifest, tag)
if err != nil {
return false, err
}
if verified {
logrus.Printf("Image manifest for %s has been verified", taggedName)
}
out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
var downloads []*downloadInfo
var layerIDs []string
defer func() {
p.graph.Release(p.sessionID, layerIDs...)
for _, d := range downloads {
p.poolRemoveWithError("pull", d.poolKey, err)
if d.tmpFile != nil {
d.tmpFile.Close()
if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
}
}
}
}()
for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility))
if err != nil {
logrus.Debugf("error getting image v1 json: %v", err)
return false, err
}
p.graph.Retain(p.sessionID, img.ID)
layerIDs = append(layerIDs, img.ID)
// Check if exists
if p.graph.Exists(img.ID) {
logrus.Debugf("Image already exists: %s", img.ID)
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Already exists", nil))
continue
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil))
d := &downloadInfo{
img: img,
poolKey: "layer:" + img.ID,
digest: manifest.FSLayers[i].BlobSum,
// TODO: seems like this chan buffer solved hanging problem in go1.5,
// this can indicate some deeper problem that somehow we never take
// error from channel in loop below
err: make(chan error, 1),
}
tmpFile, err := ioutil.TempFile("", "GetImageBlob")
if err != nil {
return false, err
}
d.tmpFile = tmpFile
downloads = append(downloads, d)
broadcaster, found := p.poolAdd("pull", d.poolKey)
broadcaster.Add(out)
d.broadcaster = broadcaster
if found {
d.err <- nil
} else {
go p.download(d)
}
}
var tagUpdated bool
for _, d := range downloads {
if err := <-d.err; err != nil {
return false, err
}
if d.layer == nil {
// Wait for a different pull to download and extract
// this layer.
err = d.broadcaster.Wait()
if err != nil {
return false, err
}
continue
}
d.tmpFile.Seek(0, 0)
reader := progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: d.broadcaster,
Formatter: p.sf,
Size: d.size,
NewLines: false,
ID: stringid.TruncateID(d.img.ID),
Action: "Extracting",
})
err = p.graph.Register(d.img, reader)
if err != nil {
return false, err
}
if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil {
return false, err
}
d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil))
d.broadcaster.Close()
tagUpdated = true
}
manifestDigest, _, err := digestFromManifest(manifest, p.repoInfo.LocalName)
if err != nil {
return false, err
}
// Check for new tag if no layers downloaded
if !tagUpdated {
repo, err := p.Get(p.repoInfo.LocalName)
if err != nil {
return false, err
}
if repo != nil {
if _, exists := repo[tag]; !exists {
tagUpdated = true
}
} else {
tagUpdated = true
}
}
if verified && tagUpdated {
out.Write(p.sf.FormatStatus(p.repo.Name()+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security."))
}
firstID := layerIDs[len(layerIDs)-1]
if utils.DigestReference(tag) {
// TODO(stevvooe): Ideally, we should always set the digest so we can
// use the digest whether we pull by it or not. Unfortunately, the tag
// store treats the digest as a separate tag, meaning there may be an
// untagged digest image that would seem to be dangling by a user.
if err = p.SetDigest(p.repoInfo.LocalName, tag, firstID); err != nil {
return false, err
}
} else {
// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
if err = p.Tag(p.repoInfo.LocalName, tag, firstID, true); err != nil {
return false, err
}
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
}
return tagUpdated, nil
}
func (s *TagStore) pullV2Tag(r *registry.Session, out io.Writer, endpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo, tag string, sf *streamformatter.StreamFormatter, auth *registry.RequestAuthorization) (bool, error) {
logrus.Debugf("Pulling tag from V2 registry: %q", tag)
remoteDigest, manifestBytes, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth)
if err != nil {
return false, err
}
// loadManifest ensures that the manifest payload has the expected digest
// if the tag is a digest reference.
localDigest, manifest, verified, err := s.loadManifest(manifestBytes, tag, remoteDigest)
if err != nil {
return false, fmt.Errorf("error verifying manifest: %s", err)
}
if verified {
logrus.Printf("Image manifest for %s has been verified", utils.ImageReference(repoInfo.CanonicalName, tag))
}
out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName))
// downloadInfo is used to pass information from download to extractor
type downloadInfo struct {
imgJSON []byte
img *image.Image
digest digest.Digest
tmpFile *os.File
length int64
downloaded bool
err chan error
}
downloads := make([]downloadInfo, len(manifest.FSLayers))
for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
var (
sumStr = manifest.FSLayers[i].BlobSum
imgJSON = []byte(manifest.History[i].V1Compatibility)
)
img, err := image.NewImgJSON(imgJSON)
if err != nil {
return false, fmt.Errorf("failed to parse json: %s", err)
}
downloads[i].img = img
// Check if exists
if s.graph.Exists(img.ID) {
logrus.Debugf("Image already exists: %s", img.ID)
continue
}
dgst, err := digest.ParseDigest(sumStr)
if err != nil {
return false, err
}
downloads[i].digest = dgst
out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil))
downloadFunc := func(di *downloadInfo) error {
logrus.Debugf("pulling blob %q to V1 img %s", sumStr, img.ID)
if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil {
if c != nil {
out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil))
<-c
out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
} else {
logrus.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
}
} else {
defer s.poolRemove("pull", "img:"+img.ID)
tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob")
if err != nil {
return err
}
r, l, err := r.GetV2ImageBlobReader(endpoint, repoInfo.RemoteName, di.digest, auth)
if err != nil {
return err
}
defer r.Close()
verifier, err := digest.NewDigestVerifier(di.digest)
if err != nil {
return err
}
if _, err := io.Copy(tmpFile, progressreader.New(progressreader.Config{
In: ioutil.NopCloser(io.TeeReader(r, verifier)),
Out: out,
Formatter: sf,
Size: int(l),
NewLines: false,
ID: stringid.TruncateID(img.ID),
Action: "Downloading",
})); err != nil {
return fmt.Errorf("unable to copy v2 image blob data: %s", err)
}
out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Verifying Checksum", nil))
if !verifier.Verified() {
return fmt.Errorf("image layer digest verification failed for %q", di.digest)
}
out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
logrus.Debugf("Downloaded %s to tempfile %s", img.ID, tmpFile.Name())
di.tmpFile = tmpFile
di.length = l
di.downloaded = true
}
di.imgJSON = imgJSON
return nil
}
downloads[i].err = make(chan error)
go func(di *downloadInfo) {
di.err <- downloadFunc(di)
}(&downloads[i])
}
var tagUpdated bool
for i := len(downloads) - 1; i >= 0; i-- {
d := &downloads[i]
if d.err != nil {
if err := <-d.err; err != nil {
return false, err
}
}
if d.downloaded {
// if tmpFile is empty assume download and extracted elsewhere
defer os.Remove(d.tmpFile.Name())
defer d.tmpFile.Close()
d.tmpFile.Seek(0, 0)
if d.tmpFile != nil {
err = s.graph.Register(d.img,
progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: out,
Formatter: sf,
Size: int(d.length),
ID: stringid.TruncateID(d.img.ID),
Action: "Extracting",
}))
if err != nil {
return false, err
}
if err := s.graph.SetDigest(d.img.ID, d.digest); err != nil {
return false, err
}
// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
}
out.Write(sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil))
tagUpdated = true
} else {
out.Write(sf.FormatProgress(stringid.TruncateID(d.img.ID), "Already exists", nil))
}
}
// Check for new tag if no layers downloaded
if !tagUpdated {
repo, err := s.Get(repoInfo.LocalName)
if err != nil {
return false, err
}
if repo != nil {
if _, exists := repo[tag]; !exists {
tagUpdated = true
}
} else {
tagUpdated = true
}
}
if verified && tagUpdated {
out.Write(sf.FormatStatus(utils.ImageReference(repoInfo.CanonicalName, tag), "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security."))
}
if localDigest != remoteDigest { // this is not a verification check.
// NOTE(stevvooe): This is a very defensive branch and should never
// happen, since all manifest digest implementations use the same
// algorithm.
logrus.WithFields(
logrus.Fields{
"local": localDigest,
"remote": remoteDigest,
}).Debugf("local digest does not match remote")
out.Write(sf.FormatStatus("", "Remote Digest: %s", remoteDigest))
}
out.Write(sf.FormatStatus("", "Digest: %s", localDigest))
if tag == localDigest.String() {
// TODO(stevvooe): Ideally, we should always set the digest so we can
// use the digest whether we pull by it or not. Unfortunately, the tag
// store treats the digest as a separate tag, meaning there may be an
// untagged digest image that would seem to be dangling by a user.
if err = s.SetDigest(repoInfo.LocalName, localDigest.String(), downloads[0].img.ID); err != nil {
return false, err
}
}
if !utils.DigestReference(tag) {
// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
if err = s.Tag(repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil {
return false, err
}
}
return tagUpdated, nil
}
func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConfig) error {
var (
sf = streamformatter.NewJSONStreamFormatter()
)
// Resolve the Repository name from fqn to RepositoryInfo
repoInfo, err := s.registryService.ResolveRepository(image)
if err != nil {
return err
}
if err := validateRepoName(repoInfo.LocalName); err != nil {
return err
}
c, err := s.poolAdd("pull", utils.ImageReference(repoInfo.LocalName, tag))
if err != nil {
if c != nil {
// Another pull of the same repository is already taking place; just wait for it to finish
imagePullConfig.OutStream.Write(sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", repoInfo.LocalName))
<-c
return nil
}
return err
}
defer s.poolRemove("pull", utils.ImageReference(repoInfo.LocalName, tag))
logName := repoInfo.LocalName
if tag != "" {
logName = utils.ImageReference(logName, tag)
}
// Attempt pulling official content from a provided v2 mirror
if repoInfo.Index.Official {
v2mirrorEndpoint, v2mirrorRepoInfo, err := configureV2Mirror(repoInfo, s.registryService)
if err != nil {
logrus.Errorf("Error configuring mirrors: %s", err)
return err
}
if v2mirrorEndpoint != nil {
logrus.Debugf("Attempting to pull from v2 mirror: %s", v2mirrorEndpoint.URL)
return s.pullFromV2Mirror(v2mirrorEndpoint, v2mirrorRepoInfo, imagePullConfig, tag, sf, logName)
}
}
logrus.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName)
endpoint, err := repoInfo.GetEndpoint(imagePullConfig.MetaHeaders)
if err != nil {
return err
}
// TODO(tiborvass): reuse client from endpoint?
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
tr := transport.NewTransport(
registry.NewTransport(registry.ReceiveTimeout, endpoint.IsSecure),
registry.DockerHeaders(imagePullConfig.MetaHeaders)...,
)
client := registry.HTTPClient(tr)
r, err := registry.NewSession(client, imagePullConfig.AuthConfig, endpoint)
if err != nil {
return err
}
if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) {
if repoInfo.Official {
s.trustService.UpdateBase()
}
logrus.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName)
if err := s.pullV2Repository(r, imagePullConfig.OutStream, repoInfo, tag, sf); err == nil {
s.eventsService.Log("pull", logName, "")
return nil
} else if err != registry.ErrDoesNotExist && err != ErrV2RegistryUnavailable {
logrus.Errorf("Error from V2 registry: %s", err)
}
logrus.Debug("image does not exist on v2 registry, falling back to v1")
}
if utils.DigestReference(tag) {
return fmt.Errorf("pulling with digest reference failed from v2 registry")
}
logrus.Debugf("pulling v1 repository with local name %q", repoInfo.LocalName)
if err = s.pullRepository(r, imagePullConfig.OutStream, repoInfo, tag, sf); err != nil {
return err
}
s.eventsService.Log("pull", logName, "")
return nil
}
func (p *v2Puller) pullV2Tag(tag, taggedName string) (tagUpdated bool, err error) {
logrus.Debugf("Pulling tag from V2 registry: %q", tag)
out := p.config.OutStream
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return false, err
}
unverifiedManifest, err := manSvc.GetByTag(tag)
if err != nil {
return false, err
}
if unverifiedManifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag %q", tag)
}
var verifiedManifest *manifest.Manifest
verifiedManifest, err = verifyManifest(unverifiedManifest, tag)
if err != nil {
return false, err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return false, err
}
imgs, err := p.getImageInfos(*verifiedManifest)
if err != nil {
return false, err
}
// By using a pipeWriter for each of the downloads to write their progress
// to, we can avoid an issue where this function returns an error but
// leaves behind running download goroutines. By splitting the writer
// with a pipe, we can close the pipe if there is any error, consequently
// causing each download to cancel due to an error writing to this pipe.
pipeReader, pipeWriter := io.Pipe()
go func() {
if _, err := io.Copy(out, pipeReader); err != nil {
logrus.Errorf("error copying from layer download progress reader: %s", err)
if err := pipeReader.CloseWithError(err); err != nil {
logrus.Errorf("error closing the progress reader: %s", err)
}
}
}()
defer func() {
if err != nil {
// All operations on the pipe are synchronous. This call will wait
// until all current readers/writers are done using the pipe then
// set the error. All successive reads/writes will return with this
// error.
pipeWriter.CloseWithError(fmt.Errorf("download canceled %+v", err))
} else {
// If no error then just close the pipe.
pipeWriter.Close()
}
}()
out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
downloads := make([]downloadInfo, len(verifiedManifest.FSLayers))
layerIDs := []string{}
defer func() {
p.graph.Release(p.sessionID, layerIDs...)
}()
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
img := imgs[i]
downloads[i].img = img
downloads[i].digest = verifiedManifest.FSLayers[i].BlobSum
p.graph.Retain(p.sessionID, img.id)
layerIDs = append(layerIDs, img.id)
p.graph.imageMutex.Lock(img.id)
// Check if exists
if p.graph.Exists(img.id) {
if err := p.validateImageInGraph(img.id, imgs, i); err != nil {
p.graph.imageMutex.Unlock(img.id)
return false, fmt.Errorf("image validation failed: %v", err)
}
logrus.Debugf("Image already exists: %s", img.id)
p.graph.imageMutex.Unlock(img.id)
continue
}
p.graph.imageMutex.Unlock(img.id)
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.id), "Pulling fs layer", nil))
downloads[i].err = make(chan error, 1)
downloads[i].out = pipeWriter
go p.download(&downloads[i])
}
for i := len(downloads) - 1; i >= 0; i-- {
d := &downloads[i]
if d.err != nil {
if err := <-d.err; err != nil {
return false, err
}
}
if d.layer != nil {
// if tmpFile is empty assume download and extracted elsewhere
defer os.Remove(d.tmpFile.Name())
defer d.tmpFile.Close()
d.tmpFile.Seek(0, 0)
if d.tmpFile != nil {
err := func() error {
reader := progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: out,
Formatter: p.sf,
Size: int(d.size),
NewLines: false,
ID: stringid.TruncateID(d.img.id),
Action: "Extracting",
})
p.graph.imageMutex.Lock(d.img.id)
defer p.graph.imageMutex.Unlock(d.img.id)
// Must recheck the data on disk if any exists.
// This protects against races where something
// else is written to the graph under this ID
// after attemptIDReuse.
if p.graph.Exists(d.img.id) {
if err := p.validateImageInGraph(d.img.id, imgs, i); err != nil {
return fmt.Errorf("image validation failed: %v", err)
}
}
if err := p.graph.register(d.img, reader); err != nil {
return err
}
if err := p.graph.setLayerDigest(d.img.id, d.digest); err != nil {
return err
}
if err := p.graph.setV1CompatibilityConfig(d.img.id, d.img.v1Compatibility); err != nil {
return err
}
return nil
}()
if err != nil {
return false, err
}
// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Pull complete", nil))
tagUpdated = true
} else {
out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Already exists", nil))
}
}
manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName)
if err != nil {
return false, err
}
// Check for new tag if no layers downloaded
if !tagUpdated {
repo, err := p.Get(p.repoInfo.LocalName)
if err != nil {
return false, err
}
if repo != nil {
if _, exists := repo[tag]; !exists {
tagUpdated = true
}
} else {
tagUpdated = true
}
}
if utils.DigestReference(tag) {
// TODO(stevvooe): Ideally, we should always set the digest so we can
// use the digest whether we pull by it or not. Unfortunately, the tag
// store treats the digest as a separate tag, meaning there may be an
// untagged digest image that would seem to be dangling by a user.
if err = p.SetDigest(p.repoInfo.LocalName, tag, downloads[0].img.id); err != nil {
return false, err
}
} else {
// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
if err = p.Tag(p.repoInfo.LocalName, tag, downloads[0].img.id, true); err != nil {
return false, err
}
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
}
return tagUpdated, nil
}
func (p *v2Puller) pullV2Tag(tag, taggedName string) (bool, error) {
logrus.Debugf("Pulling tag from V2 registry: %q", tag)
out := p.config.OutStream
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return false, err
}
manifest, err := manSvc.GetByTag(tag)
if err != nil {
return false, err
}
verified, err := p.validateManifest(manifest, tag)
if err != nil {
return false, err
}
if verified {
logrus.Printf("Image manifest for %s has been verified", taggedName)
}
out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
downloads := make([]downloadInfo, len(manifest.FSLayers))
layerIDs := []string{}
defer func() {
p.graph.Release(p.sessionID, layerIDs...)
}()
for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility))
if err != nil {
logrus.Debugf("error getting image v1 json: %v", err)
return false, err
}
downloads[i].img = img
downloads[i].digest = manifest.FSLayers[i].BlobSum
p.graph.Retain(p.sessionID, img.ID)
layerIDs = append(layerIDs, img.ID)
// Check if exists
if p.graph.Exists(img.ID) {
logrus.Debugf("Image already exists: %s", img.ID)
continue
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil))
downloads[i].err = make(chan error)
go p.download(&downloads[i])
}
var tagUpdated bool
for i := len(downloads) - 1; i >= 0; i-- {
d := &downloads[i]
if d.err != nil {
if err := <-d.err; err != nil {
return false, err
}
}
if d.layer != nil {
// if tmpFile is empty assume download and extracted elsewhere
defer os.Remove(d.tmpFile.Name())
defer d.tmpFile.Close()
d.tmpFile.Seek(0, 0)
if d.tmpFile != nil {
reader := progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: out,
Formatter: p.sf,
Size: int(d.size),
NewLines: false,
ID: stringid.TruncateID(d.img.ID),
Action: "Extracting",
})
err = p.graph.Register(d.img, reader)
if err != nil {
return false, err
}
if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil {
return false, err
}
// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil))
tagUpdated = true
} else {
out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Already exists", nil))
}
}
manifestDigest, _, err := digestFromManifest(manifest, p.repoInfo.LocalName)
if err != nil {
return false, err
}
// Check for new tag if no layers downloaded
if !tagUpdated {
repo, err := p.Get(p.repoInfo.LocalName)
if err != nil {
return false, err
}
if repo != nil {
if _, exists := repo[tag]; !exists {
tagUpdated = true
}
} else {
tagUpdated = true
}
}
if verified && tagUpdated {
out.Write(p.sf.FormatStatus(p.repo.Name()+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security."))
}
if utils.DigestReference(tag) {
// TODO(stevvooe): Ideally, we should always set the digest so we can
// use the digest whether we pull by it or not. Unfortunately, the tag
// store treats the digest as a separate tag, meaning there may be an
// untagged digest image that would seem to be dangling by a user.
if err = p.SetDigest(p.repoInfo.LocalName, tag, downloads[0].img.ID); err != nil {
return false, err
}
} else {
// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
if err = p.Tag(p.repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil {
return false, err
}
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
}
return tagUpdated, nil
}
// loadManifest loads a manifest from a byte array and verifies its content.
// The signature must be verified or an error is returned. If the manifest
// contains no signatures by a trusted key for the name in the manifest, the
// image is not considered verified. The parsed manifest object and a boolean
// for whether the manifest is verified is returned.
func (s *TagStore) loadManifest(eng *engine.Engine, manifestBytes []byte, dgst, ref string) (*registry.ManifestData, bool, error) {
sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures")
if err != nil {
return nil, false, fmt.Errorf("error parsing payload: %s", err)
}
keys, err := sig.Verify()
if err != nil {
return nil, false, fmt.Errorf("error verifying payload: %s", err)
}
payload, err := sig.Payload()
if err != nil {
return nil, false, fmt.Errorf("error retrieving payload: %s", err)
}
var manifestDigest digest.Digest
if dgst != "" {
manifestDigest, err = digest.ParseDigest(dgst)
if err != nil {
return nil, false, fmt.Errorf("invalid manifest digest from registry: %s", err)
}
dgstVerifier, err := digest.NewDigestVerifier(manifestDigest)
if err != nil {
return nil, false, fmt.Errorf("unable to verify manifest digest from registry: %s", err)
}
dgstVerifier.Write(payload)
if !dgstVerifier.Verified() {
computedDigest, _ := digest.FromBytes(payload)
return nil, false, fmt.Errorf("unable to verify manifest digest: registry has %q, computed %q", manifestDigest, computedDigest)
}
}
if utils.DigestReference(ref) && ref != manifestDigest.String() {
return nil, false, fmt.Errorf("mismatching image manifest digest: got %q, expected %q", manifestDigest, ref)
}
var manifest registry.ManifestData
if err := json.Unmarshal(payload, &manifest); err != nil {
return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err)
}
if manifest.SchemaVersion != 1 {
return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion)
}
var verified bool
for _, key := range keys {
job := eng.Job("trust_key_check")
b, err := key.MarshalJSON()
if err != nil {
return nil, false, fmt.Errorf("error marshalling public key: %s", err)
}
namespace := manifest.Name
if namespace[0] != '/' {
namespace = "/" + namespace
}
stdoutBuffer := bytes.NewBuffer(nil)
job.Args = append(job.Args, namespace)
job.Setenv("PublicKey", string(b))
// Check key has read/write permission (0x03)
job.SetenvInt("Permission", 0x03)
job.Stdout.Add(stdoutBuffer)
if err = job.Run(); err != nil {
return nil, false, fmt.Errorf("error running key check: %s", err)
}
result := engine.Tail(stdoutBuffer, 1)
logrus.Debugf("Key check result: %q", result)
if result == "verified" {
verified = true
}
}
return &manifest, verified, nil
}
// Images returns a filtered list of images. filterArgs is a JSON-encoded set
// of filter arguments which will be interpreted by pkg/parsers/filters.
// filter is a shell glob string applied to repository names. The argument
// named all controls whether all images in the graph are filtered, or just
// the heads.
func (s *TagStore) Images(filterArgs, filter string, all bool) ([]*types.Image, error) {
var (
allImages map[string]*image.Image
err error
filtTagged = true
filtLabel = false
)
imageFilters, err := filters.FromParam(filterArgs)
if err != nil {
return nil, err
}
for name := range imageFilters {
if _, ok := acceptedImageFilterTags[name]; !ok {
return nil, fmt.Errorf("Invalid filter '%s'", name)
}
}
if i, ok := imageFilters["dangling"]; ok {
for _, value := range i {
if strings.ToLower(value) == "true" {
filtTagged = false
}
}
}
_, filtLabel = imageFilters["label"]
if all && filtTagged {
allImages = s.graph.Map()
} else {
allImages = s.graph.Heads()
}
lookup := make(map[string]*types.Image)
s.Lock()
for repoName, repository := range s.Repositories {
if filter != "" {
if match, _ := path.Match(filter, repoName); !match {
continue
}
}
for ref, id := range repository {
imgRef := utils.ImageReference(repoName, ref)
image, err := s.graph.Get(id)
if err != nil {
logrus.Warnf("couldn't load %s from %s: %s", id, imgRef, err)
continue
}
if lImage, exists := lookup[id]; exists {
if filtTagged {
if utils.DigestReference(ref) {
lImage.RepoDigests = append(lImage.RepoDigests, imgRef)
} else { // Tag Ref.
lImage.RepoTags = append(lImage.RepoTags, imgRef)
}
}
} else {
// get the boolean list for if only the untagged images are requested
delete(allImages, id)
if !imageFilters.MatchKVList("label", image.ContainerConfig.Labels) {
continue
}
if filtTagged {
newImage := new(types.Image)
newImage.ParentID = image.Parent
newImage.ID = image.ID
newImage.Created = image.Created.Unix()
newImage.Size = image.Size
newImage.VirtualSize = s.graph.GetParentsSize(image) + image.Size
newImage.Labels = image.ContainerConfig.Labels
if utils.DigestReference(ref) {
newImage.RepoTags = []string{}
newImage.RepoDigests = []string{imgRef}
} else {
newImage.RepoTags = []string{imgRef}
newImage.RepoDigests = []string{}
}
lookup[id] = newImage
}
}
}
}
s.Unlock()
images := []*types.Image{}
for _, value := range lookup {
images = append(images, value)
}
// Display images which aren't part of a repository/tag
if filter == "" || filtLabel {
for _, image := range allImages {
if !imageFilters.MatchKVList("label", image.ContainerConfig.Labels) {
continue
}
newImage := new(types.Image)
newImage.ParentID = image.Parent
newImage.RepoTags = []string{"<none>:<none>"}
newImage.RepoDigests = []string{"<none>@<none>"}
newImage.ID = image.ID
newImage.Created = image.Created.Unix()
newImage.Size = image.Size
newImage.VirtualSize = s.graph.GetParentsSize(image) + image.Size
newImage.Labels = image.ContainerConfig.Labels
images = append(images, newImage)
}
}
sort.Sort(sort.Reverse(byCreated(images)))
return images, nil
}
func (s *TagStore) CmdImages(job *engine.Job) engine.Status {
var (
allImages map[string]*image.Image
err error
filt_tagged = true
filt_label = false
)
imageFilters, err := filters.FromParam(job.Getenv("filters"))
if err != nil {
return job.Error(err)
}
for name := range imageFilters {
if _, ok := acceptedImageFilterTags[name]; !ok {
return job.Errorf("Invalid filter '%s'", name)
}
}
if i, ok := imageFilters["dangling"]; ok {
for _, value := range i {
if strings.ToLower(value) == "true" {
filt_tagged = false
}
}
}
_, filt_label = imageFilters["label"]
if job.GetenvBool("all") && filt_tagged {
allImages, err = s.graph.Map()
} else {
allImages, err = s.graph.Heads()
}
if err != nil {
return job.Error(err)
}
lookup := make(map[string]*engine.Env)
s.Lock()
for repoName, repository := range s.Repositories {
if job.Getenv("filter") != "" {
if match, _ := path.Match(job.Getenv("filter"), repoName); !match {
continue
}
}
for ref, id := range repository {
imgRef := utils.ImageReference(repoName, ref)
image, err := s.graph.Get(id)
if err != nil {
log.Printf("Warning: couldn't load %s from %s: %s", id, imgRef, err)
continue
}
if out, exists := lookup[id]; exists {
if filt_tagged {
if utils.DigestReference(ref) {
out.SetList("RepoDigests", append(out.GetList("RepoDigests"), imgRef))
} else { // Tag Ref.
out.SetList("RepoTags", append(out.GetList("RepoTags"), imgRef))
}
}
} else {
// get the boolean list for if only the untagged images are requested
delete(allImages, id)
if !imageFilters.MatchKVList("label", image.ContainerConfig.Labels) {
continue
}
if filt_tagged {
out := &engine.Env{}
out.SetJson("ParentId", image.Parent)
out.SetJson("Id", image.ID)
out.SetInt64("Created", image.Created.Unix())
out.SetInt64("Size", image.Size)
out.SetInt64("VirtualSize", image.GetParentsSize(0)+image.Size)
out.SetJson("Labels", image.ContainerConfig.Labels)
if utils.DigestReference(ref) {
out.SetList("RepoTags", []string{})
out.SetList("RepoDigests", []string{imgRef})
} else {
out.SetList("RepoTags", []string{imgRef})
out.SetList("RepoDigests", []string{})
}
lookup[id] = out
}
}
}
}
s.Unlock()
outs := engine.NewTable("Created", len(lookup))
for _, value := range lookup {
outs.Add(value)
}
// Display images which aren't part of a repository/tag
if job.Getenv("filter") == "" || filt_label {
for _, image := range allImages {
if !imageFilters.MatchKVList("label", image.ContainerConfig.Labels) {
continue
}
out := &engine.Env{}
out.SetJson("ParentId", image.Parent)
out.SetList("RepoTags", []string{"<none>:<none>"})
out.SetList("RepoDigests", []string{"<none>@<none>"})
out.SetJson("Id", image.ID)
out.SetInt64("Created", image.Created.Unix())
out.SetInt64("Size", image.Size)
out.SetInt64("VirtualSize", image.GetParentsSize(0)+image.Size)
out.SetJson("Labels", image.ContainerConfig.Labels)
outs.Add(out)
}
}
outs.ReverseSort()
if _, err := outs.WriteListTo(job.Stdout); err != nil {
return job.Error(err)
}
return engine.StatusOK
}
// Images returns a filtered list of images. filterArgs is a JSON-encoded set
// of filter arguments which will be interpreted by pkg/parsers/filters.
// filter is a shell glob string applied to repository names. The argument
// named all controls whether all images in the graph are filtered, or just
// the heads.
func (s *TagStore) Images(filterArgs, filter string, all bool) ([]*types.Image, error) {
var (
allImages map[string]*image.Image
err error
filtTagged = true
filtLabel = false
)
imageFilters, err := filters.FromParam(filterArgs)
if err != nil {
return nil, err
}
for name := range imageFilters {
if _, ok := acceptedImageFilterTags[name]; !ok {
return nil, fmt.Errorf("Invalid filter '%s'", name)
}
}
if i, ok := imageFilters["dangling"]; ok {
for _, value := range i {
if v := strings.ToLower(value); v == "true" {
filtTagged = false
} else if v != "false" {
return nil, fmt.Errorf("Invalid filter 'dangling=%s'", v)
}
}
}
_, filtLabel = imageFilters["label"]
if all && filtTagged {
allImages = s.graph.Map()
} else {
allImages = s.graph.heads()
}
lookup := make(map[string]*types.Image)
s.Lock()
for repoName, repository := range s.Repositories {
filterTagName := ""
if filter != "" {
filterName := filter
// Test if the tag was in there, if yes, get the name
if strings.Contains(filterName, ":") {
filterWithTag := strings.Split(filter, ":")
filterName = filterWithTag[0]
filterTagName = filterWithTag[1]
}
if match, _ := path.Match(filterName, repoName); !match {
continue
}
if filterTagName != "" {
if _, ok := repository[filterTagName]; !ok {
continue
}
}
}
for ref, id := range repository {
imgRef := utils.ImageReference(repoName, ref)
if !strings.Contains(imgRef, filterTagName) {
continue
}
image, err := s.graph.Get(id)
if err != nil {
logrus.Warnf("couldn't load %s from %s: %s", id, imgRef, err)
continue
}
if lImage, exists := lookup[id]; exists {
if filtTagged {
if utils.DigestReference(ref) {
lImage.RepoDigests = append(lImage.RepoDigests, imgRef)
} else { // Tag Ref.
lImage.RepoTags = append(lImage.RepoTags, imgRef)
}
}
} else {
// get the boolean list for if only the untagged images are requested
delete(allImages, id)
if len(imageFilters["label"]) > 0 {
if image.Config == nil {
// Very old image that do not have image.Config (or even labels)
continue
}
// We are now sure image.Config is not nil
if !imageFilters.MatchKVList("label", image.Config.Labels) {
continue
}
}
if filtTagged {
newImage := newImage(image, s.graph.getParentsSize(image))
if utils.DigestReference(ref) {
newImage.RepoTags = []string{}
newImage.RepoDigests = []string{imgRef}
} else {
newImage.RepoTags = []string{imgRef}
newImage.RepoDigests = []string{}
}
lookup[id] = newImage
}
}
}
}
s.Unlock()
images := []*types.Image{}
for _, value := range lookup {
images = append(images, value)
}
// Display images which aren't part of a repository/tag
if filter == "" || filtLabel {
for _, image := range allImages {
if len(imageFilters["label"]) > 0 {
if image.Config == nil {
// Very old image that do not have image.Config (or even labels)
continue
}
// We are now sure image.Config is not nil
if !imageFilters.MatchKVList("label", image.Config.Labels) {
continue
}
}
newImage := newImage(image, s.graph.getParentsSize(image))
newImage.RepoTags = []string{"<none>:<none>"}
newImage.RepoDigests = []string{"<none>@<none>"}
images = append(images, newImage)
}
}
sort.Sort(sort.Reverse(byCreated(images)))
return images, nil
}
// CmdImages lists the images in a specified repository, or all top-level images if no repository is specified.
//
// Usage: docker images [OPTIONS] [REPOSITORY]
func (cli *DockerCli) CmdImages(args ...string) error {
cmd := cli.Subcmd("images", "[REPOSITORY]", "List images", true)
quiet := cmd.Bool([]string{"q", "-quiet"}, false, "Only show numeric IDs")
all := cmd.Bool([]string{"a", "-all"}, false, "Show all images (default hides intermediate images)")
noTrunc := cmd.Bool([]string{"#notrunc", "-no-trunc"}, false, "Don't truncate output")
showDigests := cmd.Bool([]string{"-digests"}, false, "Show digests")
// FIXME: --viz and --tree are deprecated. Remove them in a future version.
flViz := cmd.Bool([]string{"#v", "#viz", "#-viz"}, false, "Output graph in graphviz format")
flTree := cmd.Bool([]string{"#t", "#tree", "#-tree"}, false, "Output graph in tree format")
flFilter := opts.NewListOpts(nil)
cmd.Var(&flFilter, []string{"f", "-filter"}, "Filter output based on conditions provided")
cmd.Require(flag.Max, 1)
utils.ParseFlags(cmd, args, true)
// Consolidate all filter flags, and sanity check them early.
// They'll get process in the daemon/server.
imageFilterArgs := filters.Args{}
for _, f := range flFilter.GetAll() {
var err error
imageFilterArgs, err = filters.ParseFlag(f, imageFilterArgs)
if err != nil {
return err
}
}
matchName := cmd.Arg(0)
// FIXME: --viz and --tree are deprecated. Remove them in a future version.
if *flViz || *flTree {
v := url.Values{
"all": []string{"1"},
}
if len(imageFilterArgs) > 0 {
filterJSON, err := filters.ToParam(imageFilterArgs)
if err != nil {
return err
}
v.Set("filters", filterJSON)
}
body, _, err := readBody(cli.call("GET", "/images/json?"+v.Encode(), nil, nil))
if err != nil {
return err
}
outs := engine.NewTable("Created", 0)
if _, err := outs.ReadListFrom(body); err != nil {
return err
}
var (
printNode func(cli *DockerCli, noTrunc bool, image *engine.Env, prefix string)
startImage *engine.Env
roots = engine.NewTable("Created", outs.Len())
byParent = make(map[string]*engine.Table)
)
for _, image := range outs.Data {
if image.Get("ParentId") == "" {
roots.Add(image)
} else {
if children, exists := byParent[image.Get("ParentId")]; exists {
children.Add(image)
} else {
byParent[image.Get("ParentId")] = engine.NewTable("Created", 1)
byParent[image.Get("ParentId")].Add(image)
}
}
if matchName != "" {
if matchName == image.Get("Id") || matchName == stringid.TruncateID(image.Get("Id")) {
startImage = image
}
for _, repotag := range image.GetList("RepoTags") {
if repotag == matchName {
startImage = image
}
}
}
}
if *flViz {
fmt.Fprintf(cli.out, "digraph docker {\n")
printNode = (*DockerCli).printVizNode
} else {
printNode = (*DockerCli).printTreeNode
}
if startImage != nil {
root := engine.NewTable("Created", 1)
root.Add(startImage)
cli.WalkTree(*noTrunc, root, byParent, "", printNode)
} else if matchName == "" {
cli.WalkTree(*noTrunc, roots, byParent, "", printNode)
}
if *flViz {
fmt.Fprintf(cli.out, " base [style=invisible]\n}\n")
}
} else {
v := url.Values{}
if len(imageFilterArgs) > 0 {
filterJSON, err := filters.ToParam(imageFilterArgs)
if err != nil {
return err
}
v.Set("filters", filterJSON)
}
if cmd.NArg() == 1 {
// FIXME rename this parameter, to not be confused with the filters flag
v.Set("filter", matchName)
}
if *all {
v.Set("all", "1")
}
body, _, err := readBody(cli.call("GET", "/images/json?"+v.Encode(), nil, nil))
if err != nil {
return err
}
outs := engine.NewTable("Created", 0)
if _, err := outs.ReadListFrom(body); err != nil {
return err
}
w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
if !*quiet {
if *showDigests {
fmt.Fprintln(w, "REPOSITORY\tTAG\tDIGEST\tIMAGE ID\tCREATED\tVIRTUAL SIZE")
} else {
fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tVIRTUAL SIZE")
}
}
for _, out := range outs.Data {
outID := out.Get("Id")
if !*noTrunc {
outID = stringid.TruncateID(outID)
}
repoTags := out.GetList("RepoTags")
repoDigests := out.GetList("RepoDigests")
if len(repoTags) == 1 && repoTags[0] == "<none>:<none>" && len(repoDigests) == 1 && repoDigests[0] == "<none>@<none>" {
// dangling image - clear out either repoTags or repoDigsts so we only show it once below
repoDigests = []string{}
}
// combine the tags and digests lists
tagsAndDigests := append(repoTags, repoDigests...)
for _, repoAndRef := range tagsAndDigests {
repo, ref := parsers.ParseRepositoryTag(repoAndRef)
// default tag and digest to none - if there's a value, it'll be set below
tag := "<none>"
digest := "<none>"
if utils.DigestReference(ref) {
digest = ref
} else {
tag = ref
}
if !*quiet {
if *showDigests {
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", repo, tag, digest, outID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(out.GetInt64("Created"), 0))), units.HumanSize(float64(out.GetInt64("VirtualSize"))))
} else {
fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, outID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(out.GetInt64("Created"), 0))), units.HumanSize(float64(out.GetInt64("VirtualSize"))))
}
} else {
fmt.Fprintln(w, outID)
}
}
}
if !*quiet {
w.Flush()
}
}
return nil
}
func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (tagUpdated bool, err error) {
logrus.Debugf("Pulling tag from V2 registry: %q", tag)
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return false, err
}
unverifiedManifest, err := manSvc.GetByTag(tag)
if err != nil {
return false, err
}
if unverifiedManifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag %q", tag)
}
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifyManifest(unverifiedManifest, tag)
if err != nil {
return false, err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return false, err
}
imgs, err := p.getImageInfos(verifiedManifest)
if err != nil {
return false, err
}
out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
var downloads []*downloadInfo
var layerIDs []string
defer func() {
p.graph.Release(p.sessionID, layerIDs...)
for _, d := range downloads {
p.poolRemoveWithError("pull", d.poolKey, err)
if d.tmpFile != nil {
d.tmpFile.Close()
if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
}
}
}
}()
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
img := imgs[i]
p.graph.Retain(p.sessionID, img.id)
layerIDs = append(layerIDs, img.id)
p.graph.imageMutex.Lock(img.id)
// Check if exists
if p.graph.Exists(img.id) {
if err := p.validateImageInGraph(img.id, imgs, i); err != nil {
p.graph.imageMutex.Unlock(img.id)
return false, fmt.Errorf("image validation failed: %v", err)
}
logrus.Debugf("Image already exists: %s", img.id)
p.graph.imageMutex.Unlock(img.id)
continue
}
p.graph.imageMutex.Unlock(img.id)
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.id), "Pulling fs layer", nil))
d := &downloadInfo{
img: img,
imgIndex: i,
poolKey: "v2layer:" + img.id,
digest: verifiedManifest.FSLayers[i].BlobSum,
// TODO: seems like this chan buffer solved hanging problem in go1.5,
// this can indicate some deeper problem that somehow we never take
// error from channel in loop below
err: make(chan error, 1),
}
tmpFile, err := ioutil.TempFile("", "GetImageBlob")
if err != nil {
return false, err
}
d.tmpFile = tmpFile
downloads = append(downloads, d)
broadcaster, found := p.poolAdd("pull", d.poolKey)
broadcaster.Add(out)
d.broadcaster = broadcaster
if found {
d.err <- nil
} else {
go p.download(d)
}
}
for _, d := range downloads {
if err := <-d.err; err != nil {
return false, err
}
if d.layer == nil {
// Wait for a different pull to download and extract
// this layer.
err = d.broadcaster.Wait()
if err != nil {
return false, err
}
continue
}
d.tmpFile.Seek(0, 0)
err := func() error {
reader := progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: d.broadcaster,
Formatter: p.sf,
Size: d.size,
NewLines: false,
ID: stringid.TruncateID(d.img.id),
Action: "Extracting",
})
p.graph.imagesMutex.Lock()
defer p.graph.imagesMutex.Unlock()
p.graph.imageMutex.Lock(d.img.id)
defer p.graph.imageMutex.Unlock(d.img.id)
// Must recheck the data on disk if any exists.
// This protects against races where something
// else is written to the graph under this ID
// after attemptIDReuse.
if p.graph.Exists(d.img.id) {
if err := p.validateImageInGraph(d.img.id, imgs, d.imgIndex); err != nil {
return fmt.Errorf("image validation failed: %v", err)
}
}
if err := p.graph.register(d.img, reader); err != nil {
return err
}
if err := p.graph.setLayerDigest(d.img.id, d.digest); err != nil {
return err
}
if err := p.graph.setV1CompatibilityConfig(d.img.id, d.img.v1Compatibility); err != nil {
return err
}
return nil
}()
if err != nil {
return false, err
}
d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Pull complete", nil))
d.broadcaster.Close()
tagUpdated = true
}
manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName)
if err != nil {
return false, err
}
// Check for new tag if no layers downloaded
if !tagUpdated {
repo, err := p.Get(p.repoInfo.LocalName)
if err != nil {
return false, err
}
if repo != nil {
if _, exists := repo[tag]; !exists {
tagUpdated = true
}
} else {
tagUpdated = true
}
}
firstID := layerIDs[len(layerIDs)-1]
if utils.DigestReference(tag) {
// TODO(stevvooe): Ideally, we should always set the digest so we can
// use the digest whether we pull by it or not. Unfortunately, the tag
// store treats the digest as a separate tag, meaning there may be an
// untagged digest image that would seem to be dangling by a user.
if err = p.SetDigest(p.repoInfo.LocalName, tag, firstID); err != nil {
return false, err
}
} else {
// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
if err = p.Tag(p.repoInfo.LocalName, tag, firstID, true); err != nil {
return false, err
}
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
}
return tagUpdated, nil
}
// CmdImages lists the images in a specified repository, or all top-level images if no repository is specified.
//
// Usage: docker images [OPTIONS] [REPOSITORY]
func (cli *DockerCli) CmdImages(args ...string) error {
cmd := Cli.Subcmd("images", []string{"[REPOSITORY[:TAG]]"}, Cli.DockerCommands["images"].Description, true)
quiet := cmd.Bool([]string{"q", "-quiet"}, false, "Only show numeric IDs")
all := cmd.Bool([]string{"a", "-all"}, false, "Show all images (default hides intermediate images)")
noTrunc := cmd.Bool([]string{"#notrunc", "-no-trunc"}, false, "Don't truncate output")
showDigests := cmd.Bool([]string{"-digests"}, false, "Show digests")
flFilter := opts.NewListOpts(nil)
cmd.Var(&flFilter, []string{"f", "-filter"}, "Filter output based on conditions provided")
cmd.Require(flag.Max, 1)
cmd.ParseFlags(args, true)
// Consolidate all filter flags, and sanity check them early.
// They'll get process in the daemon/server.
imageFilterArgs := filters.Args{}
for _, f := range flFilter.GetAll() {
var err error
imageFilterArgs, err = filters.ParseFlag(f, imageFilterArgs)
if err != nil {
return err
}
}
matchName := cmd.Arg(0)
v := url.Values{}
if len(imageFilterArgs) > 0 {
filterJSON, err := filters.ToParam(imageFilterArgs)
if err != nil {
return err
}
v.Set("filters", filterJSON)
}
if cmd.NArg() == 1 {
// FIXME rename this parameter, to not be confused with the filters flag
v.Set("filter", matchName)
}
if *all {
v.Set("all", "1")
}
serverResp, err := cli.call("GET", "/images/json?"+v.Encode(), nil, nil)
if err != nil {
return err
}
defer serverResp.body.Close()
images := []types.Image{}
if err := json.NewDecoder(serverResp.body).Decode(&images); err != nil {
return err
}
w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
if !*quiet {
if *showDigests {
fmt.Fprintln(w, "REPOSITORY\tTAG\tDIGEST\tIMAGE ID\tCREATED\tVIRTUAL SIZE")
} else {
fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tVIRTUAL SIZE")
}
}
for _, image := range images {
ID := image.ID
if !*noTrunc {
ID = stringid.TruncateID(ID)
}
repoTags := image.RepoTags
repoDigests := image.RepoDigests
if len(repoTags) == 1 && repoTags[0] == "<none>:<none>" && len(repoDigests) == 1 && repoDigests[0] == "<none>@<none>" {
// dangling image - clear out either repoTags or repoDigsts so we only show it once below
repoDigests = []string{}
}
// combine the tags and digests lists
tagsAndDigests := append(repoTags, repoDigests...)
for _, repoAndRef := range tagsAndDigests {
repo, ref := parsers.ParseRepositoryTag(repoAndRef)
// default tag and digest to none - if there's a value, it'll be set below
tag := "<none>"
digest := "<none>"
if utils.DigestReference(ref) {
digest = ref
} else {
tag = ref
}
if !*quiet {
if *showDigests {
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", repo, tag, digest, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.VirtualSize)))
} else {
fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.VirtualSize)))
}
} else {
fmt.Fprintln(w, ID)
}
}
}
if !*quiet {
w.Flush()
}
return nil
}
func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error) {
logrus.Debugf("Pulling tag from V2 registry: %q", tag)
out := p.config.OutStream
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return false, err
}
manifest, err := manSvc.GetByTag(tag)
if err != nil {
return false, err
}
verified, err = p.validateManifest(manifest, tag)
if err != nil {
return false, err
}
if verified {
logrus.Printf("Image manifest for %s has been verified", taggedName)
}
// By using a pipeWriter for each of the downloads to write their progress
// to, we can avoid an issue where this function returns an error but
// leaves behind running download goroutines. By splitting the writer
// with a pipe, we can close the pipe if there is any error, consequently
// causing each download to cancel due to an error writing to this pipe.
pipeReader, pipeWriter := io.Pipe()
go func() {
if _, err := io.Copy(out, pipeReader); err != nil {
logrus.Errorf("error copying from layer download progress reader: %s", err)
}
}()
defer func() {
if err != nil {
// All operations on the pipe are synchronous. This call will wait
// until all current readers/writers are done using the pipe then
// set the error. All successive reads/writes will return with this
// error.
pipeWriter.CloseWithError(errors.New("download canceled"))
}
}()
out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
var downloads []*downloadInfo
var layerIDs []string
defer func() {
p.graph.Release(p.sessionID, layerIDs...)
}()
for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility))
if err != nil {
logrus.Debugf("error getting image v1 json: %v", err)
return false, err
}
p.graph.Retain(p.sessionID, img.ID)
layerIDs = append(layerIDs, img.ID)
// Check if exists
if p.graph.Exists(img.ID) {
logrus.Debugf("Image already exists: %s", img.ID)
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Already exists", nil))
continue
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil))
d := &downloadInfo{
img: img,
digest: manifest.FSLayers[i].BlobSum,
// TODO: seems like this chan buffer solved hanging problem in go1.5,
// this can indicate some deeper problem that somehow we never take
// error from channel in loop below
err: make(chan error, 1),
out: pipeWriter,
}
downloads = append(downloads, d)
go p.download(d)
}
// run clean for all downloads to prevent leftovers
for _, d := range downloads {
defer func(d *downloadInfo) {
if d.tmpFile != nil {
d.tmpFile.Close()
if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
}
}
}(d)
}
var tagUpdated bool
for _, d := range downloads {
if err := <-d.err; err != nil {
return false, err
}
if d.layer == nil {
continue
}
// if tmpFile is empty assume download and extracted elsewhere
d.tmpFile.Seek(0, 0)
reader := progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: out,
Formatter: p.sf,
Size: d.size,
NewLines: false,
ID: stringid.TruncateID(d.img.ID),
Action: "Extracting",
})
err = p.graph.Register(d.img, reader)
if err != nil {
return false, err
}
if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil {
return false, err
}
// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil))
tagUpdated = true
}
manifestDigest, _, err := digestFromManifest(manifest, p.repoInfo.LocalName)
if err != nil {
return false, err
}
// Check for new tag if no layers downloaded
if !tagUpdated {
repo, err := p.Get(p.repoInfo.LocalName)
if err != nil {
return false, err
}
if repo != nil {
if _, exists := repo[tag]; !exists {
tagUpdated = true
}
} else {
tagUpdated = true
}
}
if verified && tagUpdated {
out.Write(p.sf.FormatStatus(p.repo.Name()+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security."))
}
firstID := layerIDs[len(layerIDs)-1]
if utils.DigestReference(tag) {
// TODO(stevvooe): Ideally, we should always set the digest so we can
// use the digest whether we pull by it or not. Unfortunately, the tag
// store treats the digest as a separate tag, meaning there may be an
// untagged digest image that would seem to be dangling by a user.
if err = p.SetDigest(p.repoInfo.LocalName, tag, firstID); err != nil {
return false, err
}
} else {
// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
if err = p.Tag(p.repoInfo.LocalName, tag, firstID, true); err != nil {
return false, err
}
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
}
return tagUpdated, nil
}
func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, endpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) (bool, error) {
log.Debugf("Pulling tag from V2 registry: %q", tag)
manifestBytes, manifestDigest, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth)
if err != nil {
return false, err
}
// loadManifest ensures that the manifest payload has the expected digest
// if the tag is a digest reference.
manifest, verified, err := s.loadManifest(eng, manifestBytes, manifestDigest, tag)
if err != nil {
return false, fmt.Errorf("error verifying manifest: %s", err)
}
if err := checkValidManifest(manifest); err != nil {
return false, err
}
if verified {
log.Printf("Image manifest for %s has been verified", utils.ImageReference(repoInfo.CanonicalName, tag))
}
out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName))
downloads := make([]downloadInfo, len(manifest.FSLayers))
for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
var (
sumStr = manifest.FSLayers[i].BlobSum
imgJSON = []byte(manifest.History[i].V1Compatibility)
)
img, err := image.NewImgJSON(imgJSON)
if err != nil {
return false, fmt.Errorf("failed to parse json: %s", err)
}
downloads[i].img = img
// Check if exists
if s.graph.Exists(img.ID) {
log.Debugf("Image already exists: %s", img.ID)
continue
}
dgst, err := digest.ParseDigest(sumStr)
if err != nil {
return false, err
}
downloads[i].digest = dgst
out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Pulling fs layer", nil))
downloadFunc := func(di *downloadInfo) error {
log.Debugf("pulling blob %q to V1 img %s", sumStr, img.ID)
if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil {
if c != nil {
out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil))
<-c
out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Download complete", nil))
} else {
log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
}
} else {
defer s.poolRemove("pull", "img:"+img.ID)
tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob")
if err != nil {
return err
}
r, l, err := r.GetV2ImageBlobReader(endpoint, repoInfo.RemoteName, di.digest.Algorithm(), di.digest.Hex(), auth)
if err != nil {
return err
}
defer r.Close()
verifier, err := digest.NewDigestVerifier(di.digest)
if err != nil {
return err
}
if _, err := io.Copy(tmpFile, progressreader.New(progressreader.Config{
In: ioutil.NopCloser(io.TeeReader(r, verifier)),
Out: out,
Formatter: sf,
Size: int(l),
NewLines: false,
ID: common.TruncateID(img.ID),
Action: "Downloading",
})); err != nil {
return fmt.Errorf("unable to copy v2 image blob data: %s", err)
}
out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Verifying Checksum", nil))
if !verifier.Verified() {
log.Infof("Image verification failed: checksum mismatch for %q", di.digest.String())
verified = false
}
out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Download complete", nil))
log.Debugf("Downloaded %s to tempfile %s", img.ID, tmpFile.Name())
di.tmpFile = tmpFile
di.length = l
di.downloaded = true
}
di.imgJSON = imgJSON
return nil
}
if parallel {
downloads[i].err = make(chan error)
go func(di *downloadInfo) {
di.err <- downloadFunc(di)
}(&downloads[i])
} else {
err := downloadFunc(&downloads[i])
if err != nil {
return false, err
}
}
}
var tagUpdated bool
for i := len(downloads) - 1; i >= 0; i-- {
d := &downloads[i]
if d.err != nil {
err := <-d.err
if err != nil {
return false, err
}
}
if d.downloaded {
// if tmpFile is empty assume download and extracted elsewhere
defer os.Remove(d.tmpFile.Name())
defer d.tmpFile.Close()
d.tmpFile.Seek(0, 0)
if d.tmpFile != nil {
err = s.graph.Register(d.img,
progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: out,
Formatter: sf,
Size: int(d.length),
ID: common.TruncateID(d.img.ID),
Action: "Extracting",
}))
if err != nil {
return false, err
}
// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
}
out.Write(sf.FormatProgress(common.TruncateID(d.img.ID), "Pull complete", nil))
tagUpdated = true
} else {
out.Write(sf.FormatProgress(common.TruncateID(d.img.ID), "Already exists", nil))
}
}
// Check for new tag if no layers downloaded
if !tagUpdated {
repo, err := s.Get(repoInfo.LocalName)
if err != nil {
return false, err
}
if repo != nil {
if _, exists := repo[tag]; !exists {
tagUpdated = true
}
}
}
if verified && tagUpdated {
out.Write(sf.FormatStatus(utils.ImageReference(repoInfo.CanonicalName, tag), "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security."))
}
if manifestDigest != "" {
out.Write(sf.FormatStatus("", "Digest: %s", manifestDigest))
}
if utils.DigestReference(tag) {
if err = s.SetDigest(repoInfo.LocalName, tag, downloads[0].img.ID); err != nil {
return false, err
}
} else {
// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
if err = s.Set(repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil {
return false, err
}
}
return tagUpdated, nil
}
// loadManifest loads a manifest from a byte array and verifies its content.
// The signature must be verified or an error is returned. If the manifest
// contains no signatures by a trusted key for the name in the manifest, the
// image is not considered verified. The parsed manifest object and a boolean
// for whether the manifest is verified is returned.
func (s *TagStore) loadManifest(manifestBytes []byte, dgst, ref string) (*registry.ManifestData, bool, error) {
sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures")
if err != nil {
return nil, false, fmt.Errorf("error parsing payload: %s", err)
}
keys, err := sig.Verify()
if err != nil {
return nil, false, fmt.Errorf("error verifying payload: %s", err)
}
payload, err := sig.Payload()
if err != nil {
return nil, false, fmt.Errorf("error retrieving payload: %s", err)
}
var manifestDigest digest.Digest
if dgst != "" {
manifestDigest, err = digest.ParseDigest(dgst)
if err != nil {
return nil, false, fmt.Errorf("invalid manifest digest from registry: %s", err)
}
dgstVerifier, err := digest.NewDigestVerifier(manifestDigest)
if err != nil {
return nil, false, fmt.Errorf("unable to verify manifest digest from registry: %s", err)
}
dgstVerifier.Write(payload)
if !dgstVerifier.Verified() {
computedDigest, _ := digest.FromBytes(payload)
return nil, false, fmt.Errorf("unable to verify manifest digest: registry has %q, computed %q", manifestDigest, computedDigest)
}
}
if utils.DigestReference(ref) && ref != manifestDigest.String() {
return nil, false, fmt.Errorf("mismatching image manifest digest: got %q, expected %q", manifestDigest, ref)
}
var manifest registry.ManifestData
if err := json.Unmarshal(payload, &manifest); err != nil {
return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err)
}
if manifest.SchemaVersion != 1 {
return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion)
}
var verified bool
for _, key := range keys {
namespace := manifest.Name
if namespace[0] != '/' {
namespace = "/" + namespace
}
b, err := key.MarshalJSON()
if err != nil {
return nil, false, fmt.Errorf("error marshalling public key: %s", err)
}
// Check key has read/write permission (0x03)
v, err := s.trustService.CheckKey(namespace, b, 0x03)
if err != nil {
vErr, ok := err.(trust.NotVerifiedError)
if !ok {
return nil, false, fmt.Errorf("error running key check: %s", err)
}
logrus.Debugf("Key check result: %v", vErr)
}
verified = v
if verified {
logrus.Debug("Key check result: verified")
}
}
return &manifest, verified, nil
}