func (s *TagStore) History(name string) ([]*types.ImageHistory, error) { foundImage, err := s.LookupImage(name) if err != nil { return nil, err } lookupMap := make(map[string][]string) for name, repository := range s.Repositories { for tag, id := range repository { // If the ID already has a reverse lookup, do not update it unless for "latest" if _, exists := lookupMap[id]; !exists { lookupMap[id] = []string{} } lookupMap[id] = append(lookupMap[id], utils.ImageReference(name, tag)) } } history := []*types.ImageHistory{} err = foundImage.WalkHistory(func(img *image.Image) error { history = append(history, &types.ImageHistory{ ID: img.ID, Created: img.Created.Unix(), CreatedBy: strings.Join(img.ContainerConfig.Cmd.Slice(), " "), Tags: lookupMap[img.ID], Size: img.Size, Comment: img.Comment, }) return nil }) return history, err }
func (s *TagStore) Import(src string, repo string, tag string, imageImportConfig *ImageImportConfig) error { var ( sf = streamformatter.NewJSONStreamFormatter() archive archive.ArchiveReader resp *http.Response ) if src == "-" { archive = imageImportConfig.InConfig } else { u, err := url.Parse(src) if err != nil { return err } if u.Scheme == "" { u.Scheme = "http" u.Host = src u.Path = "" } imageImportConfig.OutStream.Write(sf.FormatStatus("", "Downloading from %s", u)) resp, err = httputils.Download(u.String()) if err != nil { return err } progressReader := progressreader.New(progressreader.Config{ In: resp.Body, Out: imageImportConfig.OutStream, Formatter: sf, Size: int(resp.ContentLength), NewLines: true, ID: "", Action: "Importing", }) defer progressReader.Close() archive = progressReader } img, err := s.graph.Create(archive, "", "", "Imported from "+src, "", nil, imageImportConfig.ContainerConfig) if err != nil { return err } // Optionally register the image at REPO/TAG if repo != "" { if err := s.Tag(repo, tag, img.ID, true); err != nil { return err } } imageImportConfig.OutStream.Write(sf.FormatStatus("", img.ID)) logID := img.ID if tag != "" { logID = utils.ImageReference(logID, tag) } s.eventsService.Log("import", logID, "") return nil }
func (store *TagStore) GetRepoRefs() map[string][]string { store.Lock() reporefs := make(map[string][]string) for name, repository := range store.Repositories { for tag, id := range repository { shortID := stringid.TruncateID(id) reporefs[shortID] = append(reporefs[shortID], utils.ImageReference(name, tag)) } } store.Unlock() return reporefs }
// Return a reverse-lookup table of all the names which refer to each image // Eg. {"43b5f19b10584": {"base:latest", "base:v1"}} func (store *TagStore) ByID() map[string][]string { store.Lock() defer store.Unlock() byID := make(map[string][]string) for repoName, repository := range store.Repositories { for tag, id := range repository { name := utils.ImageReference(repoName, tag) if _, exists := byID[id]; !exists { byID[id] = []string{name} } else { byID[id] = append(byID[id], name) sort.Strings(byID[id]) } } } return byID }
func (s *TagStore) pullV2Repository(r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *streamformatter.StreamFormatter) error { endpoint, err := r.V2RegistryEndpoint(repoInfo.Index) if err != nil { if repoInfo.Index.Official { glog.V(1).Infof("Unable to pull from V2 registry, falling back to v1: %s", err) return ErrV2RegistryUnavailable } return fmt.Errorf("error getting registry endpoint: %s", err) } auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, true) if err != nil { return fmt.Errorf("error getting authorization: %s", err) } var layersDownloaded bool if tag == "" { glog.V(1).Infof("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName) tags, err := r.GetV2RemoteTags(endpoint, repoInfo.RemoteName, auth) if err != nil { return err } if len(tags) == 0 { return registry.ErrDoesNotExist } for _, t := range tags { if downloaded, err := s.pullV2Tag(r, out, endpoint, repoInfo, t, sf, auth); err != nil { return err } else if downloaded { layersDownloaded = true } } } else { if downloaded, err := s.pullV2Tag(r, out, endpoint, repoInfo, tag, sf, auth); err != nil { return err } else if downloaded { layersDownloaded = true } } requestedTag := repoInfo.CanonicalName if len(tag) > 0 { requestedTag = utils.ImageReference(repoInfo.CanonicalName, tag) } WriteStatus(requestedTag, out, sf, layersDownloaded) return nil }
func (s *TagStore) Images(config *ImagesConfig) ([]*types.Image, error) { var ( allImages map[string]*image.Image err error filtTagged = true filtLabel = false ) imageFilters, err := filters.FromParam(config.Filters) if err != nil { return nil, err } for name := range imageFilters { if _, ok := acceptedImageFilterTags[name]; !ok { return nil, fmt.Errorf("Invalid filter '%s'", name) } } if i, ok := imageFilters["dangling"]; ok { for _, value := range i { if strings.ToLower(value) == "true" { filtTagged = false } } } _, filtLabel = imageFilters["label"] if config.All && filtTagged { allImages, err = s.graph.Map() } else { allImages, err = s.graph.Heads() } if err != nil { return nil, err } lookup := make(map[string]*types.Image) s.Lock() for repoName, repository := range s.Repositories { if config.Filter != "" { if match, _ := path.Match(config.Filter, repoName); !match { continue } } for ref, id := range repository { imgRef := utils.ImageReference(repoName, ref) image, err := s.graph.Get(id) if err != nil { log.Printf("Warning: couldn't load %s from %s: %s", id, imgRef, err) continue } if lImage, exists := lookup[id]; exists { if filtTagged { if utils.DigestReference(ref) { lImage.RepoDigests = append(lImage.RepoDigests, imgRef) } else { // Tag Ref. lImage.RepoTags = append(lImage.RepoTags, imgRef) } } } else { // get the boolean list for if only the untagged images are requested delete(allImages, id) if !imageFilters.MatchKVList("label", image.ContainerConfig.Labels) { continue } if filtTagged { newImage := new(types.Image) newImage.ParentId = image.Parent newImage.ID = image.ID newImage.Created = int(image.Created.Unix()) newImage.Size = int(image.Size) newImage.VirtualSize = int(image.GetParentsSize(0) + image.Size) newImage.Labels = image.ContainerConfig.Labels if utils.DigestReference(ref) { newImage.RepoTags = []string{} newImage.RepoDigests = []string{imgRef} } else { newImage.RepoTags = []string{imgRef} newImage.RepoDigests = []string{} } lookup[id] = newImage } } } } s.Unlock() images := []*types.Image{} for _, value := range lookup { images = append(images, value) } // Display images which aren't part of a repository/tag if config.Filter == "" || filtLabel { for _, image := range allImages { if !imageFilters.MatchKVList("label", image.ContainerConfig.Labels) { continue } newImage := new(types.Image) newImage.ParentId = image.Parent newImage.RepoTags = []string{"<none>:<none>"} newImage.RepoDigests = []string{"<none>@<none>"} newImage.ID = image.ID newImage.Created = int(image.Created.Unix()) newImage.Size = int(image.Size) newImage.VirtualSize = int(image.GetParentsSize(0) + image.Size) newImage.Labels = image.ContainerConfig.Labels images = append(images, newImage) } } sort.Sort(sort.Reverse(ByCreated(images))) return images, nil }
func (s *TagStore) pullV2Tag(r *registry.Session, out io.Writer, endpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo, tag string, sf *streamformatter.StreamFormatter, auth *registry.RequestAuthorization) (bool, error) { glog.V(1).Infof("Pulling tag from V2 registry: %q", tag) remoteDigest, manifestBytes, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth) if err != nil { return false, err } // loadManifest ensures that the manifest payload has the expected digest // if the tag is a digest reference. localDigest, manifest, verified, err := s.loadManifest(manifestBytes, tag, remoteDigest) if err != nil { return false, fmt.Errorf("error verifying manifest: %s", err) } if verified { glog.Infof("Image manifest for %s has been verified", utils.ImageReference(repoInfo.CanonicalName, tag)) } out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName)) // downloadInfo is used to pass information from download to extractor type downloadInfo struct { imgJSON []byte img *image.Image digest digest.Digest tmpFile *os.File length int64 downloaded bool err chan error } downloads := make([]downloadInfo, len(manifest.FSLayers)) for i := len(manifest.FSLayers) - 1; i >= 0; i-- { var ( sumStr = manifest.FSLayers[i].BlobSum imgJSON = []byte(manifest.History[i].V1Compatibility) ) img, err := image.NewImgJSON(imgJSON) if err != nil { return false, fmt.Errorf("failed to parse json: %s", err) } downloads[i].img = img // Check if exists if s.graph.Exists(img.ID) { glog.V(1).Infof("Image already exists: %s", img.ID) continue } dgst, err := digest.ParseDigest(sumStr) if err != nil { return false, err } downloads[i].digest = dgst out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil)) downloadFunc := func(di *downloadInfo) error { glog.V(1).Infof("pulling blob %q to V1 img %s", sumStr, img.ID) if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil { if c != nil { out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil)) <-c out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil)) } else { glog.V(1).Infof("Image (id: %s) pull is already running, skipping: %v", img.ID, err) } } else { defer s.poolRemove("pull", "img:"+img.ID) tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob") if err != nil { return err } r, l, err := r.GetV2ImageBlobReader(endpoint, repoInfo.RemoteName, di.digest, auth) if err != nil { return err } defer r.Close() verifier, err := digest.NewDigestVerifier(di.digest) if err != nil { return err } if _, err := io.Copy(tmpFile, progressreader.New(progressreader.Config{ In: ioutil.NopCloser(io.TeeReader(r, verifier)), Out: out, Formatter: sf, Size: int(l), NewLines: false, ID: stringid.TruncateID(img.ID), Action: "Downloading", })); err != nil { return fmt.Errorf("unable to copy v2 image blob data: %s", err) } out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Verifying Checksum", nil)) if !verifier.Verified() { return fmt.Errorf("image layer digest verification failed for %q", di.digest) } out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil)) glog.V(1).Infof("Downloaded %s to tempfile %s", img.ID, tmpFile.Name()) di.tmpFile = tmpFile di.length = l di.downloaded = true } di.imgJSON = imgJSON return nil } downloads[i].err = make(chan error) go func(di *downloadInfo) { di.err <- downloadFunc(di) }(&downloads[i]) } var tagUpdated bool for i := len(downloads) - 1; i >= 0; i-- { d := &downloads[i] if d.err != nil { if err := <-d.err; err != nil { return false, err } } if d.downloaded { // if tmpFile is empty assume download and extracted elsewhere defer os.Remove(d.tmpFile.Name()) defer d.tmpFile.Close() d.tmpFile.Seek(0, 0) if d.tmpFile != nil { err = s.graph.Register(d.img, progressreader.New(progressreader.Config{ In: d.tmpFile, Out: out, Formatter: sf, Size: int(d.length), ID: stringid.TruncateID(d.img.ID), Action: "Extracting", })) if err != nil { return false, err } // FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted) } out.Write(sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil)) tagUpdated = true } else { out.Write(sf.FormatProgress(stringid.TruncateID(d.img.ID), "Already exists", nil)) } } // Check for new tag if no layers downloaded if !tagUpdated { repo, err := s.Get(repoInfo.LocalName) if err != nil { return false, err } if repo != nil { if _, exists := repo[tag]; !exists { tagUpdated = true } } else { tagUpdated = true } } if verified && tagUpdated { out.Write(sf.FormatStatus(utils.ImageReference(repoInfo.CanonicalName, tag), "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security.")) } if localDigest != remoteDigest { // this is not a verification check. // NOTE(stevvooe): This is a very defensive branch and should never // happen, since all manifest digest implementations use the same // algorithm. out.Write(sf.FormatStatus("", "Remote Digest: %s", remoteDigest)) } out.Write(sf.FormatStatus("", "Digest: %s", localDigest)) if tag == localDigest.String() { // TODO(stevvooe): Ideally, we should always set the digest so we can // use the digest whether we pull by it or not. Unfortunately, the tag // store treats the digest as a separate tag, meaning there may be an // untagged digest image that would seem to be dangling by a user. if err = s.SetDigest(repoInfo.LocalName, localDigest.String(), downloads[0].img.ID); err != nil { return false, err } } if !utils.DigestReference(tag) { // only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest) if err = s.Tag(repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil { return false, err } } return tagUpdated, nil }
func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConfig) error { var ( sf = streamformatter.NewJSONStreamFormatter() ) // Resolve the Repository name from fqn to RepositoryInfo repoInfo, err := s.registryService.ResolveRepository(image) if err != nil { return err } if err := validateRepoName(repoInfo.LocalName); err != nil { return err } c, err := s.poolAdd("pull", utils.ImageReference(repoInfo.LocalName, tag)) if err != nil { if c != nil { // Another pull of the same repository is already taking place; just wait for it to finish imagePullConfig.OutStream.Write(sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", repoInfo.LocalName)) <-c return nil } return err } defer s.poolRemove("pull", utils.ImageReference(repoInfo.LocalName, tag)) logName := repoInfo.LocalName if tag != "" { logName = utils.ImageReference(logName, tag) } // Attempt pulling official content from a provided v2 mirror if repoInfo.Index.Official { v2mirrorEndpoint, v2mirrorRepoInfo, err := configureV2Mirror(repoInfo, s.registryService) if err != nil { glog.Errorf("Error configuring mirrors: %s", err) return err } if v2mirrorEndpoint != nil { glog.V(1).Infof("Attempting to pull from v2 mirror: %s", v2mirrorEndpoint.URL) return s.pullFromV2Mirror(v2mirrorEndpoint, v2mirrorRepoInfo, imagePullConfig, tag, sf, logName) } } glog.V(1).Infof("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName) endpoint, err := repoInfo.GetEndpoint(imagePullConfig.MetaHeaders) if err != nil { return err } // TODO(tiborvass): reuse client from endpoint? // Adds Docker-specific headers as well as user-specified headers (metaHeaders) tr := transport.NewTransport( registry.NewTransport(registry.ReceiveTimeout, endpoint.IsSecure), registry.DockerHeaders(imagePullConfig.MetaHeaders)..., ) client := registry.HTTPClient(tr) r, err := registry.NewSession(client, imagePullConfig.AuthConfig, endpoint) if err != nil { return err } if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { if repoInfo.Official { s.trustService.UpdateBase() } glog.V(1).Infof("pulling v2 repository with local name %q", repoInfo.LocalName) if err := s.pullV2Repository(r, imagePullConfig.OutStream, repoInfo, tag, sf); err == nil { s.eventsService.Log("pull", logName, "") return nil } else if err != registry.ErrDoesNotExist && err != ErrV2RegistryUnavailable { glog.Errorf("Error from V2 registry: %s", err) } glog.V(1).Info("image does not exist on v2 registry, falling back to v1") } if utils.DigestReference(tag) { return fmt.Errorf("pulling with digest reference failed from v2 registry") } glog.V(1).Infof("pulling v1 repository with local name %q", repoInfo.LocalName) if err = s.pullRepository(r, imagePullConfig.OutStream, repoInfo, tag, sf); err != nil { return err } s.eventsService.Log("pull", logName, "") return nil }
func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, askedTag string, sf *streamformatter.StreamFormatter) error { out.Write(sf.FormatStatus("", "Pulling repository %s", repoInfo.CanonicalName)) repoData, err := r.GetRepositoryData(repoInfo.RemoteName) if err != nil { if strings.Contains(err.Error(), "HTTP code: 404") { return fmt.Errorf("Error: image %s not found", utils.ImageReference(repoInfo.RemoteName, askedTag)) } // Unexpected HTTP error return err } glog.V(1).Info("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, repoInfo.RemoteName) if err != nil { glog.Errorf("unable to get remote tags: %s", err) return err } for tag, id := range tagsList { repoData.ImgList[id] = ®istry.ImgData{ ID: id, Tag: tag, Checksum: "", } } glog.V(1).Infof("Registering tags") // If no tag has been specified, pull them all if askedTag == "" { for tag, id := range tagsList { repoData.ImgList[id].Tag = tag } } else { // Otherwise, check that the tag exists and use only that one id, exists := tagsList[askedTag] if !exists { return fmt.Errorf("Tag %s not found in repository %s", askedTag, repoInfo.CanonicalName) } repoData.ImgList[id].Tag = askedTag } errors := make(chan error) layersDownloaded := false for _, image := range repoData.ImgList { downloadImage := func(img *registry.ImgData) { if askedTag != "" && img.Tag != askedTag { errors <- nil return } if img.Tag == "" { glog.V(1).Infof("Image (id: %s) present in this repository but untagged, skipping", img.ID) errors <- nil return } // ensure no two downloads of the same image happen at the same time if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil { if c != nil { out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil)) <-c out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil)) } else { glog.V(1).Infof("Image (id: %s) pull is already running, skipping: %v", img.ID, err) } errors <- nil return } defer s.poolRemove("pull", "img:"+img.ID) out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, repoInfo.CanonicalName), nil)) success := false var lastErr, err error var isDownloaded bool for _, ep := range repoInfo.Index.Mirrors { // Ensure endpoint is v1 ep = ep + "v1/" out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, repoInfo.CanonicalName, ep), nil)) if isDownloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { // Don't report errors when pulling from mirrors. glog.V(1).Infof("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err) continue } layersDownloaded = layersDownloaded || isDownloaded success = true break } if !success { for _, ep := range repoData.Endpoints { out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, repoInfo.CanonicalName, ep), nil)) if isDownloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { // It's not ideal that only the last error is returned, it would be better to concatenate the errors. // As the error is also given to the output stream the user will see the error. lastErr = err out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err), nil)) continue } layersDownloaded = layersDownloaded || isDownloaded success = true break } } if !success { err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, repoInfo.CanonicalName, lastErr) out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil)) errors <- err return } out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil)) errors <- nil } go downloadImage(image) } var lastError error for i := 0; i < len(repoData.ImgList); i++ { if err := <-errors; err != nil { lastError = err } } if lastError != nil { return lastError } for tag, id := range tagsList { if askedTag != "" && tag != askedTag { continue } if err := s.Tag(repoInfo.LocalName, tag, id, true); err != nil { return err } } requestedTag := repoInfo.CanonicalName if len(askedTag) > 0 { requestedTag = utils.ImageReference(repoInfo.CanonicalName, askedTag) } WriteStatus(requestedTag, out, sf, layersDownloaded) return nil }
func (daemon *Daemon) imgDeleteHelper(name string, list *[]types.ImageDelete, first, force, noprune bool) error { var ( repoName, tag string tags = []string{} ) repoAndTags := make(map[string][]string) // FIXME: please respect DRY and centralize repo+tag parsing in a single central place! -- shykes repoName, tag = parsers.ParseRepositoryTag(name) if tag == "" { tag = graph.DEFAULTTAG } if name == "" { return fmt.Errorf("Image name can not be blank") } img, err := daemon.Repositories().LookupImage(name) if err != nil { if r, _ := daemon.Repositories().Get(repoName); r != nil { return fmt.Errorf("No such image: %s", utils.ImageReference(repoName, tag)) } return fmt.Errorf("No such image: %s", name) } if strings.Contains(img.ID, name) { repoName = "" tag = "" } byParents, err := daemon.Graph().ByParent() if err != nil { return err } repos := daemon.Repositories().ByID()[img.ID] //If delete by id, see if the id belong only to one repository deleteByID := repoName == "" if deleteByID { for _, repoAndTag := range repos { parsedRepo, parsedTag := parsers.ParseRepositoryTag(repoAndTag) if repoName == "" || repoName == parsedRepo { repoName = parsedRepo if parsedTag != "" { repoAndTags[repoName] = append(repoAndTags[repoName], parsedTag) } } else if repoName != parsedRepo && !force && first { // the id belongs to multiple repos, like base:latest and user:test, // in that case return conflict return fmt.Errorf("Conflict, cannot delete image %s because it is tagged in multiple repositories, use -f to force", name) } else { //the id belongs to multiple repos, with -f just delete all repoName = parsedRepo if parsedTag != "" { repoAndTags[repoName] = append(repoAndTags[repoName], parsedTag) } } } } else { repoAndTags[repoName] = append(repoAndTags[repoName], tag) } if !first && len(repoAndTags) > 0 { return nil } if len(repos) <= 1 || (len(repoAndTags) <= 1 && deleteByID) { if err := daemon.canDeleteImage(img.ID, force); err != nil { return err } } // Untag the current image for repoName, tags := range repoAndTags { for _, tag := range tags { tagDeleted, err := daemon.Repositories().Delete(repoName, tag) if err != nil { return err } if tagDeleted { *list = append(*list, types.ImageDelete{ Untagged: utils.ImageReference(repoName, tag), }) daemon.EventsService.Log("untag", img.ID, "") } } } tags = daemon.Repositories().ByID()[img.ID] if (len(tags) <= 1 && repoName == "") || len(tags) == 0 { if len(byParents[img.ID]) == 0 { if err := daemon.Repositories().DeleteAll(img.ID); err != nil { return err } if err := daemon.Graph().Delete(img.ID); err != nil { return err } *list = append(*list, types.ImageDelete{ Deleted: img.ID, }) daemon.EventsService.Log("delete", img.ID, "") if img.Parent != "" && !noprune { err := daemon.imgDeleteHelper(img.Parent, list, false, force, noprune) if first { return err } } } } return nil }