// History returns a slice of ImageHistory structures for the specified image // name by walking the image lineage. func (s *TagStore) History(name string) ([]*types.ImageHistory, error) { foundImage, err := s.LookupImage(name) if err != nil { return nil, err } lookupMap := make(map[string][]string) for name, repository := range s.Repositories { for tag, id := range repository { // If the ID already has a reverse lookup, do not update it unless for "latest" if _, exists := lookupMap[id]; !exists { lookupMap[id] = []string{} } lookupMap[id] = append(lookupMap[id], utils.ImageReference(name, tag)) } } history := []*types.ImageHistory{} err = s.graph.WalkHistory(foundImage, func(img image.Image) error { history = append(history, &types.ImageHistory{ ID: img.ID, Created: img.Created.Unix(), CreatedBy: strings.Join(img.ContainerConfig.Cmd.Slice(), " "), Tags: lookupMap[img.ID], Size: img.Size, Comment: img.Comment, }) return nil }) return history, err }
func (cli *DockerCli) createContainer(config *runconfig.Config, hostConfig *runconfig.HostConfig, cidfile, name string) (*types.ContainerCreateResponse, error) { containerValues := url.Values{} if name != "" { containerValues.Set("name", name) } mergedConfig := runconfig.MergeConfigs(config, hostConfig) var containerIDFile *cidFile if cidfile != "" { var err error if containerIDFile, err = newCIDFile(cidfile); err != nil { return nil, err } defer containerIDFile.Close() } //create the container stream, statusCode, err := cli.call("POST", "/containers/create?"+containerValues.Encode(), mergedConfig, false) //if image not found try to pull it if statusCode == 404 { repo, tag := parsers.ParseRepositoryTag(config.Image) if tag == "" { tag = graph.DEFAULTTAG } fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", utils.ImageReference(repo, tag)) // we don't want to write to stdout anything apart from container.ID if err = cli.pullImageCustomOut(config.Image, cli.err); err != nil { return nil, err } // Retry if stream, _, err = cli.call("POST", "/containers/create?"+containerValues.Encode(), mergedConfig, false); err != nil { return nil, err } } else if err != nil { return nil, err } var response types.ContainerCreateResponse if err := json.NewDecoder(stream).Decode(&response); err != nil { return nil, err } for _, warning := range response.Warnings { fmt.Fprintf(cli.err, "WARNING: %s\n", warning) } if containerIDFile != nil { if err = containerIDFile.Write(response.ID); err != nil { return nil, err } } return &response, nil }
// GetRepoRefs returns a map with image IDs as keys, and slices listing // repo/tag references as the values. It covers all repositories. func (store *TagStore) GetRepoRefs() map[string][]string { store.Lock() reporefs := make(map[string][]string) for name, repository := range store.Repositories { for tag, id := range repository { shortID := stringid.TruncateID(id) reporefs[shortID] = append(reporefs[shortID], utils.ImageReference(name, tag)) } } store.Unlock() return reporefs }
// Lookup looks up an image by name in a TagStore and returns it as an // ImageInspect structure. func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) { image, err := s.LookupImage(name) if err != nil || image == nil { return nil, fmt.Errorf("No such image: %s", name) } var repoTags = make([]string, 0) var repoDigests = make([]string, 0) s.Lock() for repoName, repository := range s.Repositories { for ref, id := range repository { if id == image.ID { imgRef := utils.ImageReference(repoName, ref) if utils.DigestReference(ref) { repoDigests = append(repoDigests, imgRef) } else { repoTags = append(repoTags, imgRef) } } } } s.Unlock() imageInspect := &types.ImageInspect{ ID: image.ID, RepoTags: repoTags, RepoDigests: repoDigests, Parent: image.Parent, Comment: image.Comment, Created: image.Created.Format(time.RFC3339Nano), Container: image.Container, ContainerConfig: &image.ContainerConfig, DockerVersion: image.DockerVersion, Author: image.Author, Config: image.Config, Architecture: image.Architecture, Os: image.OS, Size: image.Size, VirtualSize: s.graph.GetParentsSize(image) + image.Size, } imageInspect.GraphDriver.Name = s.graph.driver.String() graphDriverData, err := s.graph.driver.GetMetadata(image.ID) if err != nil { return nil, err } imageInspect.GraphDriver.Data = graphDriverData return imageInspect, nil }
func (p *v2Puller) pullV2Repository(tag string) (err error) { var tags []string taggedName := p.repoInfo.LocalName if len(tag) > 0 { tags = []string{tag} taggedName = utils.ImageReference(p.repoInfo.LocalName, tag) } else { var err error manSvc, err := p.repo.Manifests(context.Background()) if err != nil { return err } tags, err = manSvc.Tags() if err != nil { return err } } poolKey := "v2:" + taggedName broadcaster, found := p.poolAdd("pull", poolKey) broadcaster.Add(p.config.OutStream) if found { // Another pull of the same repository is already taking place; just wait for it to finish return broadcaster.Wait() } // This must use a closure so it captures the value of err when the // function returns, not when the 'defer' is evaluated. defer func() { p.poolRemoveWithError("pull", poolKey, err) }() var layersDownloaded bool for _, tag := range tags { // pulledNew is true if either new layers were downloaded OR if existing images were newly tagged // TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus? pulledNew, err := p.pullV2Tag(broadcaster, tag, taggedName) if err != nil { return err } layersDownloaded = layersDownloaded || pulledNew } writeStatus(taggedName, broadcaster, p.sf, layersDownloaded) return nil }
// ByID returns a reverse-lookup table of all the names which refer to each // image - e.g. {"43b5f19b10584": {"base:latest", "base:v1"}} func (store *TagStore) ByID() map[string][]string { store.Lock() defer store.Unlock() byID := make(map[string][]string) for repoName, repository := range store.Repositories { for tag, id := range repository { name := utils.ImageReference(repoName, tag) if _, exists := byID[id]; !exists { byID[id] = []string{name} } else { byID[id] = append(byID[id], name) sort.Strings(byID[id]) } } } return byID }
func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) error { endpoint, err := r.V2RegistryEndpoint(repoInfo.Index) if err != nil { if repoInfo.Index.Official { log.Debugf("Unable to pull from V2 registry, falling back to v1: %s", err) return ErrV2RegistryUnavailable } return fmt.Errorf("error getting registry endpoint: %s", err) } auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, true) if err != nil { return fmt.Errorf("error getting authorization: %s", err) } var layersDownloaded bool if tag == "" { log.Debugf("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName) tags, err := r.GetV2RemoteTags(endpoint, repoInfo.RemoteName, auth) if err != nil { return err } if len(tags) == 0 { return registry.ErrDoesNotExist } for _, t := range tags { if downloaded, err := s.pullV2Tag(eng, r, out, endpoint, repoInfo, t, sf, parallel, auth); err != nil { return err } else if downloaded { layersDownloaded = true } } } else { if downloaded, err := s.pullV2Tag(eng, r, out, endpoint, repoInfo, tag, sf, parallel, auth); err != nil { return err } else if downloaded { layersDownloaded = true } } requestedTag := repoInfo.CanonicalName if len(tag) > 0 { requestedTag = utils.ImageReference(repoInfo.CanonicalName, tag) } WriteStatus(requestedTag, out, sf, layersDownloaded) return nil }
func (s *TagStore) CmdHistory(job *engine.Job) error { if n := len(job.Args); n != 1 { return fmt.Errorf("Usage: %s IMAGE", job.Name) } name := job.Args[0] foundImage, err := s.LookupImage(name) if err != nil { return err } lookupMap := make(map[string][]string) for name, repository := range s.Repositories { for tag, id := range repository { // If the ID already has a reverse lookup, do not update it unless for "latest" if _, exists := lookupMap[id]; !exists { lookupMap[id] = []string{} } lookupMap[id] = append(lookupMap[id], utils.ImageReference(name, tag)) } } outs := engine.NewTable("Created", 0) err = foundImage.WalkHistory(func(img *image.Image) error { out := &engine.Env{} out.SetJson("Id", img.ID) out.SetInt64("Created", img.Created.Unix()) out.Set("CreatedBy", strings.Join(img.ContainerConfig.Cmd, " ")) out.SetList("Tags", lookupMap[img.ID]) out.SetInt64("Size", img.Size) outs.Add(out) return nil }) if _, err := outs.WriteListTo(job.Stdout); err != nil { return err } return nil }
// Images returns a filtered list of images. filterArgs is a JSON-encoded set // of filter arguments which will be interpreted by pkg/parsers/filters. // filter is a shell glob string applied to repository names. The argument // named all controls whether all images in the graph are filtered, or just // the heads. func (s *TagStore) Images(filterArgs, filter string, all bool) ([]*types.Image, error) { var ( allImages map[string]*image.Image err error filtTagged = true filtLabel = false ) imageFilters, err := filters.FromParam(filterArgs) if err != nil { return nil, err } for name := range imageFilters { if _, ok := acceptedImageFilterTags[name]; !ok { return nil, fmt.Errorf("Invalid filter '%s'", name) } } if i, ok := imageFilters["dangling"]; ok { for _, value := range i { if v := strings.ToLower(value); v == "true" { filtTagged = false } else if v != "false" { return nil, fmt.Errorf("Invalid filter 'dangling=%s'", v) } } } _, filtLabel = imageFilters["label"] if all && filtTagged { allImages = s.graph.Map() } else { allImages = s.graph.Heads() } lookup := make(map[string]*types.Image) s.Lock() for repoName, repository := range s.Repositories { filterTagName := "" if filter != "" { filterName := filter // Test if the tag was in there, if yes, get the name if strings.Contains(filterName, ":") { filterWithTag := strings.Split(filter, ":") filterName = filterWithTag[0] filterTagName = filterWithTag[1] } if match, _ := path.Match(filterName, repoName); !match { continue } if filterTagName != "" { if _, ok := repository[filterTagName]; !ok { continue } } } for ref, id := range repository { imgRef := utils.ImageReference(repoName, ref) if !strings.Contains(imgRef, filterTagName) { continue } image, err := s.graph.Get(id) if err != nil { logrus.Warnf("couldn't load %s from %s: %s", id, imgRef, err) continue } if lImage, exists := lookup[id]; exists { if filtTagged { if utils.DigestReference(ref) { lImage.RepoDigests = append(lImage.RepoDigests, imgRef) } else { // Tag Ref. lImage.RepoTags = append(lImage.RepoTags, imgRef) } } } else { // get the boolean list for if only the untagged images are requested delete(allImages, id) if len(imageFilters["label"]) > 0 { if image.Config == nil { // Very old image that do not have image.Config (or even labels) continue } // We are now sure image.Config is not nil if !imageFilters.MatchKVList("label", image.Config.Labels) { continue } } if filtTagged { newImage := newImage(image, s.graph.GetParentsSize(image)) if utils.DigestReference(ref) { newImage.RepoTags = []string{} newImage.RepoDigests = []string{imgRef} } else { newImage.RepoTags = []string{imgRef} newImage.RepoDigests = []string{} } lookup[id] = newImage } } } } s.Unlock() images := []*types.Image{} for _, value := range lookup { images = append(images, value) } // Display images which aren't part of a repository/tag if filter == "" || filtLabel { for _, image := range allImages { if len(imageFilters["label"]) > 0 { if image.Config == nil { // Very old image that do not have image.Config (or even labels) continue } // We are now sure image.Config is not nil if !imageFilters.MatchKVList("label", image.Config.Labels) { continue } } newImage := newImage(image, s.graph.GetParentsSize(image)) newImage.RepoTags = []string{"<none>:<none>"} newImage.RepoDigests = []string{"<none>@<none>"} images = append(images, newImage) } } sort.Sort(sort.Reverse(byCreated(images))) return images, nil }
// Pull initiates a pull operation. image is the repository name to pull, and // tag may be either empty, or indicate a specific tag to pull. func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConfig) error { var sf = streamformatter.NewJSONStreamFormatter() // Resolve the Repository name from fqn to RepositoryInfo repoInfo, err := s.registryService.ResolveRepository(image) if err != nil { return err } // makes sure name is not empty or `scratch` if err := validateRepoName(repoInfo.LocalName); err != nil { return err } endpoints, err := s.registryService.LookupPullEndpoints(repoInfo.CanonicalName) if err != nil { return err } logName := repoInfo.LocalName if tag != "" { logName = utils.ImageReference(logName, tag) } var ( lastErr error // discardNoSupportErrors is used to track whether an endpoint encountered an error of type registry.ErrNoSupport // By default it is false, which means that if a ErrNoSupport error is encountered, it will be saved in lastErr. // As soon as another kind of error is encountered, discardNoSupportErrors is set to true, avoiding the saving of // any subsequent ErrNoSupport errors in lastErr. // It's needed for pull-by-digest on v1 endpoints: if there are only v1 endpoints configured, the error should be // returned and displayed, but if there was a v2 endpoint which supports pull-by-digest, then the last relevant // error is the ones from v2 endpoints not v1. discardNoSupportErrors bool ) for _, endpoint := range endpoints { logrus.Debugf("Trying to pull %s from %s %s", repoInfo.LocalName, endpoint.URL, endpoint.Version) puller, err := NewPuller(s, endpoint, repoInfo, imagePullConfig, sf) if err != nil { lastErr = err continue } if fallback, err := puller.Pull(tag); err != nil { if fallback { if _, ok := err.(registry.ErrNoSupport); !ok { // Because we found an error that's not ErrNoSupport, discard all subsequent ErrNoSupport errors. discardNoSupportErrors = true // save the current error lastErr = err } else if !discardNoSupportErrors { // Save the ErrNoSupport error, because it's either the first error or all encountered errors // were also ErrNoSupport errors. lastErr = err } continue } logrus.Debugf("Not continuing with error: %v", err) return err } s.eventsService.Log("pull", logName, "") return nil } if lastErr == nil { lastErr = fmt.Errorf("no endpoints found for %s", image) } return lastErr }
// Import imports an image, getting the archived layer data either from // inConfig (if src is "-"), or from a URI specified in src. Progress output is // written to outStream. Repository and tag names can optionally be given in // the repo and tag arguments, respectively. func (s *TagStore) Import(src string, repo string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, containerConfig *runconfig.Config) error { var ( sf = streamformatter.NewJSONStreamFormatter() archive io.ReadCloser resp *http.Response ) if src == "-" { archive = inConfig } else { inConfig.Close() u, err := url.Parse(src) if err != nil { return err } if u.Scheme == "" { u.Scheme = "http" u.Host = src u.Path = "" } outStream.Write(sf.FormatStatus("", "Downloading from %s", u)) resp, err = httputils.Download(u.String()) if err != nil { return err } progressReader := progressreader.New(progressreader.Config{ In: resp.Body, Out: outStream, Formatter: sf, Size: resp.ContentLength, NewLines: true, ID: "", Action: "Importing", }) archive = progressReader } defer archive.Close() if len(msg) == 0 { msg = "Imported from " + src } img, err := s.graph.Create(archive, "", "", msg, "", nil, containerConfig) if err != nil { return err } // Optionally register the image at REPO/TAG if repo != "" { if err := s.Tag(repo, tag, img.ID, true); err != nil { return err } } outStream.Write(sf.FormatStatus("", img.ID)) logID := img.ID if tag != "" { logID = utils.ImageReference(logID, tag) } s.eventsService.Log("import", logID, "") return nil }
func (s *TagStore) CmdImport(job *engine.Job) error { if n := len(job.Args); n != 2 && n != 3 { return fmt.Errorf("Usage: %s SRC REPO [TAG]", job.Name) } var ( src = job.Args[0] repo = job.Args[1] tag string sf = utils.NewStreamFormatter(job.GetenvBool("json")) archive archive.ArchiveReader resp *http.Response stdoutBuffer = bytes.NewBuffer(nil) newConfig runconfig.Config ) if len(job.Args) > 2 { tag = job.Args[2] } if src == "-" { archive = job.Stdin } else { u, err := url.Parse(src) if err != nil { return err } if u.Scheme == "" { u.Scheme = "http" u.Host = src u.Path = "" } job.Stdout.Write(sf.FormatStatus("", "Downloading from %s", u)) resp, err = utils.Download(u.String()) if err != nil { return err } progressReader := progressreader.New(progressreader.Config{ In: resp.Body, Out: job.Stdout, Formatter: sf, Size: int(resp.ContentLength), NewLines: true, ID: "", Action: "Importing", }) defer progressReader.Close() archive = progressReader } buildConfigJob := job.Eng.Job("build_config") buildConfigJob.Stdout.Add(stdoutBuffer) buildConfigJob.Setenv("changes", job.Getenv("changes")) // FIXME this should be remove when we remove deprecated config param buildConfigJob.Setenv("config", job.Getenv("config")) if err := buildConfigJob.Run(); err != nil { return err } if err := json.NewDecoder(stdoutBuffer).Decode(&newConfig); err != nil { return err } img, err := s.graph.Create(archive, "", "", "Imported from "+src, "", nil, &newConfig) if err != nil { return err } // Optionally register the image at REPO/TAG if repo != "" { if err := s.Set(repo, tag, img.ID, true); err != nil { return err } } job.Stdout.Write(sf.FormatStatus("", img.ID)) logID := img.ID if tag != "" { logID = utils.ImageReference(logID, tag) } if err = job.Eng.Job("log", "import", logID, "").Run(); err != nil { log.Errorf("Error logging event 'import' for %s: %s", logID, err) } return nil }
func (s *TagStore) CmdImages(job *engine.Job) error { var ( allImages map[string]*image.Image err error filt_tagged = true filt_label = false ) imageFilters, err := filters.FromParam(job.Getenv("filters")) if err != nil { return err } for name := range imageFilters { if _, ok := acceptedImageFilterTags[name]; !ok { return fmt.Errorf("Invalid filter '%s'", name) } } if i, ok := imageFilters["dangling"]; ok { for _, value := range i { if strings.ToLower(value) == "true" { filt_tagged = false } } } _, filt_label = imageFilters["label"] if job.GetenvBool("all") && filt_tagged { allImages, err = s.graph.Map() } else { allImages, err = s.graph.Heads() } if err != nil { return err } lookup := make(map[string]*engine.Env) s.Lock() for repoName, repository := range s.Repositories { if job.Getenv("filter") != "" { if match, _ := path.Match(job.Getenv("filter"), repoName); !match { continue } } for ref, id := range repository { imgRef := utils.ImageReference(repoName, ref) image, err := s.graph.Get(id) if err != nil { log.Printf("Warning: couldn't load %s from %s: %s", id, imgRef, err) continue } if out, exists := lookup[id]; exists { if filt_tagged { if utils.DigestReference(ref) { out.SetList("RepoDigests", append(out.GetList("RepoDigests"), imgRef)) } else { // Tag Ref. out.SetList("RepoTags", append(out.GetList("RepoTags"), imgRef)) } } } else { // get the boolean list for if only the untagged images are requested delete(allImages, id) if !imageFilters.MatchKVList("label", image.ContainerConfig.Labels) { continue } if filt_tagged { out := &engine.Env{} out.SetJson("ParentId", image.Parent) out.SetJson("Id", image.ID) out.SetInt64("Created", image.Created.Unix()) out.SetInt64("Size", image.Size) out.SetInt64("VirtualSize", image.GetParentsSize(0)+image.Size) out.SetJson("Labels", image.ContainerConfig.Labels) if utils.DigestReference(ref) { out.SetList("RepoTags", []string{}) out.SetList("RepoDigests", []string{imgRef}) } else { out.SetList("RepoTags", []string{imgRef}) out.SetList("RepoDigests", []string{}) } lookup[id] = out } } } } s.Unlock() outs := engine.NewTable("Created", len(lookup)) for _, value := range lookup { outs.Add(value) } // Display images which aren't part of a repository/tag if job.Getenv("filter") == "" || filt_label { for _, image := range allImages { if !imageFilters.MatchKVList("label", image.ContainerConfig.Labels) { continue } out := &engine.Env{} out.SetJson("ParentId", image.Parent) out.SetList("RepoTags", []string{"<none>:<none>"}) out.SetList("RepoDigests", []string{"<none>@<none>"}) out.SetJson("Id", image.ID) out.SetInt64("Created", image.Created.Unix()) out.SetInt64("Size", image.Size) out.SetInt64("VirtualSize", image.GetParentsSize(0)+image.Size) out.SetJson("Labels", image.ContainerConfig.Labels) outs.Add(out) } } outs.ReverseSort() if _, err := outs.WriteListTo(job.Stdout); err != nil { return err } return nil }
func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, askedTag string, sf *utils.StreamFormatter, parallel bool) error { out.Write(sf.FormatStatus("", "Pulling repository %s", repoInfo.CanonicalName)) repoData, err := r.GetRepositoryData(repoInfo.RemoteName) if err != nil { if strings.Contains(err.Error(), "HTTP code: 404") { return fmt.Errorf("Error: image %s not found", utils.ImageReference(repoInfo.RemoteName, askedTag)) } // Unexpected HTTP error return err } log.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, repoInfo.RemoteName, repoData.Tokens) if err != nil { log.Errorf("unable to get remote tags: %s", err) return err } for tag, id := range tagsList { repoData.ImgList[id] = ®istry.ImgData{ ID: id, Tag: tag, Checksum: "", } } log.Debugf("Registering tags") // If no tag has been specified, pull them all if askedTag == "" { for tag, id := range tagsList { repoData.ImgList[id].Tag = tag } } else { // Otherwise, check that the tag exists and use only that one id, exists := tagsList[askedTag] if !exists { return fmt.Errorf("Tag %s not found in repository %s", askedTag, repoInfo.CanonicalName) } repoData.ImgList[id].Tag = askedTag } errors := make(chan error) layers_downloaded := false for _, image := range repoData.ImgList { downloadImage := func(img *registry.ImgData) { if askedTag != "" && img.Tag != askedTag { if parallel { errors <- nil } return } if img.Tag == "" { log.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) if parallel { errors <- nil } return } // ensure no two downloads of the same image happen at the same time if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil { if c != nil { out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil)) <-c out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil)) } else { log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err) } if parallel { errors <- nil } return } defer s.poolRemove("pull", "img:"+img.ID) out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, repoInfo.CanonicalName), nil)) success := false var lastErr, err error var is_downloaded bool for _, ep := range repoInfo.Index.Mirrors { out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, repoInfo.CanonicalName, ep), nil)) if is_downloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { // Don't report errors when pulling from mirrors. log.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err) continue } layers_downloaded = layers_downloaded || is_downloaded success = true break } if !success { for _, ep := range repoData.Endpoints { out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, repoInfo.CanonicalName, ep), nil)) if is_downloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { // It's not ideal that only the last error is returned, it would be better to concatenate the errors. // As the error is also given to the output stream the user will see the error. lastErr = err out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err), nil)) continue } layers_downloaded = layers_downloaded || is_downloaded success = true break } } if !success { err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, repoInfo.CanonicalName, lastErr) out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil)) if parallel { errors <- err return } } out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil)) if parallel { errors <- nil } } if parallel { go downloadImage(image) } else { downloadImage(image) } } if parallel { var lastError error for i := 0; i < len(repoData.ImgList); i++ { if err := <-errors; err != nil { lastError = err } } if lastError != nil { return lastError } } for tag, id := range tagsList { if askedTag != "" && tag != askedTag { continue } if err := s.Set(repoInfo.LocalName, tag, id, true); err != nil { return err } } requestedTag := repoInfo.CanonicalName if len(askedTag) > 0 { requestedTag = utils.ImageReference(repoInfo.CanonicalName, askedTag) } WriteStatus(requestedTag, out, sf, layers_downloaded) return nil }
func (cli *DockerCli) CmdPull(args ...string) error { cmd := cli.Subcmd("pull", "NAME[:TAG|@DIGEST]", "Pull an image or a repository from the registry", true) allTags := cmd.Bool([]string{"a", "-all-tags"}, false, "Download all tagged images in the repository") cmd.Require(flag.Exact, 1) utils.ParseFlags(cmd, args, true) var ( v = url.Values{} remote = cmd.Arg(0) newRemote = remote ) taglessRemote, tag := parsers.ParseRepositoryTag(remote) if tag == "" && !*allTags { newRemote = utils.ImageReference(taglessRemote, graph.DEFAULTTAG) } if tag != "" && *allTags { return fmt.Errorf("tag can't be used with --all-tags/-a") } v.Set("fromImage", newRemote) // Resolve the Repository name from fqn to RepositoryInfo repoInfo, err := registry.ParseRepositoryInfo(taglessRemote) if err != nil { return err } cli.LoadConfigFile() // Resolve the Auth config relevant for this server authConfig := cli.configFile.ResolveAuthConfig(repoInfo.Index) pull := func(authConfig registry.AuthConfig) error { buf, err := json.Marshal(authConfig) if err != nil { return err } registryAuthHeader := []string{ base64.URLEncoding.EncodeToString(buf), } return cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.out, map[string][]string{ "X-Registry-Auth": registryAuthHeader, }) } if err := pull(authConfig); err != nil { if strings.Contains(err.Error(), "Status 401") { fmt.Fprintln(cli.out, "\nPlease login prior to pull:") if err := cli.CmdLogin(repoInfo.Index.GetAuthConfigKey()); err != nil { return err } authConfig := cli.configFile.ResolveAuthConfig(repoInfo.Index) return pull(authConfig) } return err } return nil }
func (s *TagStore) CmdPull(job *engine.Job) error { if n := len(job.Args); n != 1 && n != 2 { return fmt.Errorf("Usage: %s IMAGE [TAG|DIGEST]", job.Name) } var ( localName = job.Args[0] tag string sf = utils.NewStreamFormatter(job.GetenvBool("json")) authConfig = ®istry.AuthConfig{} metaHeaders map[string][]string ) // Resolve the Repository name from fqn to RepositoryInfo repoInfo, err := registry.ResolveRepositoryInfo(job, localName) if err != nil { return err } if len(job.Args) > 1 { tag = job.Args[1] } job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", &metaHeaders) c, err := s.poolAdd("pull", utils.ImageReference(repoInfo.LocalName, tag)) if err != nil { if c != nil { // Another pull of the same repository is already taking place; just wait for it to finish job.Stdout.Write(sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", repoInfo.LocalName)) <-c return nil } return err } defer s.poolRemove("pull", utils.ImageReference(repoInfo.LocalName, tag)) log.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName) endpoint, err := repoInfo.GetEndpoint() if err != nil { return err } r, err := registry.NewSession(authConfig, registry.HTTPRequestFactory(metaHeaders), endpoint, true) if err != nil { return err } logName := repoInfo.LocalName if tag != "" { logName = utils.ImageReference(logName, tag) } if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { if repoInfo.Official { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { log.Errorf("error updating trust base graph: %s", err) } } log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err == nil { if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { log.Errorf("Error logging event 'pull' for %s: %s", logName, err) } return nil } else if err != registry.ErrDoesNotExist && err != ErrV2RegistryUnavailable { log.Errorf("Error from V2 registry: %s", err) } log.Debug("image does not exist on v2 registry, falling back to v1") } log.Debugf("pulling v1 repository with local name %q", repoInfo.LocalName) if err = s.pullRepository(r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err != nil { return err } if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { log.Errorf("Error logging event 'pull' for %s: %s", logName, err) } return nil }
func (p *v1Puller) pullRepository(askedTag string) error { out := p.config.OutStream out.Write(p.sf.FormatStatus("", "Pulling repository %s", p.repoInfo.CanonicalName)) repoData, err := p.session.GetRepositoryData(p.repoInfo.RemoteName) if err != nil { if strings.Contains(err.Error(), "HTTP code: 404") { return fmt.Errorf("Error: image %s not found", utils.ImageReference(p.repoInfo.RemoteName, askedTag)) } // Unexpected HTTP error return err } logrus.Debugf("Retrieving the tag list") tagsList := make(map[string]string) if askedTag == "" { tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo.RemoteName) } else { var tagID string tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo.RemoteName, askedTag) tagsList[askedTag] = tagID } if err != nil { if err == registry.ErrRepoNotFound && askedTag != "" { return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName) } logrus.Errorf("unable to get remote tags: %s", err) return err } for tag, id := range tagsList { repoData.ImgList[id] = ®istry.ImgData{ ID: id, Tag: tag, Checksum: "", } } logrus.Debugf("Registering tags") // If no tag has been specified, pull them all if askedTag == "" { for tag, id := range tagsList { repoData.ImgList[id].Tag = tag } } else { // Otherwise, check that the tag exists and use only that one id, exists := tagsList[askedTag] if !exists { return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName) } repoData.ImgList[id].Tag = askedTag } errors := make(chan error) layersDownloaded := false imgIDs := []string{} sessionID := p.session.ID() defer func() { p.graph.Release(sessionID, imgIDs...) }() for _, imgData := range repoData.ImgList { downloadImage := func(img *registry.ImgData) { if askedTag != "" && img.Tag != askedTag { errors <- nil return } if img.Tag == "" { logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) errors <- nil return } if err := image.ValidateID(img.ID); err != nil { errors <- err return } // ensure no two downloads of the same image happen at the same time poolKey := "img:" + img.ID broadcaster, found := p.poolAdd("pull", poolKey) broadcaster.Add(out) if found { errors <- broadcaster.Wait() return } defer p.poolRemove("pull", poolKey) // we need to retain it until tagging p.graph.Retain(sessionID, img.ID) imgIDs = append(imgIDs, img.ID) broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, p.repoInfo.CanonicalName), nil)) success := false var lastErr, err error var isDownloaded bool for _, ep := range p.repoInfo.Index.Mirrors { ep += "v1/" broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil)) if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil { // Don't report errors when pulling from mirrors. logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err) continue } layersDownloaded = layersDownloaded || isDownloaded success = true break } if !success { for _, ep := range repoData.Endpoints { broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil)) if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil { // It's not ideal that only the last error is returned, it would be better to concatenate the errors. // As the error is also given to the output stream the user will see the error. lastErr = err broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err), nil)) continue } layersDownloaded = layersDownloaded || isDownloaded success = true break } } if !success { err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.CanonicalName, lastErr) broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil)) errors <- err broadcaster.CloseWithError(err) return } broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil)) errors <- nil } go downloadImage(imgData) } var lastError error for i := 0; i < len(repoData.ImgList); i++ { if err := <-errors; err != nil { lastError = err } } if lastError != nil { return lastError } for tag, id := range tagsList { if askedTag != "" && tag != askedTag { continue } if err := p.Tag(p.repoInfo.LocalName, tag, id, true); err != nil { return err } } requestedTag := p.repoInfo.LocalName if len(askedTag) > 0 { requestedTag = utils.ImageReference(p.repoInfo.LocalName, askedTag) } writeStatus(requestedTag, out, p.sf, layersDownloaded) return nil }
func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, endpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) (bool, error) { log.Debugf("Pulling tag from V2 registry: %q", tag) manifestBytes, manifestDigest, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth) if err != nil { return false, err } // loadManifest ensures that the manifest payload has the expected digest // if the tag is a digest reference. manifest, verified, err := s.loadManifest(eng, manifestBytes, manifestDigest, tag) if err != nil { return false, fmt.Errorf("error verifying manifest: %s", err) } if err := checkValidManifest(manifest); err != nil { return false, err } if verified { log.Printf("Image manifest for %s has been verified", utils.ImageReference(repoInfo.CanonicalName, tag)) } out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName)) downloads := make([]downloadInfo, len(manifest.FSLayers)) for i := len(manifest.FSLayers) - 1; i >= 0; i-- { var ( sumStr = manifest.FSLayers[i].BlobSum imgJSON = []byte(manifest.History[i].V1Compatibility) ) img, err := image.NewImgJSON(imgJSON) if err != nil { return false, fmt.Errorf("failed to parse json: %s", err) } downloads[i].img = img // Check if exists if s.graph.Exists(img.ID) { log.Debugf("Image already exists: %s", img.ID) continue } dgst, err := digest.ParseDigest(sumStr) if err != nil { return false, err } downloads[i].digest = dgst out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil)) downloadFunc := func(di *downloadInfo) error { log.Debugf("pulling blob %q to V1 img %s", sumStr, img.ID) if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil { if c != nil { out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil)) <-c out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil)) } else { log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err) } } else { defer s.poolRemove("pull", "img:"+img.ID) tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob") if err != nil { return err } r, l, err := r.GetV2ImageBlobReader(endpoint, repoInfo.RemoteName, di.digest.Algorithm(), di.digest.Hex(), auth) if err != nil { return err } defer r.Close() verifier, err := digest.NewDigestVerifier(di.digest) if err != nil { return err } if _, err := io.Copy(tmpFile, progressreader.New(progressreader.Config{ In: ioutil.NopCloser(io.TeeReader(r, verifier)), Out: out, Formatter: sf, Size: int(l), NewLines: false, ID: stringid.TruncateID(img.ID), Action: "Downloading", })); err != nil { return fmt.Errorf("unable to copy v2 image blob data: %s", err) } out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Verifying Checksum", nil)) if !verifier.Verified() { log.Infof("Image verification failed: checksum mismatch for %q", di.digest.String()) verified = false } out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil)) log.Debugf("Downloaded %s to tempfile %s", img.ID, tmpFile.Name()) di.tmpFile = tmpFile di.length = l di.downloaded = true } di.imgJSON = imgJSON return nil } if parallel { downloads[i].err = make(chan error) go func(di *downloadInfo) { di.err <- downloadFunc(di) }(&downloads[i]) } else { err := downloadFunc(&downloads[i]) if err != nil { return false, err } } } var tagUpdated bool for i := len(downloads) - 1; i >= 0; i-- { d := &downloads[i] if d.err != nil { err := <-d.err if err != nil { return false, err } } if d.downloaded { // if tmpFile is empty assume download and extracted elsewhere defer os.Remove(d.tmpFile.Name()) defer d.tmpFile.Close() d.tmpFile.Seek(0, 0) if d.tmpFile != nil { err = s.graph.Register(d.img, progressreader.New(progressreader.Config{ In: d.tmpFile, Out: out, Formatter: sf, Size: int(d.length), ID: stringid.TruncateID(d.img.ID), Action: "Extracting", })) if err != nil { return false, err } // FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted) } out.Write(sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil)) tagUpdated = true } else { out.Write(sf.FormatProgress(stringid.TruncateID(d.img.ID), "Already exists", nil)) } } // Check for new tag if no layers downloaded if !tagUpdated { repo, err := s.Get(repoInfo.LocalName) if err != nil { return false, err } if repo != nil { if _, exists := repo[tag]; !exists { tagUpdated = true } } } if verified && tagUpdated { out.Write(sf.FormatStatus(utils.ImageReference(repoInfo.CanonicalName, tag), "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security.")) } if manifestDigest != "" { out.Write(sf.FormatStatus("", "Digest: %s", manifestDigest)) } if utils.DigestReference(tag) { if err = s.SetDigest(repoInfo.LocalName, tag, downloads[0].img.ID); err != nil { return false, err } } else { // only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest) if err = s.Set(repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil { return false, err } } return tagUpdated, nil }