func (p *v2Puller) pullV2Repository(tag string) (err error) { var tags []string taggedName := p.repoInfo.LocalName if len(tag) > 0 { tags = []string{tag} taggedName = utils.ImageReference(p.repoInfo.LocalName, tag) } else { var err error manSvc, err := p.repo.Manifests(context.Background()) if err != nil { return err } tags, err = manSvc.Tags() if err != nil { return err } } poolKey := "v2:" + taggedName broadcaster, found := p.poolAdd("pull", poolKey) broadcaster.Add(p.config.OutStream) if found { // Another pull of the same repository is already taking place; just wait for it to finish return broadcaster.Wait() } // This must use a closure so it captures the value of err when the // function returns, not when the 'defer' is evaluated. defer func() { p.poolRemoveWithError("pull", poolKey, err) }() var layersDownloaded bool for _, tag := range tags { // pulledNew is true if either new layers were downloaded OR if existing images were newly tagged // TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus? pulledNew, err := p.pullV2Tag(broadcaster, tag, taggedName) if err != nil { return err } layersDownloaded = layersDownloaded || pulledNew } writeStatus(taggedName, broadcaster, p.sf, layersDownloaded) return nil }
func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (verified bool, err error) { logrus.Debugf("Pulling tag from V2 registry: %q", tag) manSvc, err := p.repo.Manifests(context.Background()) if err != nil { return false, err } manifest, err := manSvc.GetByTag(tag) if err != nil { return false, err } verified, err = p.validateManifest(manifest, tag) if err != nil { return false, err } if verified { logrus.Printf("Image manifest for %s has been verified", taggedName) } out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name())) var downloads []*downloadInfo var layerIDs []string defer func() { p.graph.Release(p.sessionID, layerIDs...) for _, d := range downloads { p.poolRemoveWithError("pull", d.poolKey, err) if d.tmpFile != nil { d.tmpFile.Close() if err := os.RemoveAll(d.tmpFile.Name()); err != nil { logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name()) } } } }() for i := len(manifest.FSLayers) - 1; i >= 0; i-- { img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility)) if err != nil { logrus.Debugf("error getting image v1 json: %v", err) return false, err } p.graph.Retain(p.sessionID, img.ID) layerIDs = append(layerIDs, img.ID) // Check if exists if p.graph.Exists(img.ID) { logrus.Debugf("Image already exists: %s", img.ID) out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Already exists", nil)) continue } out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil)) d := &downloadInfo{ img: img, poolKey: "layer:" + img.ID, digest: manifest.FSLayers[i].BlobSum, // TODO: seems like this chan buffer solved hanging problem in go1.5, // this can indicate some deeper problem that somehow we never take // error from channel in loop below err: make(chan error, 1), } tmpFile, err := ioutil.TempFile("", "GetImageBlob") if err != nil { return false, err } d.tmpFile = tmpFile downloads = append(downloads, d) broadcaster, found := p.poolAdd("pull", d.poolKey) broadcaster.Add(out) d.broadcaster = broadcaster if found { d.err <- nil } else { go p.download(d) } } var tagUpdated bool for _, d := range downloads { if err := <-d.err; err != nil { return false, err } if d.layer == nil { // Wait for a different pull to download and extract // this layer. err = d.broadcaster.Wait() if err != nil { return false, err } continue } d.tmpFile.Seek(0, 0) reader := progressreader.New(progressreader.Config{ In: d.tmpFile, Out: d.broadcaster, Formatter: p.sf, Size: d.size, NewLines: false, ID: stringid.TruncateID(d.img.ID), Action: "Extracting", }) err = p.graph.Register(d.img, reader) if err != nil { return false, err } if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil { return false, err } d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil)) d.broadcaster.Close() tagUpdated = true } manifestDigest, _, err := digestFromManifest(manifest, p.repoInfo.LocalName) if err != nil { return false, err } // Check for new tag if no layers downloaded if !tagUpdated { repo, err := p.Get(p.repoInfo.LocalName) if err != nil { return false, err } if repo != nil { if _, exists := repo[tag]; !exists { tagUpdated = true } } else { tagUpdated = true } } if verified && tagUpdated { out.Write(p.sf.FormatStatus(p.repo.Name()+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security.")) } firstID := layerIDs[len(layerIDs)-1] if utils.DigestReference(tag) { // TODO(stevvooe): Ideally, we should always set the digest so we can // use the digest whether we pull by it or not. Unfortunately, the tag // store treats the digest as a separate tag, meaning there may be an // untagged digest image that would seem to be dangling by a user. if err = p.SetDigest(p.repoInfo.LocalName, tag, firstID); err != nil { return false, err } } else { // only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest) if err = p.Tag(p.repoInfo.LocalName, tag, firstID, true); err != nil { return false, err } } if manifestDigest != "" { out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest)) } return tagUpdated, nil }
func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (tagUpdated bool, err error) { logrus.Debugf("Pulling tag from V2 registry: %q", tag) manSvc, err := p.repo.Manifests(context.Background()) if err != nil { return false, err } unverifiedManifest, err := manSvc.GetByTag(tag) if err != nil { return false, err } if unverifiedManifest == nil { return false, fmt.Errorf("image manifest does not exist for tag %q", tag) } var verifiedManifest *schema1.Manifest verifiedManifest, err = verifyManifest(unverifiedManifest, tag) if err != nil { return false, err } // remove duplicate layers and check parent chain validity err = fixManifestLayers(verifiedManifest) if err != nil { return false, err } imgs, err := p.getImageInfos(verifiedManifest) if err != nil { return false, err } out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name())) var downloads []*downloadInfo var layerIDs []string defer func() { p.graph.Release(p.sessionID, layerIDs...) for _, d := range downloads { p.poolRemoveWithError("pull", d.poolKey, err) if d.tmpFile != nil { d.tmpFile.Close() if err := os.RemoveAll(d.tmpFile.Name()); err != nil { logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name()) } } } }() for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- { img := imgs[i] p.graph.Retain(p.sessionID, img.id) layerIDs = append(layerIDs, img.id) p.graph.imageMutex.Lock(img.id) // Check if exists if p.graph.Exists(img.id) { if err := p.validateImageInGraph(img.id, imgs, i); err != nil { p.graph.imageMutex.Unlock(img.id) return false, fmt.Errorf("image validation failed: %v", err) } logrus.Debugf("Image already exists: %s", img.id) p.graph.imageMutex.Unlock(img.id) continue } p.graph.imageMutex.Unlock(img.id) out.Write(p.sf.FormatProgress(stringid.TruncateID(img.id), "Pulling fs layer", nil)) d := &downloadInfo{ img: img, imgIndex: i, poolKey: "v2layer:" + img.id, digest: verifiedManifest.FSLayers[i].BlobSum, // TODO: seems like this chan buffer solved hanging problem in go1.5, // this can indicate some deeper problem that somehow we never take // error from channel in loop below err: make(chan error, 1), } tmpFile, err := ioutil.TempFile("", "GetImageBlob") if err != nil { return false, err } d.tmpFile = tmpFile downloads = append(downloads, d) broadcaster, found := p.poolAdd("pull", d.poolKey) broadcaster.Add(out) d.broadcaster = broadcaster if found { d.err <- nil } else { go p.download(d) } } for _, d := range downloads { if err := <-d.err; err != nil { return false, err } if d.layer == nil { // Wait for a different pull to download and extract // this layer. err = d.broadcaster.Wait() if err != nil { return false, err } continue } d.tmpFile.Seek(0, 0) err := func() error { reader := progressreader.New(progressreader.Config{ In: d.tmpFile, Out: d.broadcaster, Formatter: p.sf, Size: d.size, NewLines: false, ID: stringid.TruncateID(d.img.id), Action: "Extracting", }) p.graph.imagesMutex.Lock() defer p.graph.imagesMutex.Unlock() p.graph.imageMutex.Lock(d.img.id) defer p.graph.imageMutex.Unlock(d.img.id) // Must recheck the data on disk if any exists. // This protects against races where something // else is written to the graph under this ID // after attemptIDReuse. if p.graph.Exists(d.img.id) { if err := p.validateImageInGraph(d.img.id, imgs, d.imgIndex); err != nil { return fmt.Errorf("image validation failed: %v", err) } } if err := p.graph.register(d.img, reader); err != nil { return err } if err := p.graph.setLayerDigest(d.img.id, d.digest); err != nil { return err } if err := p.graph.setV1CompatibilityConfig(d.img.id, d.img.v1Compatibility); err != nil { return err } return nil }() if err != nil { return false, err } d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Pull complete", nil)) d.broadcaster.Close() tagUpdated = true } manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName) if err != nil { return false, err } // Check for new tag if no layers downloaded if !tagUpdated { repo, err := p.Get(p.repoInfo.LocalName) if err != nil { return false, err } if repo != nil { if _, exists := repo[tag]; !exists { tagUpdated = true } } else { tagUpdated = true } } firstID := layerIDs[len(layerIDs)-1] if utils.DigestReference(tag) { // TODO(stevvooe): Ideally, we should always set the digest so we can // use the digest whether we pull by it or not. Unfortunately, the tag // store treats the digest as a separate tag, meaning there may be an // untagged digest image that would seem to be dangling by a user. if err = p.SetDigest(p.repoInfo.LocalName, tag, firstID); err != nil { return false, err } } else { // only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest) if err = p.Tag(p.repoInfo.LocalName, tag, firstID, true); err != nil { return false, err } } if manifestDigest != "" { out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest)) } return tagUpdated, nil }
func (p *v2Puller) pullV2Tag(out io.Writer, ref reference.Named) (tagUpdated bool, err error) { tagOrDigest := "" if tagged, isTagged := ref.(reference.Tagged); isTagged { tagOrDigest = tagged.Tag() } else if digested, isDigested := ref.(reference.Digested); isDigested { tagOrDigest = digested.Digest().String() } else { return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String()) } logrus.Debugf("Pulling ref from V2 registry: %q", tagOrDigest) manSvc, err := p.repo.Manifests(context.Background()) if err != nil { return false, err } unverifiedManifest, err := manSvc.GetByTag(tagOrDigest) if err != nil { return false, err } if unverifiedManifest == nil { return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest) } var verifiedManifest *schema1.Manifest verifiedManifest, err = verifyManifest(unverifiedManifest, ref) if err != nil { return false, err } rootFS := image.NewRootFS() if err := detectBaseLayer(p.config.ImageStore, verifiedManifest, rootFS); err != nil { return false, err } // remove duplicate layers and check parent chain validity err = fixManifestLayers(verifiedManifest) if err != nil { return false, err } out.Write(p.sf.FormatStatus(tagOrDigest, "Pulling from %s", p.repo.Name())) var downloads []*downloadInfo defer func() { for _, d := range downloads { p.config.Pool.removeWithError(d.poolKey, err) if d.tmpFile != nil { d.tmpFile.Close() if err := os.RemoveAll(d.tmpFile.Name()); err != nil { logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name()) } } } }() // Image history converted to the new format var history []image.History poolKey := "v2layer:" notFoundLocally := false // Note that the order of this loop is in the direction of bottom-most // to top-most, so that the downloads slice gets ordered correctly. for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- { blobSum := verifiedManifest.FSLayers[i].BlobSum poolKey += blobSum.String() var throwAway struct { ThrowAway bool `json:"throwaway,omitempty"` } if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil { return false, err } h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway) if err != nil { return false, err } history = append(history, h) if throwAway.ThrowAway { continue } // Do we have a layer on disk corresponding to the set of // blobsums up to this point? if !notFoundLocally { notFoundLocally = true diffID, err := p.blobSumService.GetDiffID(blobSum) if err == nil { rootFS.Append(diffID) if l, err := p.config.LayerStore.Get(rootFS.ChainID()); err == nil { notFoundLocally = false logrus.Debugf("Layer already exists: %s", blobSum.String()) out.Write(p.sf.FormatProgress(stringid.TruncateID(blobSum.String()), "Already exists", nil)) defer layer.ReleaseAndLog(p.config.LayerStore, l) continue } else { rootFS.DiffIDs = rootFS.DiffIDs[:len(rootFS.DiffIDs)-1] } } } out.Write(p.sf.FormatProgress(stringid.TruncateID(blobSum.String()), "Pulling fs layer", nil)) tmpFile, err := ioutil.TempFile("", "GetImageBlob") if err != nil { return false, err } d := &downloadInfo{ poolKey: poolKey, digest: blobSum, tmpFile: tmpFile, // TODO: seems like this chan buffer solved hanging problem in go1.5, // this can indicate some deeper problem that somehow we never take // error from channel in loop below err: make(chan error, 1), } downloads = append(downloads, d) broadcaster, found := p.config.Pool.add(d.poolKey) broadcaster.Add(out) d.broadcaster = broadcaster if found { d.err <- nil } else { go p.download(d) } } for _, d := range downloads { if err := <-d.err; err != nil { return false, err } if d.layer == nil { // Wait for a different pull to download and extract // this layer. err = d.broadcaster.Wait() if err != nil { return false, err } diffID, err := p.blobSumService.GetDiffID(d.digest) if err != nil { return false, err } rootFS.Append(diffID) l, err := p.config.LayerStore.Get(rootFS.ChainID()) if err != nil { return false, err } defer layer.ReleaseAndLog(p.config.LayerStore, l) continue } d.tmpFile.Seek(0, 0) reader := progressreader.New(progressreader.Config{ In: d.tmpFile, Out: d.broadcaster, Formatter: p.sf, Size: d.size, NewLines: false, ID: stringid.TruncateID(d.digest.String()), Action: "Extracting", }) inflatedLayerData, err := archive.DecompressStream(reader) if err != nil { return false, fmt.Errorf("could not get decompression stream: %v", err) } l, err := p.config.LayerStore.Register(inflatedLayerData, rootFS.ChainID()) if err != nil { return false, fmt.Errorf("failed to register layer: %v", err) } logrus.Debugf("layer %s registered successfully", l.DiffID()) rootFS.Append(l.DiffID()) // Cache mapping from this layer's DiffID to the blobsum if err := p.blobSumService.Add(l.DiffID(), d.digest); err != nil { return false, err } defer layer.ReleaseAndLog(p.config.LayerStore, l) d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.digest.String()), "Pull complete", nil)) d.broadcaster.Close() tagUpdated = true } config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), rootFS, history) if err != nil { return false, err } imageID, err := p.config.ImageStore.Create(config) if err != nil { return false, err } manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName.Name()) if err != nil { return false, err } // Check for new tag if no layers downloaded var oldTagImageID image.ID if !tagUpdated { oldTagImageID, err = p.config.TagStore.Get(ref) if err != nil || oldTagImageID != imageID { tagUpdated = true } } if tagUpdated { if canonical, ok := ref.(reference.Canonical); ok { if err = p.config.TagStore.AddDigest(canonical, imageID, true); err != nil { return false, err } } else if err = p.config.TagStore.AddTag(ref, imageID, true); err != nil { return false, err } } if manifestDigest != "" { out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest)) } return tagUpdated, nil }