// loadImage fetches the image with the given id from the graph. func (graph *Graph) loadImage(id string) (*image.Image, error) { root := graph.imageRoot(id) // Open the JSON file to decode by streaming jsonSource, err := os.Open(jsonPath(root)) if err != nil { return nil, err } defer jsonSource.Close() img := &image.Image{} dec := json.NewDecoder(jsonSource) // Decode the JSON data if err := dec.Decode(img); err != nil { return nil, err } if img.ID == "" { img.ID = id } if img.Parent == "" && img.ParentID != "" && img.ParentID.Validate() == nil { img.Parent = img.ParentID.Hex() } // compatibilityID for parent parent, err := ioutil.ReadFile(filepath.Join(root, parentFileName)) if err == nil && len(parent) > 0 { img.Parent = string(parent) } if err := image.ValidateID(img.ID); err != nil { return nil, err } if buf, err := ioutil.ReadFile(filepath.Join(root, layersizeFileName)); err != nil { if !os.IsNotExist(err) { return nil, err } // If the layersize file does not exist then set the size to a negative number // because a layer size of 0 (zero) is valid img.Size = -1 } else { // Using Atoi here instead would temporarily convert the size to a machine // dependent integer type, which causes images larger than 2^31 bytes to // display negative sizes on 32-bit machines: size, err := strconv.ParseInt(string(buf), 10, 64) if err != nil { return nil, err } img.Size = int64(size) } return img, nil }
func validateRemoteName(remoteName string) error { if !strings.Contains(remoteName, "/") { // the repository name must not be a valid image ID if err := image.ValidateID(remoteName); err == nil { return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", remoteName) } } return v2.ValidateRepositoryName(remoteName) }
func (s *TagStore) recursiveLoad(address, tmpImageDir string) error { if _, err := s.LookupImage(address); err != nil { logrus.Debugf("Loading %s", address) imageJSON, err := ioutil.ReadFile(filepath.Join(tmpImageDir, "repo", address, "json")) if err != nil { logrus.Debugf("Error reading json: %v", err) return err } layer, err := os.Open(filepath.Join(tmpImageDir, "repo", address, "layer.tar")) if err != nil { logrus.Debugf("Error reading embedded tar: %v", err) return err } img, err := image.NewImgJSON(imageJSON) if err != nil { logrus.Debugf("Error unmarshalling json: %v", err) return err } if err := image.ValidateID(img.ID); err != nil { logrus.Debugf("Error validating ID: %v", err) return err } // ensure no two downloads of the same layer happen at the same time poolKey := "layer:" + img.ID broadcaster, found := s.poolAdd("pull", poolKey) if found { logrus.Debugf("Image (id: %s) load is already running, waiting", img.ID) return broadcaster.Wait() } defer s.poolRemove("pull", poolKey) if img.Parent != "" { if !s.graph.Exists(img.Parent) { if err := s.recursiveLoad(img.Parent, tmpImageDir); err != nil { return err } } } if err := s.graph.Register(v1Descriptor{img}, layer); err != nil { return err } } logrus.Debugf("Completed processing %s", address) return nil }
// Register imports a pre-existing image into the graph. // Returns nil if the image is already registered. func (graph *Graph) Register(im image.Descriptor, layerData io.Reader) (err error) { imgID := im.ID() if err := image.ValidateID(imgID); err != nil { return err } // this is needed cause pull_v2 attemptIDReuse could deadlock graph.imagesMutex.Lock() defer graph.imagesMutex.Unlock() // We need this entire operation to be atomic within the engine. Note that // this doesn't mean Register is fully safe yet. graph.imageMutex.Lock(imgID) defer graph.imageMutex.Unlock(imgID) return graph.register(im, layerData) }
// fixManifestLayers removes repeated layers from the manifest and checks the // correctness of the parent chain. func fixManifestLayers(m *manifest.Manifest) error { images := make([]*image.Image, len(m.FSLayers)) for i := range m.FSLayers { img, err := image.NewImgJSON([]byte(m.History[i].V1Compatibility)) if err != nil { return err } images[i] = img if err := image.ValidateID(img.ID); err != nil { return err } } if images[len(images)-1].Parent != "" { return errors.New("Invalid parent ID in the base layer of the image.") } // check general duplicates to error instead of a deadlock idmap := make(map[string]struct{}) var lastID string for _, img := range images { // skip IDs that appear after each other, we handle those later if _, exists := idmap[img.ID]; img.ID != lastID && exists { return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID) } lastID = img.ID idmap[lastID] = struct{}{} } // backwards loop so that we keep the remaining indexes after removing items for i := len(images) - 2; i >= 0; i-- { if images[i].ID == images[i+1].ID { // repeated ID. remove and continue m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...) m.History = append(m.History[:i], m.History[i+1:]...) } else if images[i].Parent != images[i+1].ID { return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", images[i+1].ID, images[i].Parent) } } return nil }
func (p *v1Puller) pullRepository(askedTag string) error { out := p.config.OutStream out.Write(p.sf.FormatStatus("", "Pulling repository %s", p.repoInfo.CanonicalName)) repoData, err := p.session.GetRepositoryData(p.repoInfo.RemoteName) if err != nil { if strings.Contains(err.Error(), "HTTP code: 404") { return fmt.Errorf("Error: image %s not found", utils.ImageReference(p.repoInfo.RemoteName, askedTag)) } // Unexpected HTTP error return err } logrus.Debugf("Retrieving the tag list") tagsList := make(map[string]string) if askedTag == "" { tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo.RemoteName) } else { var tagID string tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo.RemoteName, askedTag) tagsList[askedTag] = tagID } if err != nil { if err == registry.ErrRepoNotFound && askedTag != "" { return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName) } logrus.Errorf("unable to get remote tags: %s", err) return err } for tag, id := range tagsList { repoData.ImgList[id] = ®istry.ImgData{ ID: id, Tag: tag, Checksum: "", } } logrus.Debugf("Registering tags") // If no tag has been specified, pull them all if askedTag == "" { for tag, id := range tagsList { repoData.ImgList[id].Tag = tag } } else { // Otherwise, check that the tag exists and use only that one id, exists := tagsList[askedTag] if !exists { return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName) } repoData.ImgList[id].Tag = askedTag } errors := make(chan error) layersDownloaded := false imgIDs := []string{} sessionID := p.session.ID() defer func() { p.graph.Release(sessionID, imgIDs...) }() for _, imgData := range repoData.ImgList { downloadImage := func(img *registry.ImgData) { if askedTag != "" && img.Tag != askedTag { errors <- nil return } if img.Tag == "" { logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) errors <- nil return } if err := image.ValidateID(img.ID); err != nil { errors <- err return } // ensure no two downloads of the same image happen at the same time poolKey := "img:" + img.ID broadcaster, found := p.poolAdd("pull", poolKey) broadcaster.Add(out) if found { errors <- broadcaster.Wait() return } defer p.poolRemove("pull", poolKey) // we need to retain it until tagging p.graph.Retain(sessionID, img.ID) imgIDs = append(imgIDs, img.ID) broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, p.repoInfo.CanonicalName), nil)) success := false var lastErr, err error var isDownloaded bool for _, ep := range p.repoInfo.Index.Mirrors { ep += "v1/" broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil)) if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil { // Don't report errors when pulling from mirrors. logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err) continue } layersDownloaded = layersDownloaded || isDownloaded success = true break } if !success { for _, ep := range repoData.Endpoints { broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil)) if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil { // It's not ideal that only the last error is returned, it would be better to concatenate the errors. // As the error is also given to the output stream the user will see the error. lastErr = err broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err), nil)) continue } layersDownloaded = layersDownloaded || isDownloaded success = true break } } if !success { err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.CanonicalName, lastErr) broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil)) errors <- err broadcaster.CloseWithError(err) return } broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil)) errors <- nil } go downloadImage(imgData) } var lastError error for i := 0; i < len(repoData.ImgList); i++ { if err := <-errors; err != nil { lastError = err } } if lastError != nil { return lastError } for tag, id := range tagsList { if askedTag != "" && tag != askedTag { continue } if err := p.Tag(p.repoInfo.LocalName, tag, id, true); err != nil { return err } } requestedTag := p.repoInfo.LocalName if len(askedTag) > 0 { requestedTag = utils.ImageReference(p.repoInfo.LocalName, askedTag) } writeStatus(requestedTag, out, p.sf, layersDownloaded) return nil }