func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoint string, token []string, sf *utils.StreamFormatter) error { history, err := r.GetRemoteHistory(imgId, endpoint, token) if err != nil { return err } // FIXME: Try to stream the images? // FIXME: Launch the getRemoteImage() in goroutines for _, id := range history { if !srv.runtime.graph.Exists(id) { out.Write(sf.FormatStatus("Pulling %s metadata", id)) imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token) if err != nil { // FIXME: Keep goging in case of error? return err } img, err := NewImgJSON(imgJSON) if err != nil { return fmt.Errorf("Failed to parse json: %s", err) } // Get the layer out.Write(sf.FormatStatus("Pulling %s fs layer", id)) layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token) if err != nil { return err } defer layer.Close() if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%v/%v (%v)"), sf), false, img); err != nil { return err } } } return nil }
func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils.StreamFormatter) (string, error) { out = utils.NewWriteFlusher(out) img, err := srv.runtime.repositories.LookupImage(name) if err != nil { return "", err } file, err := utils.Download(url, out) if err != nil { return "", err } defer file.Body.Close() config, _, _, err := ParseRun([]string{img.ID, "echo", "insert", url, path}, srv.runtime.capabilities) if err != nil { return "", err } b := NewBuilder(srv.runtime) c, err := b.Create(config) if err != nil { return "", err } if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("Downloading", "%v/%v (%v)"), sf), path); err != nil { return "", err } // FIXME: Handle custom repo, tag comment, author img, err = b.Commit(c, "", "", img.Comment, img.Author, nil) if err != nil { return "", err } out.Write(sf.FormatStatus(img.ID)) return img.ShortID(), nil }
// TempLayerArchive creates a temporary archive of the given image's filesystem layer. // The archive is stored on disk and will be automatically deleted as soon as has been read. // If output is not nil, a human-readable progress bar will be written to it. // FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives? func (graph *Graph) TempLayerArchive(id string, compression archive.Compression, sf *utils.StreamFormatter, output io.Writer) (*archive.TempArchive, error) { image, err := graph.Get(id) if err != nil { return nil, err } tmp, err := graph.tmp() if err != nil { return nil, err } a, err := image.TarLayer(compression) if err != nil { return nil, err } return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp.Root) }
func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Writer, sf *utils.StreamFormatter) error { var archive io.Reader var resp *http.Response if src == "-" { archive = in } else { u, err := url.Parse(src) if err != nil { return err } if u.Scheme == "" { u.Scheme = "http" u.Host = src u.Path = "" } out.Write(sf.FormatStatus("Downloading from %s", u)) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() resp, err = utils.Download(u.String(), out) if err != nil { return err } archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("Importing", "%v/%v (%v)"), sf) } img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil) if err != nil { return err } // Optionally register the image at REPO/TAG if repo != "" { if err := srv.runtime.repositories.Set(repo, tag, img.ID, true); err != nil { return err } } out.Write(sf.FormatStatus(img.ShortID())) return nil }
func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId, ep string, token []string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgId, "json")) if err != nil { return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgId, err) } out.Write(sf.FormatStatus("Pushing %s", imgId)) // Make sure we have the image's checksum checksum, err := srv.getChecksum(imgId) if err != nil { return err } imgData := ®istry.ImgData{ ID: imgId, Checksum: checksum, } // Send the json if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { out.Write(sf.FormatStatus("Image %s already uploaded ; skipping", imgData.ID)) return nil } return err } // Retrieve the tarball to be sent var layerData *TempArchive // If the archive exists, use it file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgId))) if err != nil { if os.IsNotExist(err) { // If the archive does not exist, create one from the layer layerData, err = srv.runtime.graph.TempLayerArchive(imgId, Xz, out) if err != nil { return fmt.Errorf("Failed to generate layer archive: %s", err) } } else { return err } } else { defer file.Close() st, err := file.Stat() if err != nil { return err } layerData = &TempArchive{ File: file, Size: st.Size(), } } // Send the layer if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%v/%v (%v)"), sf), ep, token); err != nil { return err } return nil }
func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) { out = utils.NewWriteFlusher(out) jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgID, "json")) if err != nil { return "", fmt.Errorf("Error while retrieving the path for {%s}: %s", imgID, err) } out.Write(sf.FormatStatus("", "Pushing %s", imgID)) imgData := ®istry.ImgData{ ID: imgID, } // Send the json if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", imgData.ID)) return "", nil } return "", err } layerData, err := srv.runtime.graph.TempLayerArchive(imgID, Uncompressed, sf, out) if err != nil { return "", fmt.Errorf("Failed to generate layer archive: %s", err) } // Send the layer if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf, false), ep, token, jsonRaw); err != nil { return "", err } else { imgData.Checksum = checksum } out.Write(sf.FormatStatus("", "")) // Send the checksum if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { return "", err } return imgData.Checksum, nil }
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error { out.Write(sf.FormatStatus("", "Pulling repository %s", localName)) repoData, err := r.GetRepositoryData(indexEp, remoteName) if err != nil { return err } utils.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens) if err != nil { utils.Debugf("%v", err) return err } for tag, id := range tagsList { repoData.ImgList[id] = ®istry.ImgData{ ID: id, Tag: tag, Checksum: "", } } utils.Debugf("Registering tags") // If no tag has been specified, pull them all if askedTag == "" { for tag, id := range tagsList { repoData.ImgList[id].Tag = tag } } else { // Otherwise, check that the tag exists and use only that one id, exists := tagsList[askedTag] if !exists { return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName) } repoData.ImgList[id].Tag = askedTag } errors := make(chan error) for _, image := range repoData.ImgList { downloadImage := func(img *registry.ImgData) { if askedTag != "" && img.Tag != askedTag { utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID) if parallel { errors <- nil } return } if img.Tag == "" { utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) if parallel { errors <- nil } return } // ensure no two downloads of the same image happen at the same time if err := srv.poolAdd("pull", "img:"+img.ID); err != nil { utils.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err) if parallel { errors <- nil } return } defer srv.poolRemove("pull", "img:"+img.ID) out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s", img.Tag, localName))) success := false var lastErr error for _, ep := range repoData.Endpoints { out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s, endpoint: %s", img.Tag, localName, ep))) if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { // Its not ideal that only the last error is returned, it would be better to concatenate the errors. // As the error is also given to the output stream the user will see the error. lastErr = err out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Error pulling", fmt.Sprintf("image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err))) continue } success = true break } if !success { out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Error pulling", fmt.Sprintf("image (%s) from %s, %s", img.Tag, localName, lastErr))) if parallel { errors <- fmt.Errorf("Could not find repository on any of the indexed registries.") return } } out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download", "complete")) if parallel { errors <- nil } } if parallel { go downloadImage(image) } else { downloadImage(image) } } if parallel { var lastError error for i := 0; i < len(repoData.ImgList); i++ { if err := <-errors; err != nil { lastError = err } } if lastError != nil { return lastError } } for tag, id := range tagsList { if askedTag != "" && tag != askedTag { continue } if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil { return err } } if err := srv.runtime.repositories.Save(); err != nil { return err } return nil }
func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) error { history, err := r.GetRemoteHistory(imgID, endpoint, token) if err != nil { return err } out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling", "dependend layers")) // FIXME: Try to stream the images? // FIXME: Launch the getRemoteImage() in goroutines for _, id := range history { // ensure no two downloads of the same layer happen at the same time if err := srv.poolAdd("pull", "layer:"+id); err != nil { utils.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err) return nil } defer srv.poolRemove("pull", "layer:"+id) if !srv.runtime.graph.Exists(id) { out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "metadata")) imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token) if err != nil { out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers")) // FIXME: Keep going in case of error? return err } img, err := NewImgJSON(imgJSON) if err != nil { out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers")) return fmt.Errorf("Failed to parse json: %s", err) } // Get the layer out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "fs layer")) layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token) if err != nil { out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers")) return err } defer layer.Close() if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf, false), img); err != nil { out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "downloading dependend layers")) return err } } out.Write(sf.FormatProgress(utils.TruncateID(id), "Download", "complete")) } return nil }
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error { out.Write(sf.FormatStatus("", "Pulling repository %s", localName)) repoData, err := r.GetRepositoryData(indexEp, remoteName) if err != nil { return err } utils.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens) if err != nil { utils.Debugf("%v", err) return err } for tag, id := range tagsList { repoData.ImgList[id] = ®istry.ImgData{ ID: id, Tag: tag, Checksum: "", } } utils.Debugf("Registering tags") // If no tag has been specified, pull them all if askedTag == "" { for tag, id := range tagsList { repoData.ImgList[id].Tag = tag } } else { // Otherwise, check that the tag exists and use only that one id, exists := tagsList[askedTag] if !exists { return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName) } repoData.ImgList[id].Tag = askedTag } errors := make(chan error) for _, image := range repoData.ImgList { downloadImage := func(img *registry.ImgData) { if askedTag != "" && img.Tag != askedTag { utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID) errors <- nil return } if img.Tag == "" { utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) errors <- nil return } out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s", img.Tag, localName))) success := false for _, ep := range repoData.Endpoints { if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) continue } success = true break } if !success { errors <- fmt.Errorf("Could not find repository on any of the indexed registries.") } errors <- nil } if parallel { go downloadImage(image) } else { downloadImage(image) } } if parallel { for i := 0; i < len(repoData.ImgList); i++ { if err := <-errors; err != nil { return err } } } for tag, id := range tagsList { if askedTag != "" && tag != askedTag { continue } if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil { return err } } if err := srv.runtime.repositories.Save(); err != nil { return err } return nil }