func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoint string, token []string, sf *utils.StreamFormatter) error {
history, err := r.GetRemoteHistory(imgId, endpoint, token)
if err != nil {
return err
}
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
for _, id := range history {
if !srv.runtime.graph.Exists(id) {
out.Write(sf.FormatStatus("Pulling %s metadata", id))
imgJson, err := r.GetRemoteImageJson(id, endpoint, token)
if err != nil {
// FIXME: Keep goging in case of error?
return err
}
img, err := NewImgJson(imgJson)
if err != nil {
return fmt.Errorf("Failed to parse json: %s", err)
}
// Get the layer
out.Write(sf.FormatStatus("Pulling %s fs layer", id))
layer, contentLength, err := r.GetRemoteImageLayer(img.Id, endpoint, token)
if err != nil {
return err
}
if err := srv.runtime.graph.Register(utils.ProgressReader(layer, contentLength, out, sf.FormatProgress("Downloading", "%v/%v (%v)"), sf), false, img); err != nil {
return err
}
}
}
return nil
}
func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) error {
history, err := r.GetRemoteHistory(imgID, endpoint, token)
if err != nil {
return err
}
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
for _, id := range history {
if !srv.runtime.graph.Exists(id) {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "metadata"))
imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token)
if err != nil {
// FIXME: Keep going in case of error?
return err
}
img, err := NewImgJSON(imgJSON)
if err != nil {
return fmt.Errorf("Failed to parse json: %s", err)
}
// Get the layer
out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "fs layer"))
layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token)
if err != nil {
return err
}
defer layer.Close()
if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf, false), img); err != nil {
return err
}
}
}
return nil
}
// FIXME: Allow to interupt current push when new push of same image is done.
func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error {
if err := srv.poolAdd("push", name); err != nil {
return err
}
defer srv.poolRemove("push", name)
out = utils.NewWriteFlusher(out)
img, err := srv.runtime.graph.Get(name)
r, err2 := registry.NewRegistry(srv.runtime.root, authConfig)
if err2 != nil {
return err2
}
if err != nil {
out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name])))
// If it fails, try to get the repository
if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists {
if err := srv.pushRepository(r, out, name, endpoint, localRepo, sf); err != nil {
return err
}
return nil
}
return err
}
out.Write(sf.FormatStatus("The push refers to an image: [%s]", name))
if err := srv.pushImage(r, out, name, img.ID, endpoint, nil, sf); err != nil {
return err
}
return nil
}
func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils.StreamFormatter) (string, error) {
out = utils.NewWriteFlusher(out)
img, err := srv.runtime.repositories.LookupImage(name)
if err != nil {
return "", err
}
file, err := utils.Download(url, out)
if err != nil {
return "", err
}
defer file.Body.Close()
config, _, _, err := ParseRun([]string{img.ID, "echo", "insert", url, path}, srv.runtime.capabilities)
if err != nil {
return "", err
}
b := NewBuilder(srv.runtime)
c, err := b.Create(config)
if err != nil {
return "", err
}
if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("Downloading", "%v/%v (%v)"), sf), path); err != nil {
return "", err
}
// FIXME: Handle custom repo, tag comment, author
img, err = b.Commit(c, "", "", img.Comment, img.Author, nil)
if err != nil {
return "", err
}
out.Write(sf.FormatStatus(img.ID))
return img.ShortID(), nil
}
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error {
out = utils.NewWriteFlusher(out)
imgList, err := srv.getImageList(localRepo)
if err != nil {
return err
}
flattenedImgList := flatten(imgList)
out.Write(sf.FormatStatus("", "Sending image list"))
var repoData *registry.RepositoryData
repoData, err = r.PushImageJSONIndex(indexEp, remoteName, flattenedImgList, false, nil)
if err != nil {
return err
}
for _, ep := range repoData.Endpoints {
out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", localName, len(localRepo)))
// This section can not be parallelized (each round depends on the previous one)
for _, round := range imgList {
// FIXME: This section can be parallelized
for _, elem := range round {
var pushTags func() error
pushTags = func() error {
out.Write(sf.FormatStatus("", "Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag))
if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
return err
}
return nil
}
if _, exists := repoData.ImgList[elem.ID]; exists {
if err := pushTags(); err != nil {
return err
}
out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID))
continue
} else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) {
if err := pushTags(); err != nil {
return err
}
out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID))
continue
}
if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil {
// FIXME: Continue on error?
return err
} else {
elem.Checksum = checksum
}
return pushTags()
}
}
}
if _, err := r.PushImageJSONIndex(indexEp, remoteName, flattenedImgList, true, repoData.Endpoints); err != nil {
return err
}
return nil
}
func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId, ep string, token []string, sf *utils.StreamFormatter) error {
out = utils.NewWriteFlusher(out)
jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgId, "json"))
if err != nil {
return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgId, err)
}
out.Write(sf.FormatStatus("Pushing %s", imgId))
// Make sure we have the image's checksum
checksum, err := srv.getChecksum(imgId)
if err != nil {
return err
}
imgData := ®istry.ImgData{
ID: imgId,
Checksum: checksum,
}
// Send the json
if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil {
if err == registry.ErrAlreadyExists {
out.Write(sf.FormatStatus("Image %s already uploaded ; skipping", imgData.ID))
return nil
}
return err
}
// Retrieve the tarball to be sent
var layerData *TempArchive
// If the archive exists, use it
file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgId)))
if err != nil {
if os.IsNotExist(err) {
// If the archive does not exist, create one from the layer
layerData, err = srv.runtime.graph.TempLayerArchive(imgId, Xz, out)
if err != nil {
return fmt.Errorf("Failed to generate layer archive: %s", err)
}
} else {
return err
}
} else {
defer file.Close()
st, err := file.Stat()
if err != nil {
return err
}
layerData = &TempArchive{
File: file,
Size: st.Size(),
}
}
// Send the layer
if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%v/%v (%v)"), sf), ep, token); err != nil {
return err
}
return nil
}
// TempLayerArchive creates a temporary archive of the given image's filesystem layer.
// The archive is stored on disk and will be automatically deleted as soon as has been read.
// If output is not nil, a human-readable progress bar will be written to it.
// FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives?
func (graph *Graph) TempLayerArchive(id string, compression archive.Compression, sf *utils.StreamFormatter, output io.Writer) (*archive.TempArchive, error) {
image, err := graph.Get(id)
if err != nil {
return nil, err
}
tmp, err := graph.tmp()
if err != nil {
return nil, err
}
a, err := image.TarLayer(compression)
if err != nil {
return nil, err
}
return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp.Root)
}
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error {
out = utils.NewWriteFlusher(out)
out.Write(sf.FormatStatus("Processing checksums"))
imgList, err := srv.getImageList(localRepo)
if err != nil {
return err
}
out.Write(sf.FormatStatus("Sending image list"))
srvName := name
parts := strings.Split(name, "/")
if len(parts) > 2 {
srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/")))
}
var repoData *registry.RepositoryData
repoData, err = r.PushImageJSONIndex(indexEp, name, imgList, false, nil)
if err != nil {
return err
}
for _, ep := range repoData.Endpoints {
out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo)))
// For each image within the repo, push them
for _, elem := range imgList {
if _, exists := repoData.ImgList[elem.ID]; exists {
out.Write(sf.FormatStatus("Image %s already on registry, skipping", name))
continue
} else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) {
fmt.Fprintf(out, "Image %s already on registry, skipping\n", name)
continue
}
if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil {
// FIXME: Continue on error?
return err
}
out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+srvName+"/tags/"+elem.Tag))
if err := r.PushRegistryTag(srvName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
return err
}
}
}
if _, err := r.PushImageJSONIndex(indexEp, name, imgList, true, repoData.Endpoints); err != nil {
return err
}
return nil
}
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error {
out = utils.NewWriteFlusher(out)
imgList, err := srv.getImageList(localRepo)
if err != nil {
return err
}
out.Write(sf.FormatStatus("Sending image list"))
var repoData *registry.RepositoryData
repoData, err = r.PushImageJSONIndex(indexEp, remoteName, imgList, false, nil)
if err != nil {
return err
}
for _, ep := range repoData.Endpoints {
out.Write(sf.FormatStatus("Pushing repository %s (%d tags)", localName, len(localRepo)))
// For each image within the repo, push them
for _, elem := range imgList {
if _, exists := repoData.ImgList[elem.ID]; exists {
out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID))
continue
} else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) {
out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID))
continue
}
if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil {
// FIXME: Continue on error?
return err
} else {
elem.Checksum = checksum
}
out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag))
if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
return err
}
}
}
if _, err := r.PushImageJSONIndex(indexEp, remoteName, imgList, true, repoData.Endpoints); err != nil {
return err
}
return nil
}
func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) {
out = utils.NewWriteFlusher(out)
jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgID, "json"))
if err != nil {
return "", fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err)
}
out.Write(sf.FormatStatus("", "Pushing %s", imgID))
imgData := ®istry.ImgData{
ID: imgID,
}
// Send the json
if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil {
if err == registry.ErrAlreadyExists {
out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", imgData.ID))
return "", nil
}
return "", err
}
layerData, err := srv.runtime.graph.TempLayerArchive(imgID, Uncompressed, sf, out)
if err != nil {
return "", fmt.Errorf("Failed to generate layer archive: %s", err)
}
// Send the layer
if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil {
return "", err
} else {
imgData.Checksum = checksum
}
// Send the checksum
if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil {
return "", err
}
return imgData.Checksum, nil
}
// FIXME: Allow to interrupt current push when new push of same image is done.
func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig, metaHeaders map[string][]string) error {
if err := srv.poolAdd("push", localName); err != nil {
return err
}
defer srv.poolRemove("push", localName)
// Resolve the Repository name from fqn to endpoint + name
endpoint, remoteName, err := registry.ResolveRepositoryName(localName)
if err != nil {
return err
}
out = utils.NewWriteFlusher(out)
img, err := srv.runtime.graph.Get(localName)
r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory(metaHeaders))
if err2 != nil {
return err2
}
if err != nil {
reposLen := len(srv.runtime.repositories.Repositories[localName])
out.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", localName, reposLen))
// If it fails, try to get the repository
if localRepo, exists := srv.runtime.repositories.Repositories[localName]; exists {
if err := srv.pushRepository(r, out, localName, remoteName, localRepo, endpoint, sf); err != nil {
return err
}
return nil
}
return err
}
var token []string
out.Write(sf.FormatStatus("", "The push refers to an image: [%s]", localName))
if _, err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil {
return err
}
return nil
}
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string, sf *utils.StreamFormatter) error {
out = utils.NewWriteFlusher(out)
out.Write(sf.FormatStatus("Processing checksums"))
imgList, err := srv.getImageList(localRepo)
if err != nil {
return err
}
out.Write(sf.FormatStatus("Sending image list"))
repoData, err := r.PushImageJSONIndex(name, imgList, false)
if err != nil {
return err
}
for _, ep := range repoData.Endpoints {
out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo)))
// For each image within the repo, push them
for _, elem := range imgList {
if _, exists := repoData.ImgList[elem.ID]; exists {
out.Write(sf.FormatStatus("Image %s already on registry, skipping", name))
continue
}
if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil {
// FIXME: Continue on error?
return err
}
out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"/users/"+name+"/"+elem.Tag))
if err := r.PushRegistryTag(name, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
return err
}
}
}
if _, err := r.PushImageJSONIndex(name, imgList, true); err != nil {
return err
}
return nil
}
func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Writer, sf *utils.StreamFormatter) error {
var archive io.Reader
var resp *http.Response
if src == "-" {
archive = in
} else {
u, err := url.Parse(src)
if err != nil {
return err
}
if u.Scheme == "" {
u.Scheme = "http"
u.Host = src
u.Path = ""
}
out.Write(sf.FormatStatus("Downloading from %s", u))
// Download with curl (pretty progress bar)
// If curl is not available, fallback to http.Get()
resp, err = utils.Download(u.String(), out)
if err != nil {
return err
}
archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("Importing", "%v/%v (%v)"), sf)
}
img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil)
if err != nil {
return err
}
// Optionally register the image at REPO/TAG
if repo != "" {
if err := srv.runtime.repositories.Set(repo, tag, img.ID, true); err != nil {
return err
}
}
out.Write(sf.FormatStatus(img.ShortID()))
return nil
}
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error {
out.Write(sf.FormatStatus("", "Pulling repository %s", localName))
repoData, err := r.GetRepositoryData(indexEp, remoteName)
if err != nil {
return err
}
utils.Debugf("Retrieving the tag list")
tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens)
if err != nil {
utils.Debugf("%v", err)
return err
}
for tag, id := range tagsList {
repoData.ImgList[id] = ®istry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
utils.Debugf("Registering tags")
// If no tag has been specified, pull them all
if askedTag == "" {
for tag, id := range tagsList {
repoData.ImgList[id].Tag = tag
}
} else {
// Otherwise, check that the tag exists and use only that one
id, exists := tagsList[askedTag]
if !exists {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName)
}
repoData.ImgList[id].Tag = askedTag
}
errors := make(chan error)
for _, image := range repoData.ImgList {
downloadImage := func(img *registry.ImgData) {
if askedTag != "" && img.Tag != askedTag {
utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID)
errors <- nil
return
}
if img.Tag == "" {
utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
errors <- nil
return
}
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s", img.Tag, localName)))
success := false
for _, ep := range repoData.Endpoints {
if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
continue
}
success = true
break
}
if !success {
errors <- fmt.Errorf("Could not find repository on any of the indexed registries.")
}
errors <- nil
}
if parallel {
go downloadImage(image)
} else {
downloadImage(image)
}
}
if parallel {
for i := 0; i < len(repoData.ImgList); i++ {
if err := <-errors; err != nil {
return err
}
}
}
for tag, id := range tagsList {
if askedTag != "" && tag != askedTag {
continue
}
if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil {
return err
}
}
if err := srv.runtime.repositories.Save(); err != nil {
return err
}
return nil
}
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, remote, askedTag string, sf *utils.StreamFormatter) error {
out.Write(sf.FormatStatus("Pulling repository %s from %s", remote, auth.IndexServerAddress()))
repoData, err := r.GetRepositoryData(remote)
if err != nil {
return err
}
utils.Debugf("Updating checksums")
// Reload the json file to make sure not to overwrite faster sums
if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
return err
}
utils.Debugf("Retrieving the tag list")
tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens)
if err != nil {
return err
}
utils.Debugf("Registering tags")
// If not specific tag have been asked, take all
if askedTag == "" {
for tag, id := range tagsList {
repoData.ImgList[id].Tag = tag
}
} else {
// Otherwise, check that the tag exists and use only that one
if id, exists := tagsList[askedTag]; !exists {
return fmt.Errorf("Tag %s not found in repositoy %s", askedTag, remote)
} else {
repoData.ImgList[id].Tag = askedTag
}
}
for _, img := range repoData.ImgList {
if askedTag != "" && img.Tag != askedTag {
utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.Id)
continue
}
out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.Id, img.Tag, remote))
success := false
for _, ep := range repoData.Endpoints {
if err := srv.pullImage(r, out, img.Id, "https://"+ep+"/v1", repoData.Tokens, sf); err != nil {
out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
continue
}
success = true
break
}
if !success {
return fmt.Errorf("Could not find repository on any of the indexed registries.")
}
}
for tag, id := range tagsList {
if askedTag != "" && tag != askedTag {
continue
}
if err := srv.runtime.repositories.Set(remote, tag, id, true); err != nil {
return err
}
}
if err := srv.runtime.repositories.Save(); err != nil {
return err
}
return nil
}
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, registryEp string, localRepo map[string]string, sf *utils.StreamFormatter) error {
out = utils.NewWriteFlusher(out)
out.Write(sf.FormatStatus("Processing checksums"))
imgList, err := srv.getImageList(localRepo)
if err != nil {
return err
}
out.Write(sf.FormatStatus("Sending image list"))
srvName := name
parts := strings.Split(name, "/")
if len(parts) > 2 {
srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/")))
}
var repoData *registry.RepositoryData
if registryEp == "" {
repoData, err = r.PushImageJSONIndex(name, imgList, false, nil)
if err != nil {
return err
}
} else {
repoData = ®istry.RepositoryData{
ImgList: make(map[string]*registry.ImgData),
Tokens: []string{},
Endpoints: []string{registryEp},
}
tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens)
if err != nil && err.Error() != "Repository not found" {
return err
} else if err == nil {
for tag, id := range tagsList {
repoData.ImgList[id] = ®istry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
}
}
for _, ep := range repoData.Endpoints {
if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) {
ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep)
}
out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo)))
// For each image within the repo, push them
for _, elem := range imgList {
if _, exists := repoData.ImgList[elem.ID]; exists {
out.Write(sf.FormatStatus("Image %s already on registry, skipping", name))
continue
} else if registryEp != "" && r.LookupRemoteImage(elem.ID, registryEp, repoData.Tokens) {
fmt.Fprintf(out, "Image %s already on registry, skipping\n", name)
continue
}
if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil {
// FIXME: Continue on error?
return err
}
out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"/repositories/"+srvName+"/tags/"+elem.Tag))
if err := r.PushRegistryTag(srvName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
return err
}
}
}
if registryEp == "" {
if _, err := r.PushImageJSONIndex(name, imgList, true, repoData.Endpoints); err != nil {
return err
}
}
return nil
}
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag, registryEp string, sf *utils.StreamFormatter) error {
out.Write(sf.FormatStatus("Pulling repository %s from %s", local, auth.IndexServerAddress()))
var repoData *registry.RepositoryData
var err error
if registryEp == "" {
repoData, err = r.GetRepositoryData(remote)
if err != nil {
return err
}
utils.Debugf("Updating checksums")
// Reload the json file to make sure not to overwrite faster sums
if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
return err
}
} else {
repoData = ®istry.RepositoryData{
Tokens: []string{},
ImgList: make(map[string]*registry.ImgData),
Endpoints: []string{registryEp},
}
}
utils.Debugf("Retrieving the tag list")
tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens)
if err != nil {
utils.Debugf("%v", err)
return err
}
if registryEp != "" {
for tag, id := range tagsList {
repoData.ImgList[id] = ®istry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
}
utils.Debugf("Registering tags")
// If no tag has been specified, pull them all
if askedTag == "" {
for tag, id := range tagsList {
repoData.ImgList[id].Tag = tag
}
} else {
// Otherwise, check that the tag exists and use only that one
id, exists := tagsList[askedTag]
if !exists {
return fmt.Errorf("Tag %s not found in repositoy %s", askedTag, local)
}
repoData.ImgList[id].Tag = askedTag
}
for _, img := range repoData.ImgList {
if askedTag != "" && img.Tag != askedTag {
utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID)
continue
}
out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, remote))
success := false
for _, ep := range repoData.Endpoints {
if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) {
ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep)
}
if err := srv.pullImage(r, out, img.ID, ep+"/v1", repoData.Tokens, sf); err != nil {
out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
continue
}
success = true
break
}
if !success {
return fmt.Errorf("Could not find repository on any of the indexed registries.")
}
}
for tag, id := range tagsList {
if askedTag != "" && tag != askedTag {
continue
}
if err := srv.runtime.repositories.Set(local, tag, id, true); err != nil {
return err
}
}
if err := srv.runtime.repositories.Save(); err != nil {
return err
}
return nil
}
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error {
out.Write(sf.FormatStatus("", "Pulling repository %s", localName))
repoData, err := r.GetRepositoryData(indexEp, remoteName)
if err != nil {
return err
}
utils.Debugf("Retrieving the tag list")
tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens)
if err != nil {
utils.Debugf("%v", err)
return err
}
for tag, id := range tagsList {
repoData.ImgList[id] = ®istry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
utils.Debugf("Registering tags")
// If no tag has been specified, pull them all
if askedTag == "" {
for tag, id := range tagsList {
repoData.ImgList[id].Tag = tag
}
} else {
// Otherwise, check that the tag exists and use only that one
id, exists := tagsList[askedTag]
if !exists {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName)
}
repoData.ImgList[id].Tag = askedTag
}
errors := make(chan error)
for _, image := range repoData.ImgList {
downloadImage := func(img *registry.ImgData) {
if askedTag != "" && img.Tag != askedTag {
utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID)
if parallel {
errors <- nil
}
return
}
if img.Tag == "" {
utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
if parallel {
errors <- nil
}
return
}
// ensure no two downloads of the same image happen at the same time
if err := srv.poolAdd("pull", "img:"+img.ID); err != nil {
utils.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
if parallel {
errors <- nil
}
return
}
defer srv.poolRemove("pull", "img:"+img.ID)
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s", img.Tag, localName)))
success := false
var lastErr error
for _, ep := range repoData.Endpoints {
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s, endpoint: %s", img.Tag, localName, ep)))
if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
// Its not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Error pulling", fmt.Sprintf("image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err)))
continue
}
success = true
break
}
if !success {
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Error pulling", fmt.Sprintf("image (%s) from %s, %s", img.Tag, localName, lastErr)))
if parallel {
errors <- fmt.Errorf("Could not find repository on any of the indexed registries.")
return
}
}
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download", "complete"))
if parallel {
errors <- nil
}
}
if parallel {
go downloadImage(image)
} else {
downloadImage(image)
}
}
if parallel {
var lastError error
for i := 0; i < len(repoData.ImgList); i++ {
if err := <-errors; err != nil {
lastError = err
}
}
if lastError != nil {
return lastError
}
}
for tag, id := range tagsList {
if askedTag != "" && tag != askedTag {
continue
}
if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil {
return err
}
}
if err := srv.runtime.repositories.Save(); err != nil {
return err
}
return nil
}
func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) error {
history, err := r.GetRemoteHistory(imgID, endpoint, token)
if err != nil {
return err
}
out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling", "dependend layers"))
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
for _, id := range history {
// ensure no two downloads of the same layer happen at the same time
if err := srv.poolAdd("pull", "layer:"+id); err != nil {
utils.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err)
return nil
}
defer srv.poolRemove("pull", "layer:"+id)
if !srv.runtime.graph.Exists(id) {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "metadata"))
imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token)
if err != nil {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers"))
// FIXME: Keep going in case of error?
return err
}
img, err := NewImgJSON(imgJSON)
if err != nil {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers"))
return fmt.Errorf("Failed to parse json: %s", err)
}
// Get the layer
out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "fs layer"))
layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token)
if err != nil {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers"))
return err
}
defer layer.Close()
if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf, false), img); err != nil {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "downloading dependend layers"))
return err
}
}
out.Write(sf.FormatProgress(utils.TruncateID(id), "Download", "complete"))
}
return nil
}
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter) error {
out.Write(sf.FormatStatus("Pulling repository %s", localName))
repoData, err := r.GetRepositoryData(indexEp, remoteName)
if err != nil {
return err
}
utils.Debugf("Updating checksums")
// Reload the json file to make sure not to overwrite faster sums
if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
return err
}
utils.Debugf("Retrieving the tag list")
tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens)
if err != nil {
utils.Debugf("%v", err)
return err
}
for tag, id := range tagsList {
repoData.ImgList[id] = ®istry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
utils.Debugf("Registering tags")
// If no tag has been specified, pull them all
if askedTag == "" {
for tag, id := range tagsList {
repoData.ImgList[id].Tag = tag
}
} else {
// Otherwise, check that the tag exists and use only that one
id, exists := tagsList[askedTag]
if !exists {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName)
}
repoData.ImgList[id].Tag = askedTag
}
for _, img := range repoData.ImgList {
if askedTag != "" && img.Tag != askedTag {
utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID)
continue
}
if img.Tag == "" {
utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
continue
}
out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName))
success := false
for _, ep := range repoData.Endpoints {
if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
continue
}
success = true
break
}
if !success {
return fmt.Errorf("Could not find repository on any of the indexed registries.")
}
}
for tag, id := range tagsList {
if askedTag != "" && tag != askedTag {
continue
}
if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil {
return err
}
}
if err := srv.runtime.repositories.Save(); err != nil {
return err
}
return nil
}
