func (s *TagStore) pullMRepository(r *registry.Session, out io.Writer, containerID, containerImage, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool, mirrors []string) error {
out.Write(sf.FormatStatus("", "Pulling (parallel:%v) repository %s:%s", parallel, localName, askedTag))
if askedTag == "" {
return fmt.Errorf("Error: tag is empty")
}
repoData, err := r.GetRepositoryData(remoteName)
if err != nil {
if strings.Contains(err.Error(), "HTTP code: 404") {
return fmt.Errorf("Error: image %s not found", remoteName)
}
// Unexpected HTTP error
return err
}
log.Debugf("Retrieving the tag list")
tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens)
if err != nil {
log.Errorf("%v", err)
return err
}
id, exists := tagsList[askedTag]
if !exists {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName)
}
img, exists := repoData.ImgList[id]
if !exists {
return fmt.Errorf("Image ID %s not found in repository %s", id, localName)
}
img.Tag = askedTag
var lastErr error
success := false
var is_downloaded bool
if mirrors != nil {
for _, ep := range mirrors {
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling and merging image (%s) from %s, mirror: %s", img.Tag, localName, ep), nil))
if is_downloaded, err = s.pullAndMergeImage(r, out, containerID, containerImage, img.ID, ep, repoData.Tokens, sf); err != nil {
// Don't report errors when pulling from mirrors.
log.Debugf("Error pulling and merging image (%s) from %s, mirror: %s, %s", img.Tag, localName, ep, err)
continue
}
success = true
break
}
}
if !success {
for _, ep := range repoData.Endpoints {
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling and merging image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil))
if is_downloaded, err = s.pullAndMergeImage(r, out, containerID, containerImage, img.ID, ep, repoData.Tokens, sf); err != nil {
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling and merging image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil))
continue
}
success = true
break
}
}
if !success {
err := fmt.Errorf("Error pulling and merging image (%s) from %s, %v", img.Tag, localName, lastErr)
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), err.Error(), nil))
return err
}
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Layer is downloaded ? %v", is_downloaded), nil))
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Complete", nil))
if err = s.Set(localName, img.Tag, img.ID, true); err != nil {
return err
}
return nil
}
func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, askedTag string, sf *utils.StreamFormatter, parallel bool) error {
out.Write(sf.FormatStatus("", "Pulling repository %s", repoInfo.CanonicalName))
repoData, err := r.GetRepositoryData(repoInfo.RemoteName)
if err != nil {
if strings.Contains(err.Error(), "HTTP code: 404") {
return fmt.Errorf("Error: image %s:%s not found", repoInfo.RemoteName, askedTag)
}
// Unexpected HTTP error
return err
}
log.Debugf("Retrieving the tag list")
tagsList, err := r.GetRemoteTags(repoData.Endpoints, repoInfo.RemoteName, repoData.Tokens)
if err != nil {
log.Errorf("unable to get remote tags: %s", err)
return err
}
for tag, id := range tagsList {
repoData.ImgList[id] = ®istry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
log.Debugf("Registering tags")
// If no tag has been specified, pull them all
var imageId string
if askedTag == "" {
for tag, id := range tagsList {
repoData.ImgList[id].Tag = tag
}
} else {
// Otherwise, check that the tag exists and use only that one
id, exists := tagsList[askedTag]
if !exists {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, repoInfo.CanonicalName)
}
imageId = id
repoData.ImgList[id].Tag = askedTag
}
errors := make(chan error)
layers_downloaded := false
for _, image := range repoData.ImgList {
downloadImage := func(img *registry.ImgData) {
if askedTag != "" && img.Tag != askedTag {
if parallel {
errors <- nil
}
return
}
if img.Tag == "" {
log.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
if parallel {
errors <- nil
}
return
}
// ensure no two downloads of the same image happen at the same time
if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil {
if c != nil {
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil))
<-c
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil))
} else {
log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
}
if parallel {
errors <- nil
}
return
}
defer s.poolRemove("pull", "img:"+img.ID)
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, repoInfo.CanonicalName), nil))
success := false
var lastErr, err error
var is_downloaded bool
for _, ep := range repoInfo.Index.Mirrors {
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, repoInfo.CanonicalName, ep), nil))
if is_downloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
// Don't report errors when pulling from mirrors.
log.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err)
continue
}
layers_downloaded = layers_downloaded || is_downloaded
success = true
break
}
if !success {
for _, ep := range repoData.Endpoints {
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, repoInfo.CanonicalName, ep), nil))
if is_downloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err), nil))
continue
}
layers_downloaded = layers_downloaded || is_downloaded
success = true
break
}
}
if !success {
err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, repoInfo.CanonicalName, lastErr)
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), err.Error(), nil))
if parallel {
errors <- err
return
}
}
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil))
if parallel {
errors <- nil
}
}
if parallel {
go downloadImage(image)
} else {
downloadImage(image)
}
}
if parallel {
var lastError error
for i := 0; i < len(repoData.ImgList); i++ {
if err := <-errors; err != nil {
lastError = err
}
}
if lastError != nil {
return lastError
}
}
for tag, id := range tagsList {
if askedTag != "" && id != imageId {
continue
}
if err := s.Set(repoInfo.LocalName, tag, id, true); err != nil {
return err
}
}
requestedTag := repoInfo.CanonicalName
if len(askedTag) > 0 {
requestedTag = repoInfo.CanonicalName + ":" + askedTag
}
WriteStatus(requestedTag, out, sf, layers_downloaded)
return nil
}
// TODO rewrite this whole PoC
func main() {
flag.Usage = func() {
flag.PrintDefaults()
}
flag.Parse()
if debug {
os.Setenv("DEBUG", "1")
log.SetLevel(log.DebugLevel)
}
if flag.NArg() == 0 {
fmt.Println("ERROR: no image names provided")
flag.Usage()
os.Exit(1)
}
// make tempDir
tempDir, err := ioutil.TempDir("", "docker-fetch-")
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
defer os.RemoveAll(tempDir)
fetcher := NewFetcher(tempDir)
sc := registry.NewServiceConfig(rOptions)
for _, arg := range flag.Args() {
remote, tagName := parsers.ParseRepositoryTag(arg)
if tagName == "" {
tagName = "latest"
}
repInfo, err := sc.NewRepositoryInfo(remote)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
log.Debugf("%#v %q\n", repInfo, tagName)
idx, err := repInfo.GetEndpoint()
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "Pulling %s:%s from %s\n", repInfo.RemoteName, tagName, idx)
var session *registry.Session
if s, ok := fetcher.sessions[idx.String()]; ok {
session = s
} else {
// TODO(vbatts) obviously the auth and http factory shouldn't be nil here
session, err = registry.NewSession(nil, nil, idx, timeout)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
rd, err := session.GetRepositoryData(repInfo.RemoteName)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
log.Debugf("rd: %#v", rd)
// produce the "repositories" file for the archive
if _, ok := fetcher.repositories[repInfo.RemoteName]; !ok {
fetcher.repositories[repInfo.RemoteName] = graph.Repository{}
}
log.Debugf("repositories: %#v", fetcher.repositories)
if len(rd.Endpoints) == 0 {
log.Fatalf("expected registry endpoints, but received none from the index")
}
tags, err := session.GetRemoteTags(rd.Endpoints, repInfo.RemoteName, rd.Tokens)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if hash, ok := tags[tagName]; ok {
fetcher.repositories[repInfo.RemoteName][tagName] = hash
}
log.Debugf("repositories: %#v", fetcher.repositories)
imgList, err := session.GetRemoteHistory(fetcher.repositories[repInfo.RemoteName][tagName], rd.Endpoints[0], rd.Tokens)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
log.Debugf("imgList: %#v", imgList)
for _, imgID := range imgList {
// pull layers and jsons
buf, _, err := session.GetRemoteImageJSON(imgID, rd.Endpoints[0], rd.Tokens)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err = os.MkdirAll(filepath.Join(fetcher.Root, imgID), 0755); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fh, err := os.Create(filepath.Join(fetcher.Root, imgID, "json"))
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if _, err = fh.Write(buf); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fh.Close()
log.Debugf("%s", fh.Name())
tarRdr, err := session.GetRemoteImageLayer(imgID, rd.Endpoints[0], rd.Tokens, 0)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fh, err = os.Create(filepath.Join(fetcher.Root, imgID, "layer.tar"))
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
// the body is usually compressed
gzRdr, err := gzip.NewReader(tarRdr)
if err != nil {
log.Debugf("image layer for %q is not gzipped", imgID)
// the archive may not be gzipped, so just copy the stream
if _, err = io.Copy(fh, tarRdr); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
} else {
// no error, so gzip decompress the stream
if _, err = io.Copy(fh, gzRdr); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err = gzRdr.Close(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
if err = tarRdr.Close(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err = fh.Close(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
log.Debugf("%s", fh.Name())
}
}
// marshal the "repositories" file for writing out
log.Debugf("repositories: %q", fetcher.repositories)
buf, err := json.Marshal(fetcher.repositories)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fh, err := os.Create(filepath.Join(fetcher.Root, "repositories"))
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if _, err = fh.Write(buf); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fh.Close()
log.Debugf("%s", fh.Name())
var output io.WriteCloser
if outputStream == "-" {
output = os.Stdout
} else {
output, err = os.Create(outputStream)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
defer output.Close()
if err = os.Chdir(fetcher.Root); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
tarStream, err := archive.Tar(".", archive.Uncompressed)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if _, err = io.Copy(output, tarStream); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}