func (s *TagStore) pullAndMergeImage(r *registry.Session, out io.Writer, containerID, containerImage, imgID, endpoint string, token []string, sf *utils.StreamFormatter) (bool, error) {
newHistory, err := r.GetRemoteHistory(imgID, endpoint, token)
if err != nil {
return false, err
}
oldHistory, err := r.GetRemoteHistory(containerImage, endpoint, token)
if err != nil {
return false, err
}
// Compare the differences between the two image
compareHistory := make(map[string]string, len(oldHistory))
for _, id := range oldHistory {
compareHistory[id] = id
}
var history []string
for _, id := range newHistory {
if _, ok := compareHistory[id]; !ok {
history = append(history, id)
}
}
layers_downloaded := false
for i := len(history) - 1; i >= 0; i-- {
id := history[i]
// ensure no two downloads of the same layer happen at the same time
if c, err := s.poolAdd("pull", "layer:"+id); err != nil {
log.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err)
<-c
}
defer s.poolRemove("pull", "layer:"+id)
out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling metadata", nil))
var (
imgJSON []byte
imgSize int
err error
img *image.Image
)
retries := 5
for j := 1; j <= retries; j++ {
imgJSON, imgSize, err = r.GetRemoteImageJSON(id, endpoint, token)
if err != nil && j == retries {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
return layers_downloaded, err
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
}
img, err = image.NewImgJSON(imgJSON)
layers_downloaded = true
if err != nil && j == retries {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
return layers_downloaded, fmt.Errorf("Failed to parse json: %s", err)
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else {
break
}
}
for j := 1; j <= retries; j++ {
// Get the layer
status := "Pulling fs layer"
if j > 1 {
status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
}
out.Write(sf.FormatProgress(utils.TruncateID(id), status, nil))
layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token, int64(imgSize))
if uerr, ok := err.(*url.Error); ok {
err = uerr.Err
}
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
return layers_downloaded, err
}
layers_downloaded = true
defer layer.Close()
if !s.graph.Exists(id) {
// register when first pull layer
err = s.graph.Register(img, imgJSON,
utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading"))
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error downloading dependent layers", nil))
return layers_downloaded, err
}
}
// add layer to container
dest := path.Join(s.graph.Driver().MountPath(), containerID, "rootfs")
out.Write(sf.FormatProgress(utils.TruncateID(id), fmt.Sprintf("Merge layer to container rootfs %s", dest), nil))
err = archive.ApplyLayer(dest, layer)
if err != nil && j == retries {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error merge layers", nil))
return layers_downloaded, err
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else {
break
}
}
}
return layers_downloaded, nil
}
func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) (bool, error) {
history, err := r.GetRemoteHistory(imgID, endpoint, token)
if err != nil {
return false, err
}
out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling dependent layers", nil))
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
layers_downloaded := false
for i := len(history) - 1; i >= 0; i-- {
id := history[i]
// ensure no two downloads of the same layer happen at the same time
if c, err := s.poolAdd("pull", "layer:"+id); err != nil {
log.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err)
<-c
}
defer s.poolRemove("pull", "layer:"+id)
if !s.graph.Exists(id) {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling metadata", nil))
var (
imgJSON []byte
imgSize int
err error
img *image.Image
)
retries := 5
for j := 1; j <= retries; j++ {
imgJSON, imgSize, err = r.GetRemoteImageJSON(id, endpoint, token)
if err != nil && j == retries {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
return layers_downloaded, err
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
}
img, err = image.NewImgJSON(imgJSON)
layers_downloaded = true
if err != nil && j == retries {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
return layers_downloaded, fmt.Errorf("Failed to parse json: %s", err)
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else {
break
}
}
for j := 1; j <= retries; j++ {
// Get the layer
status := "Pulling fs layer"
if j > 1 {
status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
}
out.Write(sf.FormatProgress(utils.TruncateID(id), status, nil))
layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token, int64(imgSize))
if uerr, ok := err.(*url.Error); ok {
err = uerr.Err
}
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
return layers_downloaded, err
}
layers_downloaded = true
defer layer.Close()
err = s.graph.Register(img,
utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading"))
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
out.Write(sf.FormatProgress(utils.TruncateID(id), "Error downloading dependent layers", nil))
return layers_downloaded, err
} else {
break
}
}
}
out.Write(sf.FormatProgress(utils.TruncateID(id), "Download complete", nil))
}
return layers_downloaded, nil
}
// TODO rewrite this whole PoC
func main() {
flag.Usage = func() {
flag.PrintDefaults()
}
flag.Parse()
if debug {
os.Setenv("DEBUG", "1")
log.SetLevel(log.DebugLevel)
}
if flag.NArg() == 0 {
fmt.Println("ERROR: no image names provided")
flag.Usage()
os.Exit(1)
}
// make tempDir
tempDir, err := ioutil.TempDir("", "docker-fetch-")
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
defer os.RemoveAll(tempDir)
fetcher := NewFetcher(tempDir)
sc := registry.NewServiceConfig(rOptions)
for _, arg := range flag.Args() {
remote, tagName := parsers.ParseRepositoryTag(arg)
if tagName == "" {
tagName = "latest"
}
repInfo, err := sc.NewRepositoryInfo(remote)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
log.Debugf("%#v %q\n", repInfo, tagName)
idx, err := repInfo.GetEndpoint()
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "Pulling %s:%s from %s\n", repInfo.RemoteName, tagName, idx)
var session *registry.Session
if s, ok := fetcher.sessions[idx.String()]; ok {
session = s
} else {
// TODO(vbatts) obviously the auth and http factory shouldn't be nil here
session, err = registry.NewSession(nil, nil, idx, timeout)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
rd, err := session.GetRepositoryData(repInfo.RemoteName)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
log.Debugf("rd: %#v", rd)
// produce the "repositories" file for the archive
if _, ok := fetcher.repositories[repInfo.RemoteName]; !ok {
fetcher.repositories[repInfo.RemoteName] = graph.Repository{}
}
log.Debugf("repositories: %#v", fetcher.repositories)
if len(rd.Endpoints) == 0 {
log.Fatalf("expected registry endpoints, but received none from the index")
}
tags, err := session.GetRemoteTags(rd.Endpoints, repInfo.RemoteName, rd.Tokens)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if hash, ok := tags[tagName]; ok {
fetcher.repositories[repInfo.RemoteName][tagName] = hash
}
log.Debugf("repositories: %#v", fetcher.repositories)
imgList, err := session.GetRemoteHistory(fetcher.repositories[repInfo.RemoteName][tagName], rd.Endpoints[0], rd.Tokens)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
log.Debugf("imgList: %#v", imgList)
for _, imgID := range imgList {
// pull layers and jsons
buf, _, err := session.GetRemoteImageJSON(imgID, rd.Endpoints[0], rd.Tokens)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err = os.MkdirAll(filepath.Join(fetcher.Root, imgID), 0755); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fh, err := os.Create(filepath.Join(fetcher.Root, imgID, "json"))
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if _, err = fh.Write(buf); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fh.Close()
log.Debugf("%s", fh.Name())
tarRdr, err := session.GetRemoteImageLayer(imgID, rd.Endpoints[0], rd.Tokens, 0)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fh, err = os.Create(filepath.Join(fetcher.Root, imgID, "layer.tar"))
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
// the body is usually compressed
gzRdr, err := gzip.NewReader(tarRdr)
if err != nil {
log.Debugf("image layer for %q is not gzipped", imgID)
// the archive may not be gzipped, so just copy the stream
if _, err = io.Copy(fh, tarRdr); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
} else {
// no error, so gzip decompress the stream
if _, err = io.Copy(fh, gzRdr); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err = gzRdr.Close(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
if err = tarRdr.Close(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err = fh.Close(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
log.Debugf("%s", fh.Name())
}
}
// marshal the "repositories" file for writing out
log.Debugf("repositories: %q", fetcher.repositories)
buf, err := json.Marshal(fetcher.repositories)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fh, err := os.Create(filepath.Join(fetcher.Root, "repositories"))
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if _, err = fh.Write(buf); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fh.Close()
log.Debugf("%s", fh.Name())
var output io.WriteCloser
if outputStream == "-" {
output = os.Stdout
} else {
output, err = os.Create(outputStream)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
defer output.Close()
if err = os.Chdir(fetcher.Root); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
tarStream, err := archive.Tar(".", archive.Uncompressed)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if _, err = io.Copy(output, tarStream); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}