Exemple #1
0
func (s *TagStore) pullAndMergeImage(r *registry.Session, out io.Writer, containerID, containerImage, imgID, endpoint string, token []string, sf *utils.StreamFormatter) (bool, error) {
	newHistory, err := r.GetRemoteHistory(imgID, endpoint, token)
	if err != nil {
		return false, err
	}
	oldHistory, err := r.GetRemoteHistory(containerImage, endpoint, token)
	if err != nil {
		return false, err
	}
	// Compare the differences between the two image
	compareHistory := make(map[string]string, len(oldHistory))
	for _, id := range oldHistory {
		compareHistory[id] = id
	}
	var history []string
	for _, id := range newHistory {
		if _, ok := compareHistory[id]; !ok {
			history = append(history, id)
		}
	}

	layers_downloaded := false
	for i := len(history) - 1; i >= 0; i-- {
		id := history[i]

		// ensure no two downloads of the same layer happen at the same time
		if c, err := s.poolAdd("pull", "layer:"+id); err != nil {
			log.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err)
			<-c
		}
		defer s.poolRemove("pull", "layer:"+id)

		out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling metadata", nil))
		var (
			imgJSON []byte
			imgSize int
			err     error
			img     *image.Image
		)
		retries := 5
		for j := 1; j <= retries; j++ {
			imgJSON, imgSize, err = r.GetRemoteImageJSON(id, endpoint, token)
			if err != nil && j == retries {
				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
				return layers_downloaded, err
			} else if err != nil {
				time.Sleep(time.Duration(j) * 500 * time.Millisecond)
				continue
			}
			img, err = image.NewImgJSON(imgJSON)
			layers_downloaded = true
			if err != nil && j == retries {
				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
				return layers_downloaded, fmt.Errorf("Failed to parse json: %s", err)
			} else if err != nil {
				time.Sleep(time.Duration(j) * 500 * time.Millisecond)
				continue
			} else {
				break
			}
		}

		for j := 1; j <= retries; j++ {
			// Get the layer
			status := "Pulling fs layer"
			if j > 1 {
				status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
			}
			out.Write(sf.FormatProgress(utils.TruncateID(id), status, nil))
			layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token, int64(imgSize))
			if uerr, ok := err.(*url.Error); ok {
				err = uerr.Err
			}
			if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
				time.Sleep(time.Duration(j) * 500 * time.Millisecond)
				continue
			} else if err != nil {
				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
				return layers_downloaded, err
			}
			layers_downloaded = true
			defer layer.Close()
			if !s.graph.Exists(id) {
				// register when first pull layer
				err = s.graph.Register(img, imgJSON,
					utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading"))
				if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
					continue
				} else if err != nil {
					out.Write(sf.FormatProgress(utils.TruncateID(id), "Error downloading dependent layers", nil))
					return layers_downloaded, err
				}
			}
			// add layer to container
			dest := path.Join(s.graph.Driver().MountPath(), containerID, "rootfs")
			out.Write(sf.FormatProgress(utils.TruncateID(id), fmt.Sprintf("Merge layer to container rootfs %s", dest), nil))
			err = archive.ApplyLayer(dest, layer)
			if err != nil && j == retries {
				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error merge layers", nil))
				return layers_downloaded, err
			} else if err != nil {
				time.Sleep(time.Duration(j) * 500 * time.Millisecond)
				continue
			} else {
				break
			}
		}
	}
	return layers_downloaded, nil
}
Exemple #2
0
func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) (bool, error) {
	history, err := r.GetRemoteHistory(imgID, endpoint, token)
	if err != nil {
		return false, err
	}
	out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling dependent layers", nil))
	// FIXME: Try to stream the images?
	// FIXME: Launch the getRemoteImage() in goroutines

	layers_downloaded := false
	for i := len(history) - 1; i >= 0; i-- {
		id := history[i]

		// ensure no two downloads of the same layer happen at the same time
		if c, err := s.poolAdd("pull", "layer:"+id); err != nil {
			log.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err)
			<-c
		}
		defer s.poolRemove("pull", "layer:"+id)

		if !s.graph.Exists(id) {
			out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling metadata", nil))
			var (
				imgJSON []byte
				imgSize int
				err     error
				img     *image.Image
			)
			retries := 5
			for j := 1; j <= retries; j++ {
				imgJSON, imgSize, err = r.GetRemoteImageJSON(id, endpoint, token)
				if err != nil && j == retries {
					out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
					return layers_downloaded, err
				} else if err != nil {
					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
					continue
				}
				img, err = image.NewImgJSON(imgJSON)
				layers_downloaded = true
				if err != nil && j == retries {
					out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
					return layers_downloaded, fmt.Errorf("Failed to parse json: %s", err)
				} else if err != nil {
					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
					continue
				} else {
					break
				}
			}

			for j := 1; j <= retries; j++ {
				// Get the layer
				status := "Pulling fs layer"
				if j > 1 {
					status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
				}
				out.Write(sf.FormatProgress(utils.TruncateID(id), status, nil))
				layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token, int64(imgSize))
				if uerr, ok := err.(*url.Error); ok {
					err = uerr.Err
				}
				if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
					continue
				} else if err != nil {
					out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil))
					return layers_downloaded, err
				}
				layers_downloaded = true
				defer layer.Close()

				err = s.graph.Register(img,
					utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading"))
				if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
					continue
				} else if err != nil {
					out.Write(sf.FormatProgress(utils.TruncateID(id), "Error downloading dependent layers", nil))
					return layers_downloaded, err
				} else {
					break
				}
			}
		}
		out.Write(sf.FormatProgress(utils.TruncateID(id), "Download complete", nil))
	}
	return layers_downloaded, nil
}
Exemple #3
0
// TODO rewrite this whole PoC
func main() {
	flag.Usage = func() {
		flag.PrintDefaults()
	}
	flag.Parse()
	if debug {
		os.Setenv("DEBUG", "1")
		log.SetLevel(log.DebugLevel)
	}
	if flag.NArg() == 0 {
		fmt.Println("ERROR: no image names provided")
		flag.Usage()
		os.Exit(1)
	}

	// make tempDir
	tempDir, err := ioutil.TempDir("", "docker-fetch-")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	defer os.RemoveAll(tempDir)

	fetcher := NewFetcher(tempDir)
	sc := registry.NewServiceConfig(rOptions)

	for _, arg := range flag.Args() {
		remote, tagName := parsers.ParseRepositoryTag(arg)
		if tagName == "" {
			tagName = "latest"
		}

		repInfo, err := sc.NewRepositoryInfo(remote)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		log.Debugf("%#v %q\n", repInfo, tagName)

		idx, err := repInfo.GetEndpoint()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		fmt.Fprintf(os.Stderr, "Pulling %s:%s from %s\n", repInfo.RemoteName, tagName, idx)

		var session *registry.Session
		if s, ok := fetcher.sessions[idx.String()]; ok {
			session = s
		} else {
			// TODO(vbatts) obviously the auth and http factory shouldn't be nil here
			session, err = registry.NewSession(nil, nil, idx, timeout)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
		}

		rd, err := session.GetRepositoryData(repInfo.RemoteName)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		log.Debugf("rd: %#v", rd)

		// produce the "repositories" file for the archive
		if _, ok := fetcher.repositories[repInfo.RemoteName]; !ok {
			fetcher.repositories[repInfo.RemoteName] = graph.Repository{}
		}
		log.Debugf("repositories: %#v", fetcher.repositories)

		if len(rd.Endpoints) == 0 {
			log.Fatalf("expected registry endpoints, but received none from the index")
		}

		tags, err := session.GetRemoteTags(rd.Endpoints, repInfo.RemoteName, rd.Tokens)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		if hash, ok := tags[tagName]; ok {
			fetcher.repositories[repInfo.RemoteName][tagName] = hash
		}
		log.Debugf("repositories: %#v", fetcher.repositories)

		imgList, err := session.GetRemoteHistory(fetcher.repositories[repInfo.RemoteName][tagName], rd.Endpoints[0], rd.Tokens)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		log.Debugf("imgList: %#v", imgList)

		for _, imgID := range imgList {
			// pull layers and jsons
			buf, _, err := session.GetRemoteImageJSON(imgID, rd.Endpoints[0], rd.Tokens)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			if err = os.MkdirAll(filepath.Join(fetcher.Root, imgID), 0755); err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			fh, err := os.Create(filepath.Join(fetcher.Root, imgID, "json"))
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			if _, err = fh.Write(buf); err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			fh.Close()
			log.Debugf("%s", fh.Name())

			tarRdr, err := session.GetRemoteImageLayer(imgID, rd.Endpoints[0], rd.Tokens, 0)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			fh, err = os.Create(filepath.Join(fetcher.Root, imgID, "layer.tar"))
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			// the body is usually compressed
			gzRdr, err := gzip.NewReader(tarRdr)
			if err != nil {
				log.Debugf("image layer for %q is not gzipped", imgID)
				// the archive may not be gzipped, so just copy the stream
				if _, err = io.Copy(fh, tarRdr); err != nil {
					fmt.Fprintln(os.Stderr, err)
					os.Exit(1)
				}
			} else {
				// no error, so gzip decompress the stream
				if _, err = io.Copy(fh, gzRdr); err != nil {
					fmt.Fprintln(os.Stderr, err)
					os.Exit(1)
				}
				if err = gzRdr.Close(); err != nil {
					fmt.Fprintln(os.Stderr, err)
					os.Exit(1)
				}
			}
			if err = tarRdr.Close(); err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			if err = fh.Close(); err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			log.Debugf("%s", fh.Name())
		}
	}

	// marshal the "repositories" file for writing out
	log.Debugf("repositories: %q", fetcher.repositories)
	buf, err := json.Marshal(fetcher.repositories)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	fh, err := os.Create(filepath.Join(fetcher.Root, "repositories"))
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if _, err = fh.Write(buf); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	fh.Close()
	log.Debugf("%s", fh.Name())

	var output io.WriteCloser
	if outputStream == "-" {
		output = os.Stdout
	} else {
		output, err = os.Create(outputStream)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
	defer output.Close()

	if err = os.Chdir(fetcher.Root); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	tarStream, err := archive.Tar(".", archive.Uncompressed)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if _, err = io.Copy(output, tarStream); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}