Beispiel #1
0
func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId, ep string, token []string, sf *utils.StreamFormatter) error {
	out = utils.NewWriteFlusher(out)
	jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgId, "json"))
	if err != nil {
		return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgId, err)
	}
	out.Write(sf.FormatStatus("Pushing %s", imgId))

	// Make sure we have the image's checksum
	checksum, err := srv.getChecksum(imgId)
	if err != nil {
		return err
	}
	imgData := &registry.ImgData{
		ID:       imgId,
		Checksum: checksum,
	}

	// Send the json
	if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil {
		if err == registry.ErrAlreadyExists {
			out.Write(sf.FormatStatus("Image %s already uploaded ; skipping", imgData.ID))
			return nil
		}
		return err
	}

	// Retrieve the tarball to be sent
	var layerData *TempArchive
	// If the archive exists, use it
	file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgId)))
	if err != nil {
		if os.IsNotExist(err) {
			// If the archive does not exist, create one from the layer
			layerData, err = srv.runtime.graph.TempLayerArchive(imgId, Xz, out)
			if err != nil {
				return fmt.Errorf("Failed to generate layer archive: %s", err)
			}
		} else {
			return err
		}
	} else {
		defer file.Close()
		st, err := file.Stat()
		if err != nil {
			return err
		}
		layerData = &TempArchive{
			File: file,
			Size: st.Size(),
		}
	}

	// Send the layer
	if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%v/%v (%v)"), sf), ep, token); err != nil {
		return err
	}
	return nil
}
Beispiel #2
0
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error {
	out = utils.NewWriteFlusher(out)
	imgList, err := srv.getImageList(localRepo)
	if err != nil {
		return err
	}
	flattenedImgList := flatten(imgList)
	out.Write(sf.FormatStatus("", "Sending image list"))

	var repoData *registry.RepositoryData
	repoData, err = r.PushImageJSONIndex(indexEp, remoteName, flattenedImgList, false, nil)
	if err != nil {
		return err
	}

	for _, ep := range repoData.Endpoints {
		out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", localName, len(localRepo)))
		// This section can not be parallelized (each round depends on the previous one)
		for _, round := range imgList {
			// FIXME: This section can be parallelized
			for _, elem := range round {
				var pushTags func() error
				pushTags = func() error {
					out.Write(sf.FormatStatus("", "Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag))
					if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
						return err
					}
					return nil
				}
				if _, exists := repoData.ImgList[elem.ID]; exists {
					if err := pushTags(); err != nil {
						return err
					}
					out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID))
					continue
				} else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) {
					if err := pushTags(); err != nil {
						return err
					}
					out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID))
					continue
				}
				if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil {
					// FIXME: Continue on error?
					return err
				} else {
					elem.Checksum = checksum
				}
				return pushTags()
			}
		}
	}

	if _, err := r.PushImageJSONIndex(indexEp, remoteName, flattenedImgList, true, repoData.Endpoints); err != nil {
		return err
	}

	return nil
}
Beispiel #3
0
func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoint string, token []string, sf *utils.StreamFormatter) error {
	history, err := r.GetRemoteHistory(imgId, endpoint, token)
	if err != nil {
		return err
	}

	// FIXME: Try to stream the images?
	// FIXME: Launch the getRemoteImage() in goroutines
	for _, id := range history {
		if !srv.runtime.graph.Exists(id) {
			out.Write(sf.FormatStatus("Pulling %s metadata", id))
			imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token)
			if err != nil {
				// FIXME: Keep goging in case of error?
				return err
			}
			img, err := NewImgJSON(imgJSON)
			if err != nil {
				return fmt.Errorf("Failed to parse json: %s", err)
			}

			// Get the layer
			out.Write(sf.FormatStatus("Pulling %s fs layer", id))
			layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token)
			if err != nil {
				return err
			}
			defer layer.Close()
			if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%v/%v (%v)"), sf), false, img); err != nil {
				return err
			}
		}
	}
	return nil
}
Beispiel #4
0
func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoint string, token []string, json bool) error {
	history, err := r.GetRemoteHistory(imgId, endpoint, token)
	if err != nil {
		return err
	}

	// FIXME: Try to stream the images?
	// FIXME: Launch the getRemoteImage() in goroutines
	for _, id := range history {
		if !srv.runtime.graph.Exists(id) {
			fmt.Fprintf(out, utils.FormatStatus("Pulling %s metadata", json), id)
			imgJson, err := r.GetRemoteImageJson(id, endpoint, token)
			if err != nil {
				// FIXME: Keep goging in case of error?
				return err
			}
			img, err := NewImgJson(imgJson)
			if err != nil {
				return fmt.Errorf("Failed to parse json: %s", err)
			}

			// Get the layer
			fmt.Fprintf(out, utils.FormatStatus("Pulling %s fs layer", json), id)
			layer, contentLength, err := r.GetRemoteImageLayer(img.Id, endpoint, token)
			if err != nil {
				return err
			}
			if err := srv.runtime.graph.Register(utils.ProgressReader(layer, contentLength, out, utils.FormatProgress("%v/%v (%v)", json), json), false, img); err != nil {
				return err
			}
		}
	}
	return nil
}
Beispiel #5
0
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error {
	out = utils.NewWriteFlusher(out)
	out.Write(sf.FormatStatus("Processing checksums"))
	imgList, err := srv.getImageList(localRepo)
	if err != nil {
		return err
	}
	out.Write(sf.FormatStatus("Sending image list"))
	srvName := name
	parts := strings.Split(name, "/")
	if len(parts) > 2 {
		srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/")))
	}

	var repoData *registry.RepositoryData
	repoData, err = r.PushImageJSONIndex(indexEp, name, imgList, false, nil)
	if err != nil {
		return err
	}

	for _, ep := range repoData.Endpoints {
		out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo)))
		// For each image within the repo, push them
		for _, elem := range imgList {
			if _, exists := repoData.ImgList[elem.ID]; exists {
				out.Write(sf.FormatStatus("Image %s already on registry, skipping", name))
				continue
			} else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) {
				fmt.Fprintf(out, "Image %s already on registry, skipping\n", name)
				continue
			}
			if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil {
				// FIXME: Continue on error?
				return err
			}
			out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+srvName+"/tags/"+elem.Tag))
			if err := r.PushRegistryTag(srvName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
				return err
			}
		}
	}

	if _, err := r.PushImageJSONIndex(indexEp, name, imgList, true, repoData.Endpoints); err != nil {
		return err
	}

	return nil
}
Beispiel #6
0
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error {
	out = utils.NewWriteFlusher(out)

	imgList, err := srv.getImageList(localRepo)
	if err != nil {
		return err
	}
	out.Write(sf.FormatStatus("Sending image list"))

	var repoData *registry.RepositoryData
	repoData, err = r.PushImageJSONIndex(indexEp, remoteName, imgList, false, nil)
	if err != nil {
		return err
	}

	for _, ep := range repoData.Endpoints {
		out.Write(sf.FormatStatus("Pushing repository %s (%d tags)", localName, len(localRepo)))
		// For each image within the repo, push them
		for _, elem := range imgList {
			if _, exists := repoData.ImgList[elem.ID]; exists {
				out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID))
				continue
			} else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) {
				out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID))
				continue
			}
			if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil {
				// FIXME: Continue on error?
				return err
			} else {
				elem.Checksum = checksum
			}
			out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag))
			if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
				return err
			}
		}
	}

	if _, err := r.PushImageJSONIndex(indexEp, remoteName, imgList, true, repoData.Endpoints); err != nil {
		return err
	}

	return nil
}
Beispiel #7
0
func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) error {
	history, err := r.GetRemoteHistory(imgID, endpoint, token)
	if err != nil {
		return err
	}
	out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling", "dependend layers"))
	// FIXME: Try to stream the images?
	// FIXME: Launch the getRemoteImage() in goroutines

	for _, id := range history {

		// ensure no two downloads of the same layer happen at the same time
		if err := srv.poolAdd("pull", "layer:"+id); err != nil {
			utils.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err)
			return nil
		}
		defer srv.poolRemove("pull", "layer:"+id)

		if !srv.runtime.graph.Exists(id) {
			out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "metadata"))
			imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token)
			if err != nil {
				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers"))
				// FIXME: Keep going in case of error?
				return err
			}
			img, err := NewImgJSON(imgJSON)
			if err != nil {
				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers"))
				return fmt.Errorf("Failed to parse json: %s", err)
			}

			// Get the layer
			out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "fs layer"))
			layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token)
			if err != nil {
				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers"))
				return err
			}
			defer layer.Close()
			if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf, false), img); err != nil {
				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "downloading dependend layers"))
				return err
			}
		}
		out.Write(sf.FormatProgress(utils.TruncateID(id), "Download", "complete"))

	}
	return nil
}
Beispiel #8
0
func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) {
	out = utils.NewWriteFlusher(out)
	jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgID, "json"))
	if err != nil {
		return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err)
	}
	out.Write(sf.FormatStatus("", "Pushing %s", imgID))

	imgData := &registry.ImgData{
		ID: imgID,
	}

	// Send the json
	if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil {
		if err == registry.ErrAlreadyExists {
			out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", imgData.ID))
			return "", nil
		}
		return "", err
	}

	layerData, err := srv.runtime.graph.TempLayerArchive(imgID, archive.Uncompressed, sf, out)
	if err != nil {
		return "", fmt.Errorf("Failed to generate layer archive: %s", err)
	}
	defer os.RemoveAll(layerData.Name())

	// Send the layer
	checksum, err = r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf, false), ep, token, jsonRaw)
	if err != nil {
		return "", err
	}
	imgData.Checksum = checksum

	out.Write(sf.FormatStatus("", ""))

	// Send the checksum
	if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil {
		return "", err
	}

	return imgData.Checksum, nil
}
Beispiel #9
0
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string, sf *utils.StreamFormatter) error {
	out = utils.NewWriteFlusher(out)
	out.Write(sf.FormatStatus("Processing checksums"))
	imgList, err := srv.getImageList(localRepo)
	if err != nil {
		return err
	}
	out.Write(sf.FormatStatus("Sending image list"))

	repoData, err := r.PushImageJSONIndex(name, imgList, false)
	if err != nil {
		return err
	}

	for _, ep := range repoData.Endpoints {
		out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo)))
		// For each image within the repo, push them
		for _, elem := range imgList {
			if _, exists := repoData.ImgList[elem.ID]; exists {
				out.Write(sf.FormatStatus("Image %s already on registry, skipping", name))
				continue
			}
			if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil {
				// FIXME: Continue on error?
				return err
			}
			out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"/users/"+name+"/"+elem.Tag))
			if err := r.PushRegistryTag(name, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
				return err
			}
		}
	}

	if _, err := r.PushImageJSONIndex(name, imgList, true); err != nil {
		return err
	}
	return nil
}
Beispiel #10
0
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string) error {
	out = utils.NewWriteFlusher(out)
	fmt.Fprintf(out, "Processing checksums\n")
	imgList, err := srv.getImageList(localRepo)
	if err != nil {
		return err
	}
	fmt.Fprintf(out, "Sending images list\n")

	repoData, err := r.PushImageJsonIndex(name, imgList, false)
	if err != nil {
		return err
	}

	for _, ep := range repoData.Endpoints {
		fmt.Fprintf(out, "Pushing repository %s to %s (%d tags)\r\n", name, ep, len(localRepo))
		// For each image within the repo, push them
		for _, elem := range imgList {
			if _, exists := repoData.ImgList[elem.Id]; exists {
				fmt.Fprintf(out, "Image %s already on registry, skipping\n", name)
				continue
			}
			if err := srv.pushImage(r, out, name, elem.Id, ep, repoData.Tokens); err != nil {
				// FIXME: Continue on error?
				return err
			}
			fmt.Fprintf(out, "Pushing tags for rev [%s] on {%s}\n", elem.Id, ep+"/users/"+name+"/"+elem.Tag)
			if err := r.PushRegistryTag(name, elem.Id, elem.Tag, ep, repoData.Tokens); err != nil {
				return err
			}
		}
	}

	if _, err := r.PushImageJsonIndex(name, imgList, true); err != nil {
		return err
	}
	return nil
}
Beispiel #11
0
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, registryEp string, localRepo map[string]string, sf *utils.StreamFormatter) error {
	out = utils.NewWriteFlusher(out)
	out.Write(sf.FormatStatus("Processing checksums"))
	imgList, err := srv.getImageList(localRepo)
	if err != nil {
		return err
	}
	out.Write(sf.FormatStatus("Sending image list"))
	srvName := name
	parts := strings.Split(name, "/")
	if len(parts) > 2 {
		srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/")))
	}

	var repoData *registry.RepositoryData
	if registryEp == "" {
		repoData, err = r.PushImageJSONIndex(name, imgList, false, nil)
		if err != nil {
			return err
		}
	} else {
		repoData = &registry.RepositoryData{
			ImgList:   make(map[string]*registry.ImgData),
			Tokens:    []string{},
			Endpoints: []string{registryEp},
		}
		tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens)
		if err != nil && err.Error() != "Repository not found" {
			return err
		} else if err == nil {
			for tag, id := range tagsList {
				repoData.ImgList[id] = &registry.ImgData{
					ID:       id,
					Tag:      tag,
					Checksum: "",
				}
			}
		}
	}

	for _, ep := range repoData.Endpoints {
		if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) {
			ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep)
		}
		out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo)))
		// For each image within the repo, push them
		for _, elem := range imgList {
			if _, exists := repoData.ImgList[elem.ID]; exists {
				out.Write(sf.FormatStatus("Image %s already on registry, skipping", name))
				continue
			} else if registryEp != "" && r.LookupRemoteImage(elem.ID, registryEp, repoData.Tokens) {
				fmt.Fprintf(out, "Image %s already on registry, skipping\n", name)
				continue
			}
			if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil {
				// FIXME: Continue on error?
				return err
			}
			out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"/repositories/"+srvName+"/tags/"+elem.Tag))
			if err := r.PushRegistryTag(srvName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
				return err
			}
		}
	}

	if registryEp == "" {
		if _, err := r.PushImageJSONIndex(name, imgList, true, repoData.Endpoints); err != nil {
			return err
		}
	}

	return nil
}
Beispiel #12
0
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag, registryEp string, sf *utils.StreamFormatter) error {
	out.Write(sf.FormatStatus("Pulling repository %s from %s", local, auth.IndexServerAddress()))

	var repoData *registry.RepositoryData
	var err error
	if registryEp == "" {
		repoData, err = r.GetRepositoryData(remote)
		if err != nil {
			return err
		}

		utils.Debugf("Updating checksums")
		// Reload the json file to make sure not to overwrite faster sums
		if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
			return err
		}
	} else {
		repoData = &registry.RepositoryData{
			Tokens:    []string{},
			ImgList:   make(map[string]*registry.ImgData),
			Endpoints: []string{registryEp},
		}
	}

	utils.Debugf("Retrieving the tag list")
	tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens)
	if err != nil {
		utils.Debugf("%v", err)
		return err
	}

	if registryEp != "" {
		for tag, id := range tagsList {
			repoData.ImgList[id] = &registry.ImgData{
				ID:       id,
				Tag:      tag,
				Checksum: "",
			}
		}
	}

	utils.Debugf("Registering tags")
	// If no tag has been specified, pull them all
	if askedTag == "" {
		for tag, id := range tagsList {
			repoData.ImgList[id].Tag = tag
		}
	} else {
		// Otherwise, check that the tag exists and use only that one
		id, exists := tagsList[askedTag]
		if !exists {
			return fmt.Errorf("Tag %s not found in repositoy %s", askedTag, local)
		}
		repoData.ImgList[id].Tag = askedTag
	}

	for _, img := range repoData.ImgList {
		if askedTag != "" && img.Tag != askedTag {
			utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID)
			continue
		}
		out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, remote))
		success := false
		for _, ep := range repoData.Endpoints {
			if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) {
				ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep)
			}
			if err := srv.pullImage(r, out, img.ID, ep+"/v1", repoData.Tokens, sf); err != nil {
				out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
				continue
			}
			success = true
			break
		}
		if !success {
			return fmt.Errorf("Could not find repository on any of the indexed registries.")
		}
	}
	for tag, id := range tagsList {
		if askedTag != "" && tag != askedTag {
			continue
		}
		if err := srv.runtime.repositories.Set(local, tag, id, true); err != nil {
			return err
		}
	}
	if err := srv.runtime.repositories.Save(); err != nil {
		return err
	}

	return nil
}
Beispiel #13
0
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error {
	out.Write(sf.FormatStatus("", "Pulling repository %s", localName))

	repoData, err := r.GetRepositoryData(indexEp, remoteName)
	if err != nil {
		return err
	}

	utils.Debugf("Retrieving the tag list")
	tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens)
	if err != nil {
		utils.Debugf("%v", err)
		return err
	}

	for tag, id := range tagsList {
		repoData.ImgList[id] = &registry.ImgData{
			ID:       id,
			Tag:      tag,
			Checksum: "",
		}
	}

	utils.Debugf("Registering tags")
	// If no tag has been specified, pull them all
	if askedTag == "" {
		for tag, id := range tagsList {
			repoData.ImgList[id].Tag = tag
		}
	} else {
		// Otherwise, check that the tag exists and use only that one
		id, exists := tagsList[askedTag]
		if !exists {
			return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName)
		}
		repoData.ImgList[id].Tag = askedTag
	}

	errors := make(chan error)
	for _, image := range repoData.ImgList {
		downloadImage := func(img *registry.ImgData) {
			if askedTag != "" && img.Tag != askedTag {
				utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID)
				if parallel {
					errors <- nil
				}
				return
			}

			if img.Tag == "" {
				utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
				if parallel {
					errors <- nil
				}
				return
			}

			// ensure no two downloads of the same image happen at the same time
			if err := srv.poolAdd("pull", "img:"+img.ID); err != nil {
				utils.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
				if parallel {
					errors <- nil
				}
				return
			}
			defer srv.poolRemove("pull", "img:"+img.ID)

			out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s", img.Tag, localName)))
			success := false
			var lastErr error
			for _, ep := range repoData.Endpoints {
				out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s, endpoint: %s", img.Tag, localName, ep)))
				if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
					// Its not ideal that only the last error  is returned, it would be better to concatenate the errors.
					// As the error is also given to the output stream the user will see the error.
					lastErr = err
					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Error pulling", fmt.Sprintf("image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err)))
					continue
				}
				success = true
				break
			}
			if !success {
				out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Error pulling", fmt.Sprintf("image (%s) from %s, %s", img.Tag, localName, lastErr)))
				if parallel {
					errors <- fmt.Errorf("Could not find repository on any of the indexed registries.")
					return
				}
			}
			out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download", "complete"))

			if parallel {
				errors <- nil
			}
		}

		if parallel {
			go downloadImage(image)
		} else {
			downloadImage(image)
		}
	}
	if parallel {
		var lastError error
		for i := 0; i < len(repoData.ImgList); i++ {
			if err := <-errors; err != nil {
				lastError = err
			}
		}
		if lastError != nil {
			return lastError
		}

	}
	for tag, id := range tagsList {
		if askedTag != "" && tag != askedTag {
			continue
		}
		if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil {
			return err
		}
	}
	if err := srv.runtime.repositories.Save(); err != nil {
		return err
	}

	return nil
}
Beispiel #14
0
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error {
	out.Write(sf.FormatStatus("", "Pulling repository %s", localName))

	repoData, err := r.GetRepositoryData(indexEp, remoteName)
	if err != nil {
		return err
	}

	utils.Debugf("Retrieving the tag list")
	tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens)
	if err != nil {
		utils.Debugf("%v", err)
		return err
	}

	for tag, id := range tagsList {
		repoData.ImgList[id] = &registry.ImgData{
			ID:       id,
			Tag:      tag,
			Checksum: "",
		}
	}

	utils.Debugf("Registering tags")
	// If no tag has been specified, pull them all
	if askedTag == "" {
		for tag, id := range tagsList {
			repoData.ImgList[id].Tag = tag
		}
	} else {
		// Otherwise, check that the tag exists and use only that one
		id, exists := tagsList[askedTag]
		if !exists {
			return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName)
		}
		repoData.ImgList[id].Tag = askedTag
	}

	errors := make(chan error)
	for _, image := range repoData.ImgList {
		downloadImage := func(img *registry.ImgData) {
			if askedTag != "" && img.Tag != askedTag {
				utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID)
				errors <- nil
				return
			}

			if img.Tag == "" {
				utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
				errors <- nil
				return
			}
			out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s", img.Tag, localName)))
			success := false
			for _, ep := range repoData.Endpoints {
				if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
					out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
					continue
				}
				success = true
				break
			}
			if !success {
				errors <- fmt.Errorf("Could not find repository on any of the indexed registries.")
			}
			errors <- nil
		}

		if parallel {
			go downloadImage(image)
		} else {
			downloadImage(image)
		}
	}

	if parallel {
		for i := 0; i < len(repoData.ImgList); i++ {
			if err := <-errors; err != nil {
				return err
			}
		}
	}

	for tag, id := range tagsList {
		if askedTag != "" && tag != askedTag {
			continue
		}
		if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil {
			return err
		}
	}
	if err := srv.runtime.repositories.Save(); err != nil {
		return err
	}

	return nil
}
Beispiel #15
0
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter) error {
	out.Write(sf.FormatStatus("Pulling repository %s", localName))

	repoData, err := r.GetRepositoryData(indexEp, remoteName)
	if err != nil {
		return err
	}

	utils.Debugf("Updating checksums")
	// Reload the json file to make sure not to overwrite faster sums
	if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
		return err
	}

	utils.Debugf("Retrieving the tag list")
	tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens)
	if err != nil {
		utils.Debugf("%v", err)
		return err
	}

	for tag, id := range tagsList {
		repoData.ImgList[id] = &registry.ImgData{
			ID:       id,
			Tag:      tag,
			Checksum: "",
		}
	}

	utils.Debugf("Registering tags")
	// If no tag has been specified, pull them all
	if askedTag == "" {
		for tag, id := range tagsList {
			repoData.ImgList[id].Tag = tag
		}
	} else {
		// Otherwise, check that the tag exists and use only that one
		id, exists := tagsList[askedTag]
		if !exists {
			return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName)
		}
		repoData.ImgList[id].Tag = askedTag
	}

	for _, img := range repoData.ImgList {
		if askedTag != "" && img.Tag != askedTag {
			utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID)
			continue
		}

		if img.Tag == "" {
			utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
			continue
		}
		out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName))
		success := false
		for _, ep := range repoData.Endpoints {
			if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
				out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
				continue
			}
			success = true
			break
		}
		if !success {
			return fmt.Errorf("Could not find repository on any of the indexed registries.")
		}
	}
	for tag, id := range tagsList {
		if askedTag != "" && tag != askedTag {
			continue
		}
		if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil {
			return err
		}
	}
	if err := srv.runtime.repositories.Save(); err != nil {
		return err
	}

	return nil
}
Beispiel #16
0
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, remote, askedTag string, json bool) error {
	fmt.Fprintf(out, utils.FormatStatus("Pulling repository %s from %s", json), remote, auth.IndexServerAddress())
	repoData, err := r.GetRepositoryData(remote)
	if err != nil {
		return err
	}

	utils.Debugf("Updating checksums")
	// Reload the json file to make sure not to overwrite faster sums
	if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
		return err
	}

	utils.Debugf("Retrieving the tag list")
	tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens)
	if err != nil {
		return err
	}
	utils.Debugf("Registering tags")
	// If not specific tag have been asked, take all
	if askedTag == "" {
		for tag, id := range tagsList {
			repoData.ImgList[id].Tag = tag
		}
	} else {
		// Otherwise, check that the tag exists and use only that one
		if id, exists := tagsList[askedTag]; !exists {
			return fmt.Errorf("Tag %s not found in repositoy %s", askedTag, remote)
		} else {
			repoData.ImgList[id].Tag = askedTag
		}
	}

	for _, img := range repoData.ImgList {
		if askedTag != "" && img.Tag != askedTag {
			utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.Id)
			continue
		}
		fmt.Fprintf(out, utils.FormatStatus("Pulling image %s (%s) from %s", json), img.Id, img.Tag, remote)
		success := false
		for _, ep := range repoData.Endpoints {
			if err := srv.pullImage(r, out, img.Id, "https://"+ep+"/v1", repoData.Tokens, json); err != nil {
				fmt.Fprintf(out, utils.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint\n", json), askedTag, err)
				continue
			}
			success = true
			break
		}
		if !success {
			return fmt.Errorf("Could not find repository on any of the indexed registries.")
		}
	}
	for tag, id := range tagsList {
		if askedTag != "" && tag != askedTag {
			continue
		}
		if err := srv.runtime.repositories.Set(remote, tag, id, true); err != nil {
			return err
		}
	}
	if err := srv.runtime.repositories.Save(); err != nil {
		return err
	}

	return nil
}