Beispiel #1
0
func (fetcher *RemoteV1Fetcher) fetchFromEndpoint(request *FetchRequest, endpointURL string, imgID string, logger lager.Logger) (*dockerImage, error) {
	history, err := request.Session.GetRemoteHistory(imgID, endpointURL)
	if err != nil {
		return nil, err
	}

	for _, layerID := range history {
		fetcher.Retainer.Retain(layercake.DockerImageID(layerID))
		defer fetcher.Retainer.Release(layercake.DockerImageID(layerID))
	}

	var allLayers []*dockerLayer
	remainingQuota := request.MaxSize
	for i := len(history) - 1; i >= 0; i-- {
		layer, err := fetcher.fetchLayer(request, endpointURL, history[i], remainingQuota, logger)
		if err != nil {
			return nil, err
		}

		allLayers = append(allLayers, layer)

		remainingQuota = remainingQuota - layer.size
		if remainingQuota < 0 {
			return nil, ErrQuotaExceeded
		}
	}

	return &dockerImage{allLayers}, nil
}
Beispiel #2
0
func (fetcher *RemoteV2Fetcher) Fetch(request *FetchRequest) (*FetchResponse, error) {
	images, auth, hashes, err := fetcher.fetchImagesMetadata(request)
	if err != nil {
		return nil, err
	}

	remainingQuota := request.MaxSize

	for i := len(images) - 1; i >= 0; i-- {
		img := images[i]
		fetcher.Retainer.Retain(layercake.DockerImageID(img.ID))
		defer fetcher.Retainer.Release(layercake.DockerImageID(img.ID))

		var size int64
		if size, err = fetcher.fetchLayer(request, img, hashes[i], auth, remainingQuota); err != nil {
			return nil, err
		}

		remainingQuota = remainingQuota - size
		if remainingQuota < 0 {
			return nil, ErrQuotaExceeded
		}
	}

	return &FetchResponse{ImageID: images[0].ID}, nil
}
Beispiel #3
0
func (fetcher *RemoteV1Fetcher) fetchLayer(request *FetchRequest, endpointURL string, layerID string, remaining int64, logger lager.Logger) (*dockerLayer, error) {
	fetcher.GraphLock.Acquire(layerID)
	defer fetcher.GraphLock.Release(layerID)

	if img, err := fetcher.Cake.Get(layercake.DockerImageID(layerID)); err == nil {
		request.Logger.Info("using-cached", lager.Data{
			"layer": layerID,
			"size":  img.Size,
		})

		return &dockerLayer{imgEnv(img, request.Logger), imgVolumes(img), img.Size}, nil
	}

	imgJSON, imgSize, err := request.Session.GetRemoteImageJSON(layerID, endpointURL)
	if err != nil {
		return nil, fmt.Errorf("get remote image JSON: %v", err)
	}

	img, err := image.NewImgJSON(imgJSON)
	if err != nil {
		return nil, fmt.Errorf("new image JSON: %v", err)
	}

	layer, err := request.Session.GetRemoteImageLayer(img.ID, endpointURL, int64(imgSize))
	if err != nil {
		return nil, fmt.Errorf("get remote image layer: %v", err)
	}

	defer layer.Close()

	started := time.Now()

	request.Logger.Info("downloading", lager.Data{
		"layer": layerID,
	})

	err = fetcher.Cake.Register(img, &QuotaedReader{R: layer, N: remaining})
	if err != nil {
		return nil, fmt.Errorf("register: %s", err)
	}

	request.Logger.Info("downloaded", lager.Data{
		"layer": layerID,
		"took":  time.Since(started),
		"vols":  imgVolumes(img),
		"size":  img.Size,
	})

	return &dockerLayer{imgEnv(img, request.Logger), imgVolumes(img), img.Size}, nil
}
Beispiel #4
0
func (fetcher *RemoteV2Fetcher) fetchLayer(request *FetchRequest, img *image.Image, hash digest.Digest, auth *registry.RequestAuthorization, remaining int64) (int64, error) {
	fetcher.GraphLock.Acquire(img.ID)
	defer fetcher.GraphLock.Release(img.ID)

	if img, err := fetcher.Cake.Get(layercake.DockerImageID(img.ID)); err == nil {
		return img.Size, nil
	}

	reader, _, err := request.Session.GetV2ImageBlobReader(request.Endpoint, request.RemotePath, hash, auth)
	if err != nil {
		return 0, FetchError("GetV2ImageBlobReader", request.Endpoint.URL.Host, request.Path, err)
	}
	defer reader.Close()

	err = fetcher.Cake.Register(img, &QuotaedReader{R: reader, N: remaining})
	if err != nil {
		return 0, FetchError("GraphRegister", request.Endpoint.URL.Host, request.Path, err)
	}

	return img.Size, nil
}
func (provider *dockerRootFSProvider) ProvideRootFS(logger lager.Logger, id string, url *url.URL, shouldNamespace bool, quota int64) (string, process.Env, error) {
	tag := "latest"
	if len(url.Fragment) > 0 {
		tag = url.Fragment
	}

	fetchedID, envvars, volumes, err := provider.repoFetcher.Fetch(logger, url, tag, quota)
	if err != nil {
		return "", nil, err
	}

	var imageID layercake.ID = layercake.DockerImageID(fetchedID)
	if shouldNamespace {
		provider.mutex.Lock()
		imageID, err = provider.namespace(imageID)
		provider.mutex.Unlock()
		if err != nil {
			return "", nil, err
		}
	}

	containerID := layercake.ContainerID(id)
	err = provider.graph.Create(containerID, imageID)
	if err != nil {
		return "", nil, err
	}

	rootPath, err := provider.graph.Path(containerID)
	if err != nil {
		return "", nil, err
	}

	for _, v := range volumes {
		if err = provider.volumeCreator.Create(rootPath, v); err != nil {
			return "", nil, err
		}
	}

	return rootPath, envvars, nil
}
					Expect(gc.Remove(layercake.ContainerID("child"))).To(Succeed())
					Expect(fakeCake.RemoveCallCount()).To(Equal(0))
				})
			})
		})

		Context("when removing fails", func() {
			It("returns an error", func() {
				fakeCake.RemoveReturns(errors.New("cake failure"))
				Expect(gc.Remove(layercake.ContainerID("whatever"))).To(MatchError("cake failure"))
			})
		})

		Context("when the layer has a parent", func() {
			BeforeEach(func() {
				child2parent[layercake.ContainerID("child")] = layercake.DockerImageID("parent")
			})

			Context("and the parent has no other children", func() {
				It("removes the layer, and its parent", func() {
					Expect(gc.Remove(layercake.ContainerID("child"))).To(Succeed())

					Expect(fakeCake.RemoveCallCount()).To(Equal(2))
					Expect(fakeCake.RemoveArgsForCall(0)).To(Equal(layercake.ContainerID("child")))
					Expect(fakeCake.RemoveArgsForCall(1)).To(Equal(layercake.DockerImageID("parent")))
				})
			})

			Context("when removing fails", func() {
				It("does not remove any more layers", func() {
					fakeCake.RemoveReturns(errors.New("cake failure"))
Beispiel #7
0
func (UnderscoreIDer) ProvideID(path string) layercake.ID {
	return layercake.DockerImageID(strings.Replace(path, "/", "_", -1))
}
				fakeRepositoryFetcher.FetchResult = "some-image-id"
				fakeCake.PathReturns("/some/graph/driver/mount/point", nil)

				mountpoint, envvars, err := provider.ProvideRootFS(
					logger,
					"some-id",
					parseURL("docker:///some-repository-name"),
					false,
					0,
				)
				Expect(err).ToNot(HaveOccurred())

				Expect(fakeCake.CreateCallCount()).To(Equal(1))
				id, parent := fakeCake.CreateArgsForCall(0)
				Expect(id).To(Equal(layercake.ContainerID("some-id")))
				Expect(parent).To(Equal(layercake.DockerImageID("some-image-id")))

				Expect(fakeRepositoryFetcher.Fetched()).To(ContainElement(
					fake_repository_fetcher.FetchSpec{
						Repository: "docker:///some-repository-name",
						Tag:        "latest",
					},
				))

				Expect(mountpoint).To(Equal("/some/graph/driver/mount/point"))
				Expect(envvars).To(Equal(
					process.Env{
						"env1": "env1Value",
						"env2": "env2Value",
					},
				))
	It("releases all the layers after fetching", func() {
		setupSuccessfulV2Fetch(server, false)

		released := make(map[layercake.ID]bool)
		retainer.ReleaseStub = func(id layercake.ID) {
			released[id] = true
		}

		cake.GetStub = func(id layercake.ID) (*image.Image, error) {
			Expect(released).To(BeEmpty())
			return nil, errors.New("no layer")
		}

		fetcher.Fetch(fetchRequest)

		Expect(released).To(HaveKey(layercake.DockerImageID("banana-pie-1")))
		Expect(released).To(HaveKey(layercake.DockerImageID("banana-pie-2")))
	})

	Context("when none of the layers already exist", func() {
		BeforeEach(func() {
			setupSuccessfulV2Fetch(server, false)
		})

		It("downloads all layers of the given tag of a repository and returns its image id", func() {
			layers := 0

			cake.RegisterStub = func(image *image.Image, layer archive.ArchiveReader) error {
				Expect(image.ID).To(Equal(fmt.Sprintf("banana-pie-%d", layers+1)))
				parent := ""
				if layers > 0 {
Beispiel #10
0
	"github.com/cloudfoundry-incubator/garden-linux/layercake"
	"github.com/cloudfoundry-incubator/garden-linux/layercake/fake_cake"
	"github.com/cloudfoundry/gunk/command_runner/fake_command_runner"
	. "github.com/cloudfoundry/gunk/command_runner/fake_command_runner/matchers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pivotal-golang/lager/lagertest"
)

var _ = Describe("BtrfsCleaningCake", func() {
	var (
		cleaner              *layercake.BtrfsCleaningCake
		runner               *fake_command_runner.FakeCommandRunner
		fakeCake             *fake_cake.FakeCake
		listSubvolumesOutput string
		layerId              = layercake.DockerImageID("the-layer")
		btrfsMountPoint      = "/absolute/btrfs_mount"

		listSubVolumeErr error
		graphDriverErr   error

		removedDirectories []string
	)

	BeforeEach(func() {
		graphDriverErr = nil
		listSubVolumeErr = nil
		removedDirectories = []string{}

		runner = fake_command_runner.New()
		fakeCake = new(fake_cake.FakeCake)
		retainer = new(fake_retainer.FakeRetainer)
		fetcher = &RemoteV1Fetcher{
			Cake:      cake,
			Retainer:  retainer,
			GraphLock: lock,
		}

		cake.GetReturns(&image.Image{}, nil)
	})

	It("retains all the layers before starting", func() {
		setupSuccessfulFetch(endpoint1Server)

		retained := make(map[layercake.ID]bool)
		cake.GetStub = func(id layercake.ID) (*image.Image, error) {
			Expect(retained).To(HaveKey(layercake.DockerImageID("layer-1")))
			Expect(retained).To(HaveKey(layercake.DockerImageID("layer-2")))
			Expect(retained).To(HaveKey(layercake.DockerImageID("layer-3")))
			return nil, errors.New("no layer")
		}

		retainer.RetainStub = func(id layercake.ID) {
			retained[id] = true
		}

		fetcher.Fetch(fetchRequest)
	})

	It("releases all the layers after fetching", func() {
		setupSuccessfulFetch(endpoint1Server)
import (
	"github.com/cloudfoundry-incubator/garden-linux/layercake"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("retainer", func() {
	var (
		retainer layercake.Retainer
		id       layercake.ID
	)

	BeforeEach(func() {
		retainer = layercake.NewRetainer()
		id = layercake.DockerImageID("banana")
	})

	Describe("IsHeld", func() {
		Context("when the image is not retained", func() {
			It("should not be held", func() {
				Expect(retainer.IsHeld(id)).To(BeFalse())
			})
		})

		Context("when the image is retained", func() {
			BeforeEach(func() {
				retainer.Retain(id)
			})

			It("should be held", func() {