func (fetcher *RemoteV1Fetcher) fetchFromEndpoint(request *FetchRequest, endpointURL string, imgID string, logger lager.Logger) (*dockerImage, error) {
history, err := request.Session.GetRemoteHistory(imgID, endpointURL)
if err != nil {
return nil, err
}
for _, layerID := range history {
fetcher.Retainer.Retain(layercake.DockerImageID(layerID))
defer fetcher.Retainer.Release(layercake.DockerImageID(layerID))
}
var allLayers []*dockerLayer
remainingQuota := request.MaxSize
for i := len(history) - 1; i >= 0; i-- {
layer, err := fetcher.fetchLayer(request, endpointURL, history[i], remainingQuota, logger)
if err != nil {
return nil, err
}
allLayers = append(allLayers, layer)
remainingQuota = remainingQuota - layer.size
if remainingQuota < 0 {
return nil, ErrQuotaExceeded
}
}
return &dockerImage{allLayers}, nil
}
func (fetcher *RemoteV2Fetcher) Fetch(request *FetchRequest) (*FetchResponse, error) {
images, auth, hashes, err := fetcher.fetchImagesMetadata(request)
if err != nil {
return nil, err
}
remainingQuota := request.MaxSize
for i := len(images) - 1; i >= 0; i-- {
img := images[i]
fetcher.Retainer.Retain(layercake.DockerImageID(img.ID))
defer fetcher.Retainer.Release(layercake.DockerImageID(img.ID))
var size int64
if size, err = fetcher.fetchLayer(request, img, hashes[i], auth, remainingQuota); err != nil {
return nil, err
}
remainingQuota = remainingQuota - size
if remainingQuota < 0 {
return nil, ErrQuotaExceeded
}
}
return &FetchResponse{ImageID: images[0].ID}, nil
}
func (fetcher *RemoteV1Fetcher) fetchLayer(request *FetchRequest, endpointURL string, layerID string, remaining int64, logger lager.Logger) (*dockerLayer, error) {
fetcher.GraphLock.Acquire(layerID)
defer fetcher.GraphLock.Release(layerID)
if img, err := fetcher.Cake.Get(layercake.DockerImageID(layerID)); err == nil {
request.Logger.Info("using-cached", lager.Data{
"layer": layerID,
"size": img.Size,
})
return &dockerLayer{imgEnv(img, request.Logger), imgVolumes(img), img.Size}, nil
}
imgJSON, imgSize, err := request.Session.GetRemoteImageJSON(layerID, endpointURL)
if err != nil {
return nil, fmt.Errorf("get remote image JSON: %v", err)
}
img, err := image.NewImgJSON(imgJSON)
if err != nil {
return nil, fmt.Errorf("new image JSON: %v", err)
}
layer, err := request.Session.GetRemoteImageLayer(img.ID, endpointURL, int64(imgSize))
if err != nil {
return nil, fmt.Errorf("get remote image layer: %v", err)
}
defer layer.Close()
started := time.Now()
request.Logger.Info("downloading", lager.Data{
"layer": layerID,
})
err = fetcher.Cake.Register(img, &QuotaedReader{R: layer, N: remaining})
if err != nil {
return nil, fmt.Errorf("register: %s", err)
}
request.Logger.Info("downloaded", lager.Data{
"layer": layerID,
"took": time.Since(started),
"vols": imgVolumes(img),
"size": img.Size,
})
return &dockerLayer{imgEnv(img, request.Logger), imgVolumes(img), img.Size}, nil
}
func (fetcher *RemoteV2Fetcher) fetchLayer(request *FetchRequest, img *image.Image, hash digest.Digest, auth *registry.RequestAuthorization, remaining int64) (int64, error) {
fetcher.GraphLock.Acquire(img.ID)
defer fetcher.GraphLock.Release(img.ID)
if img, err := fetcher.Cake.Get(layercake.DockerImageID(img.ID)); err == nil {
return img.Size, nil
}
reader, _, err := request.Session.GetV2ImageBlobReader(request.Endpoint, request.RemotePath, hash, auth)
if err != nil {
return 0, FetchError("GetV2ImageBlobReader", request.Endpoint.URL.Host, request.Path, err)
}
defer reader.Close()
err = fetcher.Cake.Register(img, &QuotaedReader{R: reader, N: remaining})
if err != nil {
return 0, FetchError("GraphRegister", request.Endpoint.URL.Host, request.Path, err)
}
return img.Size, nil
}
func (provider *dockerRootFSProvider) ProvideRootFS(logger lager.Logger, id string, url *url.URL, shouldNamespace bool, quota int64) (string, process.Env, error) {
tag := "latest"
if len(url.Fragment) > 0 {
tag = url.Fragment
}
fetchedID, envvars, volumes, err := provider.repoFetcher.Fetch(logger, url, tag, quota)
if err != nil {
return "", nil, err
}
var imageID layercake.ID = layercake.DockerImageID(fetchedID)
if shouldNamespace {
provider.mutex.Lock()
imageID, err = provider.namespace(imageID)
provider.mutex.Unlock()
if err != nil {
return "", nil, err
}
}
containerID := layercake.ContainerID(id)
err = provider.graph.Create(containerID, imageID)
if err != nil {
return "", nil, err
}
rootPath, err := provider.graph.Path(containerID)
if err != nil {
return "", nil, err
}
for _, v := range volumes {
if err = provider.volumeCreator.Create(rootPath, v); err != nil {
return "", nil, err
}
}
return rootPath, envvars, nil
}
Expect(gc.Remove(layercake.ContainerID("child"))).To(Succeed())
Expect(fakeCake.RemoveCallCount()).To(Equal(0))
})
})
})
Context("when removing fails", func() {
It("returns an error", func() {
fakeCake.RemoveReturns(errors.New("cake failure"))
Expect(gc.Remove(layercake.ContainerID("whatever"))).To(MatchError("cake failure"))
})
})
Context("when the layer has a parent", func() {
BeforeEach(func() {
child2parent[layercake.ContainerID("child")] = layercake.DockerImageID("parent")
})
Context("and the parent has no other children", func() {
It("removes the layer, and its parent", func() {
Expect(gc.Remove(layercake.ContainerID("child"))).To(Succeed())
Expect(fakeCake.RemoveCallCount()).To(Equal(2))
Expect(fakeCake.RemoveArgsForCall(0)).To(Equal(layercake.ContainerID("child")))
Expect(fakeCake.RemoveArgsForCall(1)).To(Equal(layercake.DockerImageID("parent")))
})
})
Context("when removing fails", func() {
It("does not remove any more layers", func() {
fakeCake.RemoveReturns(errors.New("cake failure"))
func (UnderscoreIDer) ProvideID(path string) layercake.ID {
return layercake.DockerImageID(strings.Replace(path, "/", "_", -1))
}
fakeRepositoryFetcher.FetchResult = "some-image-id"
fakeCake.PathReturns("/some/graph/driver/mount/point", nil)
mountpoint, envvars, err := provider.ProvideRootFS(
logger,
"some-id",
parseURL("docker:///some-repository-name"),
false,
0,
)
Expect(err).ToNot(HaveOccurred())
Expect(fakeCake.CreateCallCount()).To(Equal(1))
id, parent := fakeCake.CreateArgsForCall(0)
Expect(id).To(Equal(layercake.ContainerID("some-id")))
Expect(parent).To(Equal(layercake.DockerImageID("some-image-id")))
Expect(fakeRepositoryFetcher.Fetched()).To(ContainElement(
fake_repository_fetcher.FetchSpec{
Repository: "docker:///some-repository-name",
Tag: "latest",
},
))
Expect(mountpoint).To(Equal("/some/graph/driver/mount/point"))
Expect(envvars).To(Equal(
process.Env{
"env1": "env1Value",
"env2": "env2Value",
},
))
It("releases all the layers after fetching", func() {
setupSuccessfulV2Fetch(server, false)
released := make(map[layercake.ID]bool)
retainer.ReleaseStub = func(id layercake.ID) {
released[id] = true
}
cake.GetStub = func(id layercake.ID) (*image.Image, error) {
Expect(released).To(BeEmpty())
return nil, errors.New("no layer")
}
fetcher.Fetch(fetchRequest)
Expect(released).To(HaveKey(layercake.DockerImageID("banana-pie-1")))
Expect(released).To(HaveKey(layercake.DockerImageID("banana-pie-2")))
})
Context("when none of the layers already exist", func() {
BeforeEach(func() {
setupSuccessfulV2Fetch(server, false)
})
It("downloads all layers of the given tag of a repository and returns its image id", func() {
layers := 0
cake.RegisterStub = func(image *image.Image, layer archive.ArchiveReader) error {
Expect(image.ID).To(Equal(fmt.Sprintf("banana-pie-%d", layers+1)))
parent := ""
if layers > 0 {
"github.com/cloudfoundry-incubator/garden-linux/layercake"
"github.com/cloudfoundry-incubator/garden-linux/layercake/fake_cake"
"github.com/cloudfoundry/gunk/command_runner/fake_command_runner"
. "github.com/cloudfoundry/gunk/command_runner/fake_command_runner/matchers"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pivotal-golang/lager/lagertest"
)
var _ = Describe("BtrfsCleaningCake", func() {
var (
cleaner *layercake.BtrfsCleaningCake
runner *fake_command_runner.FakeCommandRunner
fakeCake *fake_cake.FakeCake
listSubvolumesOutput string
layerId = layercake.DockerImageID("the-layer")
btrfsMountPoint = "/absolute/btrfs_mount"
listSubVolumeErr error
graphDriverErr error
removedDirectories []string
)
BeforeEach(func() {
graphDriverErr = nil
listSubVolumeErr = nil
removedDirectories = []string{}
runner = fake_command_runner.New()
fakeCake = new(fake_cake.FakeCake)
retainer = new(fake_retainer.FakeRetainer)
fetcher = &RemoteV1Fetcher{
Cake: cake,
Retainer: retainer,
GraphLock: lock,
}
cake.GetReturns(&image.Image{}, nil)
})
It("retains all the layers before starting", func() {
setupSuccessfulFetch(endpoint1Server)
retained := make(map[layercake.ID]bool)
cake.GetStub = func(id layercake.ID) (*image.Image, error) {
Expect(retained).To(HaveKey(layercake.DockerImageID("layer-1")))
Expect(retained).To(HaveKey(layercake.DockerImageID("layer-2")))
Expect(retained).To(HaveKey(layercake.DockerImageID("layer-3")))
return nil, errors.New("no layer")
}
retainer.RetainStub = func(id layercake.ID) {
retained[id] = true
}
fetcher.Fetch(fetchRequest)
})
It("releases all the layers after fetching", func() {
setupSuccessfulFetch(endpoint1Server)
import (
"github.com/cloudfoundry-incubator/garden-linux/layercake"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("retainer", func() {
var (
retainer layercake.Retainer
id layercake.ID
)
BeforeEach(func() {
retainer = layercake.NewRetainer()
id = layercake.DockerImageID("banana")
})
Describe("IsHeld", func() {
Context("when the image is not retained", func() {
It("should not be held", func() {
Expect(retainer.IsHeld(id)).To(BeFalse())
})
})
Context("when the image is retained", func() {
BeforeEach(func() {
retainer.Retain(id)
})
It("should be held", func() {