func TestBuild_LookupImage_ExistLocally(t *testing.T) { var ( nilImage *docker.Image b, c = makeBuild(t, "", Config{}) resultImage = &docker.Image{ID: "789"} name = "ubuntu:latest" localImages = []*imagename.ImageName{ imagename.NewFromString("debian:7.7"), imagename.NewFromString("debian:latest"), imagename.NewFromString("ubuntu:12.04"), imagename.NewFromString("ubuntu:14.04"), imagename.NewFromString("ubuntu:latest"), } ) c.On("InspectImage", name).Return(nilImage, nil).Once() c.On("ListImages").Return(localImages, nil).Once() c.On("InspectImage", name).Return(resultImage, nil).Once() result, err := b.lookupImage(name) if err != nil { t.Fatal(err) } assert.Equal(t, resultImage, result) c.AssertExpectations(t) }
// ListImageTags returns the list of images instances obtained from all tags existing in the registry func (c *DockerClient) ListImageTags(name string) (images []*imagename.ImageName, err error) { img := imagename.NewFromString(name) if img.Storage == imagename.StorageS3 { return c.s3storage.ListTags(name) } return dockerclient.RegistryListTags(imagename.NewFromString(name), c.auth) }
func TestClientResolveVersions(t *testing.T) { t.Skip() dockerCli, err := dockerclient.New() if err != nil { t.Fatal(err) } client, err := NewClient(&DockerClient{ Docker: dockerCli, }) if err != nil { t.Fatal(err) } containers := []*Container{ &Container{ Name: config.NewContainerName("test", "test"), Image: imagename.NewFromString("golang:1.4.*"), }, } if err := client.resolveVersions(true, true, template.Vars{}, containers); err != nil { t.Fatal(err) } pretty.Println(containers) }
// NewContainerFromDocker converts a container object given by // docker client to a local Container object func NewContainerFromDocker(dockerContainer *docker.Container) (*Container, error) { cfg, err := config.NewFromDocker(dockerContainer) if err != nil { if _, ok := err.(config.ErrNotRockerCompose); !ok { return nil, err } } return &Container{ ID: dockerContainer.ID, Image: imagename.NewFromString(dockerContainer.Config.Image), ImageID: dockerContainer.Image, Name: config.NewContainerNameFromString(dockerContainer.Name), Created: dockerContainer.Created, State: &ContainerState{ Running: dockerContainer.State.Running, Paused: dockerContainer.State.Paused, Restarting: dockerContainer.State.Restarting, OOMKilled: dockerContainer.State.OOMKilled, Pid: dockerContainer.State.Pid, ExitCode: dockerContainer.State.ExitCode, Error: dockerContainer.State.Error, StartedAt: dockerContainer.State.StartedAt, FinishedAt: dockerContainer.State.FinishedAt, }, Config: cfg, container: dockerContainer, }, nil }
// Execute runs the command func (c *CommandPush) Execute(b *Build) (State, error) { if len(c.cfg.args) != 1 { return b.state, fmt.Errorf("PUSH requires exactly one argument") } if b.state.ImageID == "" { return b.state, fmt.Errorf("Cannot PUSH empty image") } if err := b.client.TagImage(b.state.ImageID, c.cfg.args[0]); err != nil { return b.state, err } image := imagename.NewFromString(c.cfg.args[0]) artifact := imagename.Artifact{ Name: image, Pushed: b.cfg.Push, Tag: image.GetTag(), ImageID: b.state.ImageID, BuildTime: time.Now(), } // push image and add some lines to artifacts if b.cfg.Push { digest, err := b.client.PushImage(image.String()) if err != nil { return b.state, err } artifact.SetDigest(digest) } else { log.Infof("| Don't push. Pass --push flag to actually push to the registry") } // Publish artifact files if b.cfg.ArtifactsPath != "" { if err := os.MkdirAll(b.cfg.ArtifactsPath, 0755); err != nil { return b.state, fmt.Errorf("Failed to create directory %s for the artifacts, error: %s", b.cfg.ArtifactsPath, err) } filePath := filepath.Join(b.cfg.ArtifactsPath, artifact.GetFileName()) artifacts := imagename.Artifacts{ RockerArtifacts: []imagename.Artifact{artifact}, } content, err := yaml.Marshal(artifacts) if err != nil { return b.state, err } if err := ioutil.WriteFile(filePath, content, 0644); err != nil { return b.state, fmt.Errorf("Failed to write artifact file %s, error: %s", filePath, err) } log.Infof("| Saved artifact file %s", filePath) log.Debugf("Artifact properties: %# v", pretty.Formatter(artifact)) } return b.state, nil }
func TestBuild_LookupImage_PullAndNotExist(t *testing.T) { var ( b, c = makeBuild(t, "", Config{Pull: true}) name = "ubuntu:latest" remoteImages = []*imagename.ImageName{ imagename.NewFromString("debian:7.7"), imagename.NewFromString("debian:latest"), imagename.NewFromString("ubuntu:12.04"), imagename.NewFromString("ubuntu:14.04"), } ) c.On("ListImageTags", name).Return(remoteImages, nil).Once() _, err := b.lookupImage(name) assert.EqualError(t, err, "Image not found: ubuntu:latest (also checked in the remote registry)") c.AssertExpectations(t) }
// NewContainerFromConfig makes a single Container object from a spec Config object. func NewContainerFromConfig(name *config.ContainerName, containerConfig *config.Container) *Container { container := &Container{ Name: name, State: &ContainerState{ Running: containerConfig.State.Bool(), }, Config: containerConfig, } if containerConfig.Image != nil { container.Image = imagename.NewFromString(*containerConfig.Image) } return container }
// PullImage pulls docker image func (c *DockerClient) PullImage(name string) error { image := imagename.NewFromString(name) // e.g. s3:bucket-name/image-name if image.Storage == imagename.StorageS3 { if isOld, warning := imagename.WarnIfOldS3ImageName(name); isOld { c.log.Warn(warning) } return c.s3storage.Pull(name) } var ( pipeReader, pipeWriter = io.Pipe() fdOut, isTerminalOut = term.GetFdInfo(c.log.Out) out = c.log.Out errch = make(chan error, 1) ) if !isTerminalOut { out = c.log.Writer() } opts := docker.PullImageOptions{ Repository: image.NameWithRegistry(), Registry: image.Registry, Tag: image.GetTag(), OutputStream: pipeWriter, RawJSONStream: true, } c.log.Infof("| Pull image %s", image) c.log.Debugf("Pull image %s with options: %# v", image, opts) go func() { errch <- jsonmessage.DisplayJSONMessagesStream(pipeReader, out, fdOut, isTerminalOut) }() auth, err := dockerclient.GetAuthForRegistry(c.auth, image) if err != nil { return fmt.Errorf("Failed to authenticate registry %s, error: %s", image.Registry, err) } if err := c.client.PullImage(opts, auth); err != nil { return err } pipeWriter.Close() return <-errch }
// ListImages lists all pulled images in the local docker registry func (c *DockerClient) ListImages() (images []*imagename.ImageName, err error) { var dockerImages []docker.APIImages if dockerImages, err = c.client.ListImages(docker.ListImagesOptions{}); err != nil { return } images = []*imagename.ImageName{} for _, image := range dockerImages { for _, repoTag := range image.RepoTags { images = append(images, imagename.NewFromString(repoTag)) } } return }
// TagImage adds tag to the image func (c *DockerClient) TagImage(imageID, imageName string) error { img := imagename.NewFromString(imageName) if isOld, warning := imagename.WarnIfOldS3ImageName(imageName); isOld { c.log.Warn(warning) } c.log.Infof("| Tag %.12s -> %s", imageID, img) opts := docker.TagImageOptions{ Repo: img.NameWithRegistry(), Tag: img.GetTag(), Force: true, } c.log.Debugf("Tag image %s with options: %# v", imageID, opts) return c.client.TagImage(imageID, opts) }
// GetBridgeIP gets the ip address of docker network bridge // it is useful when you want to loose couple containers and not have tightly link them // container A may publish port 8125 to host network and container B may access this port through // a bridge ip address; it's a hacky solution, any better way to obtain bridge ip without ssh access // to host machine is welcome // // Here we create a dummy container and look at .NetworkSettings.Gateway value // // TODO: maybe we don't need this anymore since docker 1.8 seem to specify all existing containers // in a /etc/hosts file of every contianer. Need to research it further. // // https://github.com/docker/docker/issues/1143 // https://github.com/docker/docker/issues/11247 // func GetBridgeIP(client *docker.Client) (ip string, err error) { // Ensure empty image existing _, err = client.InspectImage(emptyImageName) if err != nil && err.Error() == "no such image" { log.Infof("Pulling image %s to obtain network bridge address", emptyImageName) if _, err := PullDockerImage(client, imagename.NewFromString(emptyImageName), nil); err != nil { return "", err } } else if err != nil { return "", fmt.Errorf("Failed to inspect image %s, error: %s", emptyImageName, err) } container, err := client.CreateContainer(docker.CreateContainerOptions{ Config: &docker.Config{ Image: emptyImageName, Cmd: []string{"/bin/sh", "-c", "while true; do sleep 1; done"}, }, HostConfig: &docker.HostConfig{}, }) if err != nil { return "", fmt.Errorf("Failed to create dummy network container, error: %s", err) } defer func() { removeOpts := docker.RemoveContainerOptions{ ID: container.ID, Force: true, RemoveVolumes: true, } if err2 := client.RemoveContainer(removeOpts); err2 != nil && err == nil { err = err2 } }() if err := client.StartContainer(container.ID, &docker.HostConfig{}); err != nil { return "", fmt.Errorf("Failed to start dummy network container %.12s, error: %s", container.ID, err) } inspect, err := client.InspectContainer(container.ID) if err != nil { return "", fmt.Errorf("Failed to inspect dummy network container %.12s, error: %s", container.ID, err) } return inspect.NetworkSettings.Gateway, nil }
// ListTags returns the list of parsed tags existing for given image name on S3 func (s *StorageS3) ListTags(imageName string) (images []*imagename.ImageName, err error) { image := imagename.NewFromString(imageName) params := &s3.ListObjectsInput{ Bucket: aws.String(image.Registry), MaxKeys: aws.Int64(1000), Prefix: aws.String(image.Name), } resp, err := s.s3.ListObjects(params) if err != nil { return nil, err } for _, s3Obj := range resp.Contents { split := strings.Split(*s3Obj.Key, "/") if len(split) < 2 { continue } imgName := strings.Join(split[:len(split)-1], "/") imgName = fmt.Sprintf("s3.amazonaws.com/%s/%s", image.Registry, imgName) tag := strings.TrimSuffix(split[len(split)-1], ".tar") candidate := imagename.New(imgName, tag) if candidate.Name != image.Name { continue } if image.Contains(candidate) || image.Tag == candidate.Tag { images = append(images, candidate) } } return }
// pushImageInner pushes the image is the inner straightforward push without retries func (c *DockerClient) pushImageInner(imageName string) (digest string, err error) { img := imagename.NewFromString(imageName) // Use direct S3 image pusher instead if img.Storage == imagename.StorageS3 { if isOld, warning := imagename.WarnIfOldS3ImageName(imageName); isOld { c.log.Warn(warning) } return c.s3storage.Push(imageName) } var ( buf bytes.Buffer pipeReader, pipeWriter = io.Pipe() outStream = io.MultiWriter(pipeWriter, &buf) fdOut, isTerminalOut = term.GetFdInfo(c.log.Out) out = c.log.Out opts = docker.PushImageOptions{ Name: img.NameWithRegistry(), Tag: img.GetTag(), Registry: img.Registry, OutputStream: outStream, RawJSONStream: true, } errch = make(chan error, 1) ) if !isTerminalOut { out = c.log.Writer() } c.log.Infof("| Push %s", img) c.log.Debugf("Push with options: %# v", opts) // TODO: DisplayJSONMessagesStream may fail by client.PushImage run without errors go func() { errch <- jsonmessage.DisplayJSONMessagesStream(pipeReader, out, fdOut, isTerminalOut) }() auth, err := dockerclient.GetAuthForRegistry(c.auth, img) if err != nil { return "", fmt.Errorf("Failed to authenticate registry %s, error: %s", img.Registry, err) } if err := c.client.PushImage(opts, auth); err != nil { return "", err } pipeWriter.Close() if err := <-errch; err != nil { return "", fmt.Errorf("Failed to process json stream, error %s", err) } // It is the best way to have pushed image digest so far matches := captureDigest.FindStringSubmatch(buf.String()) if len(matches) > 0 { digest = matches[1] } return digest, nil }
func TestClientClean(t *testing.T) { // This test involves interaction with docker // enable it in case you want to test Clean() functionality t.Skip() dockerCli, err := dockerclient.New() if err != nil { t.Fatal(err) } // Create number of images to test createdContainers := []string{} createdImages := []string{} defer func() { for _, id := range createdContainers { if err := dockerCli.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true}); err != nil { t.Error(err) } } for _, id := range createdImages { if err := dockerCli.RemoveImageExtended(id, docker.RemoveImageOptions{Force: true}); err != nil { if err.Error() == "no such image" { continue } t.Error(err) } } }() for i := 1; i <= 5; i++ { c, err := dockerCli.CreateContainer(docker.CreateContainerOptions{ Config: &docker.Config{ Image: "gliderlabs/alpine:3.1", Cmd: []string{"true"}, }, }) if err != nil { t.Fatal(err) } createdContainers = append(createdContainers, c.ID) commitOpts := docker.CommitContainerOptions{ Container: c.ID, Repository: "rocker-compose-test-image-clean", Tag: fmt.Sprintf("%d", i), } img, err := dockerCli.CommitContainer(commitOpts) if err != nil { t.Fatal(err) } createdImages = append(createdImages, img.ID) // Make sure images have different timestamps time.Sleep(time.Second) } //////////////////////// cli, err := NewClient(&DockerClient{Docker: dockerCli, KeepImages: 2}) if err != nil { t.Fatal(err) } yml := ` namespace: test containers: main: image: rocker-compose-test-image-clean:5 ` config, err := config.ReadConfig("test.yml", strings.NewReader(yml), map[string]interface{}{}, map[string]interface{}{}, false) if err != nil { t.Fatal(err) } if err := cli.Clean(config); err != nil { t.Fatal(err) } // test that images left all, err := dockerCli.ListImages(docker.ListImagesOptions{}) if err != nil { t.Fatal(err) } n := 0 for _, image := range all { for _, repoTag := range image.RepoTags { imageName := imagename.NewFromString(repoTag) if imageName.Name == "rocker-compose-test-image-clean" { n++ } } } assert.Equal(t, 2, n, "Expected images to be cleaned up") // test removed images list removed := cli.GetRemovedImages() assert.Equal(t, 3, len(removed), "Expected to remove a particular number of images") assert.EqualValues(t, "rocker-compose-test-image-clean:3", removed[0].String(), "removed wrong image") assert.EqualValues(t, "rocker-compose-test-image-clean:2", removed[1].String(), "removed wrong image") assert.EqualValues(t, "rocker-compose-test-image-clean:1", removed[2].String(), "removed wrong image") }
"strings" "testing" "github.com/stretchr/testify/assert" ) var ( configTemplateVars = Vars{ "mykey": "myval", "n": "5", "data": map[string]string{ "foo": "bar", }, "RockerArtifacts": []imagename.Artifact{ imagename.Artifact{ Name: imagename.NewFromString("alpine:3.2"), Tag: "3.2", }, imagename.Artifact{ Name: imagename.NewFromString("golang:1.5"), Tag: "1.5", Digest: "sha256:ead434", }, imagename.Artifact{ Name: imagename.NewFromString("data:master"), Tag: "master", Digest: "sha256:fafe14", }, imagename.Artifact{ Name: imagename.NewFromString("ssh:latest"), Tag: "latest",
// Push pushes image tarball directly to S3 func (s *StorageS3) Push(imageName string) (digest string, err error) { img := imagename.NewFromString(imageName) if img.Storage != imagename.StorageS3 { return "", fmt.Errorf("Can only push images with s3 storage specified, got: %s", img) } if img.Registry == "" { return "", fmt.Errorf("Cannot push image to S3, missing bucket name, got: %s", img) } var image *docker.Image if image, err = s.client.InspectImage(img.String()); err != nil { return "", err } if digest, err = s.CacheGet(image.ID); err != nil { return "", err } var tmpf string defer func() { if tmpf != "" { os.Remove(tmpf) } }() // Not cached, make tar if digest == "" { if tmpf, digest, err = s.MakeTar(imageName); err != nil { return "", err } } var ( ext = ".tar" imgPathDigest = fmt.Sprintf("%s/%s%s", img.Name, digest, ext) imgPathTag = fmt.Sprintf("%s/%s%s", img.Name, img.Tag, ext) ) // Make HEAD request to s3 and check if image already uploaded _, headErr := s.s3.HeadObject(&s3.HeadObjectInput{ Bucket: aws.String(img.Registry), Key: aws.String(imgPathDigest), }) // Object not found, need to store if headErr != nil { // Other error, raise then if e, ok := headErr.(awserr.RequestFailure); !ok || e.StatusCode() != 404 { return "", headErr } // In case we do not have archive if tmpf == "" { var digest2 string if tmpf, digest2, err = s.MakeTar(imageName); err != nil { return "", err } // Verify digest (TODO: remote this check in future?) if digest != digest2 { return "", fmt.Errorf("The new digest does no equal old one (shouldn't happen) %s != %s", digest, digest2) } } uploader := s3manager.NewUploaderWithClient(s.s3, func(u *s3manager.Uploader) { u.PartSize = 64 * 1024 * 1024 // 64MB per part }) fd, err := os.Open(tmpf) if err != nil { return "", err } defer fd.Close() log.Infof("| Uploading image to s3.amazonaws.com/%s/%s", img.Registry, imgPathDigest) uploadParams := &s3manager.UploadInput{ Bucket: aws.String(img.Registry), Key: aws.String(imgPathDigest), ContentType: aws.String("application/x-tar"), Body: fd, Metadata: map[string]*string{ "Tag": aws.String(img.Tag), "ImageID": aws.String(image.ID), "Digest": aws.String(digest), }, } if err := s.retryer.Outer(func() error { _, err := uploader.Upload(uploadParams) return err }); err != nil { return "", fmt.Errorf("Failed to upload object to S3, error: %s", err) } } // Make a content addressable copy of an image file copyParams := &s3.CopyObjectInput{ Bucket: aws.String(img.Registry), CopySource: aws.String(img.Registry + "/" + imgPathDigest), Key: aws.String(imgPathTag), } log.Infof("| Make alias s3.amazonaws.com/%s/%s", img.Registry, imgPathTag) if _, err = s.s3.CopyObject(copyParams); err != nil { return "", fmt.Errorf("Failed to PUT object to S3, error: %s", err) } return digest, nil }
// MakeTar makes a tar out of docker image and gives a temporary file and a digest func (s *StorageS3) MakeTar(imageName string) (tmpfile string, digest string, err error) { img := imagename.NewFromString(imageName) var image *docker.Image if image, err = s.client.InspectImage(img.String()); err != nil { return "", "", err } tmpf, err := ioutil.TempFile("", "rocker_image_") if err != nil { return "", "", err } defer tmpf.Close() var ( cleanup = func() { os.Remove(tmpf.Name()) } humanSize = units.HumanSize(float64(image.VirtualSize)) errch = make(chan error, 1) pipeReader, pipeWriter = io.Pipe() tr = tar.NewReader(pipeReader) tw = tar.NewWriter(tmpf) hash = sha256.New() tarHashStream = io.MultiWriter(tw, hash) exportParams = docker.ExportImageOptions{ Name: img.String(), OutputStream: pipeWriter, } ) defer pipeWriter.Close() log.Infof("| Buffering image to a file %s (%s)", tmpf.Name(), humanSize) go func() { errch <- s.client.ExportImage(exportParams) }() // Iterate through the files in the archive. for { hdr, err := tr.Next() if err == io.EOF { break } if err != nil { cleanup() return "", "", err } // Skip "repositories" file, we will write our own if hdr.Name == "repositories" { io.Copy(ioutil.Discard, tr) continue } // Write any other file tw.WriteHeader(hdr) if _, err := io.Copy(tarHashStream, tr); err != nil { cleanup() return "", "", err } } // Write "repositories" file digest = fmt.Sprintf("sha256-%x", hash.Sum(nil)) reposBody, err := json.Marshal(Repositories{ img.NameWithRegistry(): { img.Tag: image.ID, digest: image.ID, }, }) if err != nil { cleanup() return "", "", err } hdr := &tar.Header{ Name: "repositories", Mode: 0644, Size: int64(len(reposBody)), } if err := tw.WriteHeader(hdr); err != nil { cleanup() return "", "", err } if _, err := tw.Write(reposBody); err != nil { cleanup() return "", "", err } // Finish tar if err := tw.Close(); err != nil { cleanup() return "", "", err } if err := <-errch; err != nil { cleanup() return "", "", fmt.Errorf("Failed to export docker image %s, error: %s", img, err) } // Cache digest by image ID if err := s.CachePut(image.ID, digest); err != nil { return "", "", fmt.Errorf("Failed to save digest cache, error: %s", err) } return tmpf.Name(), digest, nil }
// Clean finds the obsolete image tags from container specs that exist in docker daemon, // skipping topN images that we want to keep (keep_images, default 5) and deletes them. func (client *DockerClient) Clean(config *config.Config) error { // do not pull same image twice images := map[imagename.ImageName]*imagename.Tags{} keep := client.KeepImages // keep 5 latest images by default if keep == 0 { keep = 5 } for _, container := range GetContainersFromConfig(config) { if container.Image == nil { continue } images[*container.Image] = &imagename.Tags{} } if len(images) == 0 { return nil } // Go through every image and list existing tags all, err := client.Docker.ListImages(docker.ListImagesOptions{}) if err != nil { return fmt.Errorf("Failed to list all images, error: %s", err) } // collect tags for every image for _, image := range all { for _, repoTag := range image.RepoTags { imageName := imagename.NewFromString(repoTag) for img := range images { if img.IsSameKind(*imageName) { images[img].Items = append(images[img].Items, &imagename.Tag{ ID: image.ID, Name: *imageName, Created: image.Created, }) } } } } // for every image, delete obsolete tags for name, tags := range images { toDelete := tags.GetOld(keep) if len(toDelete) == 0 { continue } log.Infof("Cleanup: removing %d tags of image %s", len(toDelete), name.NameWithRegistry()) for _, n := range toDelete { if name.GetTag() == n.GetTag() { log.Infof("Cleanup: skipping %s because it is in the spec", n) continue } wasRemoved := true log.Infof("Cleanup: remove %s", n) if err := client.Docker.RemoveImageExtended(n.String(), docker.RemoveImageOptions{Force: false}); err != nil { // 409 is conflict, which means there is a container exists running under this image if e, ok := err.(*docker.Error); ok && e.Status == 409 { log.Infof("Cleanup: skip %s because there is an existing container using it", n) wasRemoved = false } else { return err } } // cannot refer to &n because of for loop if wasRemoved { removed := n client.removedImages = append(client.removedImages, &removed) } } } return nil }
// lookupImage looks up for the image by name and returns *docker.Image object (result of the inspect) // `Pull` config option defines whether we want to update the latest version of the image from the remote registry // See build.Config struct for more details about other build config options. // // If `Pull` is false, it tries to lookup locally by exact matching, e.g. if the image is already // pulled with that exact name given (no fuzzy semver matching) // // Then the function fetches the list of all pulled images and tries to match one of them by the given name. // // If `Pull` is set to true or if it cannot find the image locally, it then fetches all image // tags from the remote registry and finds the best match for the given image name. // // If it cannot find the image either locally or in the remote registry, it returns `nil` // // In case the given image has sha256 tag, it looks for it locally and pulls if it's not found. // No semver matching is done for sha256 tagged images. // // See also TestBuild_LookupImage_* test cases in build_test.go func (b *Build) lookupImage(name string) (img *docker.Image, err error) { var ( candidate, remoteCandidate *imagename.ImageName imgName = imagename.NewFromString(name) pull = false hub = b.cfg.Pull isSha = imgName.TagIsSha() ) // If hub is true, then there is no sense to inspect the local image if !hub || isSha { if isOld, warning := imagename.WarnIfOldS3ImageName(name); isOld { log.Warn(warning) } // Try to inspect image as is, without version resolution if img, err := b.client.InspectImage(imgName.String()); err != nil || img != nil { return img, err } } if isSha { // If we are still here and image not found locally, we want to pull it candidate = imgName hub = false pull = true } if !isSha && !hub { // List local images var localImages = []*imagename.ImageName{} if localImages, err = b.client.ListImages(); err != nil { return nil, err } // Resolve local candidate candidate = imgName.ResolveVersion(localImages, true) } // In case we want to include external images as well, pulling list of available // images from the remote registry if hub || candidate == nil { log.Debugf("Getting list of tags for %s from the registry", imgName) var remoteImages []*imagename.ImageName if remoteImages, err = b.client.ListImageTags(imgName.String()); err != nil { err = fmt.Errorf("Failed to list tags of image %s from the remote registry, error: %s", imgName, err) return } // Since we found the remote image, we want to pull it if remoteCandidate = imgName.ResolveVersion(remoteImages, false); remoteCandidate != nil { pull = true candidate = remoteCandidate // If we've found needed image on s3 it should be pulled in the same name style as lookuped image candidate.IsOldS3Name = imgName.IsOldS3Name } } // If not candidate found, it's an error if candidate == nil { err = fmt.Errorf("Image not found: %s (also checked in the remote registry)", imgName) return } if !isSha && imgName.GetTag() != candidate.GetTag() { if remoteCandidate != nil { log.Infof("Resolve %s --> %s (found remotely)", imgName, candidate.GetTag()) } else { log.Infof("Resolve %s --> %s", imgName, candidate.GetTag()) } } if pull { if err = b.client.PullImage(candidate.String()); err != nil { return } } return b.client.InspectImage(candidate.String()) }
// Pull imports docker image from tar artifact stored on S3 func (s *StorageS3) Pull(name string) error { img := imagename.NewFromString(name) if img.Storage != imagename.StorageS3 { return fmt.Errorf("Can only pull images with s3 storage specified, got: %s", img) } if img.Registry == "" { return fmt.Errorf("Cannot pull image from S3, missing bucket name, got: %s", img) } // TODO: here we use tmp file, but we can stream from S3 directly to Docker tmpf, err := ioutil.TempFile("", "rocker_image_") if err != nil { return err } defer os.Remove(tmpf.Name()) var ( // Create a downloader with the s3 client and custom options downloader = s3manager.NewDownloaderWithClient(s.s3, func(d *s3manager.Downloader) { d.PartSize = 64 * 1024 * 1024 // 64MB per part }) imgPath = img.Name + "/" + img.Tag + ".tar" downloadParams = &s3.GetObjectInput{ Bucket: aws.String(img.Registry), Key: aws.String(imgPath), } ) log.Infof("| Import %s/%s.tar to %s", img.NameWithRegistry(), img.Tag, tmpf.Name()) if err := s.retryer.Outer(func() error { _, err := downloader.Download(tmpf, downloadParams) return err }); err != nil { return fmt.Errorf("Failed to download object from S3, error: %s", err) } fd, err := os.Open(tmpf.Name()) if err != nil { return err } defer fd.Close() // Read through tar reader to patch repositories file since we might // mave a different tag property var ( pipeReader, pipeWriter = io.Pipe() tr = tar.NewReader(fd) tw = tar.NewWriter(pipeWriter) errch = make(chan error, 1) loadOptions = docker.LoadImageOptions{ InputStream: pipeReader, } ) go func() { errch <- s.client.LoadImage(loadOptions) }() // Iterate through the files in the archive. for { hdr, err := tr.Next() if err == io.EOF { break } if err != nil { return fmt.Errorf("Failed to read tar content, error: %s", err) } // Skip "repositories" file, we will write our own if hdr.Name == "repositories" { // Read repositories file and pass to JSON decoder r1 := Repositories{} data, err := ioutil.ReadAll(tr) if err != nil { return fmt.Errorf("Failed to read `repositories` file content, error: %s", err) } if err := json.Unmarshal(data, &r1); err != nil { return fmt.Errorf("Failed to parse `repositories` file json, error: %s", err) } var imageID string // Read first key from repositories for _, tags := range r1 { for _, id := range tags { imageID = id break } break } // Make a new repositories struct r2 := Repositories{ img.NameWithRegistry(): { img.GetTag(): imageID, }, } // Write repositories file to the stream reposBody, err := json.Marshal(r2) if err != nil { return fmt.Errorf("Failed to marshal `repositories` file json, error: %s", err) } hdr := &tar.Header{ Name: "repositories", Mode: 0644, Size: int64(len(reposBody)), } if err := tw.WriteHeader(hdr); err != nil { return fmt.Errorf("Failed to write `repositories` file tar header, error: %s", err) } if _, err := tw.Write(reposBody); err != nil { return fmt.Errorf("Failed to write `repositories` file to tar, error: %s", err) } continue } // Passthrough other files as is if err := tw.WriteHeader(hdr); err != nil { return fmt.Errorf("Failed to passthough tar header, error: %s", err) } if _, err := io.Copy(tw, tr); err != nil { return fmt.Errorf("Failed to passthough tar content, error: %s", err) } } // Finish tar if err := tw.Close(); err != nil { return fmt.Errorf("Failed to close tar writer, error: %s", err) } // Close pipeWriter if err := pipeWriter.Close(); err != nil { return fmt.Errorf("Failed to close tar pipeWriter, error: %s", err) } if err := <-errch; err != nil { errch <- fmt.Errorf("Failed to import image, error: %s", err) } return nil }
// resolveVersions walks through the list of images and resolves their tags in case they are not strict func (client *DockerClient) resolveVersions(local, hub bool, vars template.Vars, containers []*Container) (err error) { // Provide function getter of all images to fetch only once var available []*imagename.ImageName getImages := func() ([]*imagename.ImageName, error) { if available == nil { available = []*imagename.ImageName{} if !local { return available, nil } // retrieving images currently available in docker var dockerImages []docker.APIImages if dockerImages, err = client.Docker.ListImages(docker.ListImagesOptions{}); err != nil { return nil, err } for _, image := range dockerImages { for _, repoTag := range image.RepoTags { available = append(available, imagename.NewFromString(repoTag)) } } } return available, nil } resolved := map[string]*imagename.ImageName{} // check images for each container for _, container := range containers { // error in configuration, fail fast if container.Image == nil { err = fmt.Errorf("Image is not specified for the container: %s", container.Name) return } // Version specified in variables var k string k = fmt.Sprintf("v_image_%s", container.Image.NameWithRegistry()) if tag, ok := vars[k]; ok { log.Infof("Resolve %s --> %s (derived by variable %s)", container.Image, tag, k) container.Image.SetTag(tag.(string)) } k = fmt.Sprintf("v_container_%s", container.Name.Name) if tag, ok := vars[k]; ok { log.Infof("Resolve %s --> %s (derived by variable %s)", container.Image, tag, k) container.Image.SetTag(tag.(string)) } // Do not resolve anything if the image is strict, e.g. "redis:2.8.11" or "redis:latest" if container.Image.IsStrict() { continue } // already resolved it for other container if _, ok := resolved[container.Image.String()]; ok { container.Image = resolved[container.Image.String()] continue } // Override to not change the common images slice var images []*imagename.ImageName if images, err = getImages(); err != nil { return err } // looking locally first candidate := container.Image.ResolveVersion(images, true) // in case we want to include external images as well, pulling list of available // images from repository or central docker hub if hub || candidate == nil { log.Debugf("Getting list of tags for %s from the registry", container.Image) var remote []*imagename.ImageName if container.Image.Storage == imagename.StorageS3 { s3storage := s3.New(client.Docker, os.TempDir()) remote, err = s3storage.ListTags(container.Image.String()) } else { remote, err = dockerclient.RegistryListTags(container.Image, client.Auth) } if err != nil { return fmt.Errorf("Failed to list tags of image %s for container %s from the remote registry, error: %s", container.Image, container.Name, err) } log.Debugf("remote: %v", remote) // Re-Resolve having hub tags candidate = container.Image.ResolveVersion(append(images, remote...), false) } if candidate == nil { err = fmt.Errorf("Image not found: %s", container.Image) return } candidate.IsOldS3Name = container.Image.IsOldS3Name log.Infof("Resolve %s --> %s", container.Image, candidate.GetTag()) container.Image = candidate resolved[container.Image.String()] = candidate } return }
// ReadConfig reads and parses the config from io.Reader stream. // Before parsing it processes config through a template engine implemented in template.go. func ReadConfig(configName string, reader io.Reader, vars template.Vars, funcs map[string]interface{}, print bool) (*Config, error) { config := &Config{} basedir, err := os.Getwd() if err != nil { return nil, fmt.Errorf("Failed to get working dir, error: %s", err) } if configName == "-" { configName = "<STDIN>" } else { // if file given, process volume paths relative to the manifest file basedir = filepath.Dir(configName) } data, err := template.Process(configName, reader, vars, funcs) if err != nil { return nil, fmt.Errorf("Failed to process config template, error: %s", err) } if print { fmt.Print(data.String()) os.Exit(0) } if err := yaml.Unmarshal(data.Bytes(), config); err != nil { return nil, fmt.Errorf("Failed to parse YAML config, error: %s", err) } // empty namespace is a backward compatible docker-compose format // we will try to guess the namespace my parent directory name if config.Namespace == "" { parentDir := filepath.Base(basedir) config.Namespace = regexp.MustCompile("[^a-z0-9\\-\\_]").ReplaceAllString(parentDir, "") } // Save vars to config config.Vars = vars // Read extra data type ConfigExtra struct { Containers map[string]map[string]interface{} } extra := &ConfigExtra{} if err := yaml.Unmarshal(data.Bytes(), extra); err != nil { return nil, fmt.Errorf("Failed to parse YAML config extra properties, error: %s", err) } // Initialize YAML keys // Index yaml fields for better search yamlFields := make(map[string]bool) for _, v := range getYamlFields() { yamlFields[v] = true } // Function that gets HOME (initialize only once) homeMemo := "" getHome := func() (h string, err error) { if homeMemo == "" { if homeMemo, err = homedir.Dir(); err != nil { return "", err } } return homeMemo, nil } // Process aliases on the first run, have to do it before extends // because Golang randomizes maps, sometimes inherited containers // process earlier then dependencies; also do initial validation for name, container := range config.Containers { if container == nil { return nil, fmt.Errorf("Invalid specification for container `%s` in %s", name, configName) } // Handle aliases if container.Command != nil { if container.Cmd == nil { container.Cmd = container.Command } container.Command = nil } if container.Link != nil { if container.Links == nil { container.Links = container.Link } container.Link = nil } if container.Label != nil { if container.Labels == nil { container.Labels = container.Label } container.Label = nil } if container.Hosts != nil { if container.AddHost == nil { container.AddHost = container.Hosts } container.Hosts = nil } if container.ExtraHosts != nil { if container.AddHost == nil { container.AddHost = container.ExtraHosts } container.ExtraHosts = nil } if container.WorkingDir != nil { if container.Workdir == nil { container.Workdir = container.WorkingDir } container.WorkingDir = nil } if container.Environment != nil { if container.Env == nil { container.Env = container.Environment } container.Environment = nil } // Process extra data extraFields := map[string]interface{}{} for key, val := range extra.Containers[name] { if !yamlFields[key] { extraFields[key] = val } } if len(extraFields) > 0 { container.Extra = extraFields } // pretty.Println(name, container.Extra) } // Process extending containers configuration for name, container := range config.Containers { if container.Extends != "" { if container.Extends == name { return nil, fmt.Errorf("Container %s: cannot extend from itself", name) } if _, ok := config.Containers[container.Extends]; !ok { return nil, fmt.Errorf("Container %s: cannot find container %s to extend from", name, container.Extends) } // TODO: build dependency graph by extends hierarchy to allow multiple inheritance if config.Containers[container.Extends].Extends != "" { return nil, fmt.Errorf("Container %s: cannot extend from %s: multiple inheritance is not allowed yet", name, container.Extends) } container.ExtendFrom(config.Containers[container.Extends]) } // Validate image if container.Image == nil { return nil, fmt.Errorf("Image should be specified for container: %s", name) } img := imagename.NewFromString(*container.Image) if !img.IsStrict() && !img.HasVersionRange() && !img.All() { return nil, fmt.Errorf("Image `%s` for container `%s`: image without tag is not allowed", *container.Image, name) } // Set namespace for all containers inside for k := range container.VolumesFrom { container.VolumesFrom[k].DefaultNamespace(config.Namespace) } for k := range container.Links { container.Links[k].DefaultNamespace(config.Namespace) } for k := range container.WaitFor { container.WaitFor[k].DefaultNamespace(config.Namespace) } if container.Net != nil && container.Net.Type == "container" { container.Net.Container.DefaultNamespace(config.Namespace) } // Fix exposed ports for k, port := range container.Expose { if !strings.Contains(port, "/") { container.Expose[k] = port + "/tcp" } } // Process relative paths in volumes for i, volume := range container.Volumes { split := strings.SplitN(volume, ":", 2) if len(split) == 1 { continue } if strings.HasPrefix(split[0], "~") { home, err := getHome() if err != nil { return nil, fmt.Errorf("Failed to get HOME path, error: %s", err) } split[0] = strings.Replace(split[0], "~", home, 1) } if !path.IsAbs(split[0]) { split[0] = path.Join(basedir, split[0]) } container.Volumes[i] = strings.Join(split, ":") } } return config, nil }