Example #1
0
func TestBuild_LookupImage_ExistLocally(t *testing.T) {
	var (
		nilImage *docker.Image

		b, c        = makeBuild(t, "", Config{})
		resultImage = &docker.Image{ID: "789"}
		name        = "ubuntu:latest"

		localImages = []*imagename.ImageName{
			imagename.NewFromString("debian:7.7"),
			imagename.NewFromString("debian:latest"),
			imagename.NewFromString("ubuntu:12.04"),
			imagename.NewFromString("ubuntu:14.04"),
			imagename.NewFromString("ubuntu:latest"),
		}
	)

	c.On("InspectImage", name).Return(nilImage, nil).Once()
	c.On("ListImages").Return(localImages, nil).Once()
	c.On("InspectImage", name).Return(resultImage, nil).Once()

	result, err := b.lookupImage(name)
	if err != nil {
		t.Fatal(err)
	}

	assert.Equal(t, resultImage, result)
	c.AssertExpectations(t)
}
Example #2
0
// ListImageTags returns the list of images instances obtained from all tags existing in the registry
func (c *DockerClient) ListImageTags(name string) (images []*imagename.ImageName, err error) {
	img := imagename.NewFromString(name)
	if img.Storage == imagename.StorageS3 {
		return c.s3storage.ListTags(name)
	}
	return dockerclient.RegistryListTags(imagename.NewFromString(name), c.auth)
}
Example #3
0
func TestClientResolveVersions(t *testing.T) {
	t.Skip()

	dockerCli, err := dockerclient.New()
	if err != nil {
		t.Fatal(err)
	}

	client, err := NewClient(&DockerClient{
		Docker: dockerCli,
	})
	if err != nil {
		t.Fatal(err)
	}

	containers := []*Container{
		&Container{
			Name:  config.NewContainerName("test", "test"),
			Image: imagename.NewFromString("golang:1.4.*"),
		},
	}

	if err := client.resolveVersions(true, true, template.Vars{}, containers); err != nil {
		t.Fatal(err)
	}

	pretty.Println(containers)
}
Example #4
0
// NewContainerFromDocker converts a container object given by
// docker client to a local Container object
func NewContainerFromDocker(dockerContainer *docker.Container) (*Container, error) {
	cfg, err := config.NewFromDocker(dockerContainer)
	if err != nil {
		if _, ok := err.(config.ErrNotRockerCompose); !ok {
			return nil, err
		}
	}
	return &Container{
		ID:      dockerContainer.ID,
		Image:   imagename.NewFromString(dockerContainer.Config.Image),
		ImageID: dockerContainer.Image,
		Name:    config.NewContainerNameFromString(dockerContainer.Name),
		Created: dockerContainer.Created,
		State: &ContainerState{
			Running:    dockerContainer.State.Running,
			Paused:     dockerContainer.State.Paused,
			Restarting: dockerContainer.State.Restarting,
			OOMKilled:  dockerContainer.State.OOMKilled,
			Pid:        dockerContainer.State.Pid,
			ExitCode:   dockerContainer.State.ExitCode,
			Error:      dockerContainer.State.Error,
			StartedAt:  dockerContainer.State.StartedAt,
			FinishedAt: dockerContainer.State.FinishedAt,
		},
		Config:    cfg,
		container: dockerContainer,
	}, nil
}
Example #5
0
// Execute runs the command
func (c *CommandPush) Execute(b *Build) (State, error) {
	if len(c.cfg.args) != 1 {
		return b.state, fmt.Errorf("PUSH requires exactly one argument")
	}

	if b.state.ImageID == "" {
		return b.state, fmt.Errorf("Cannot PUSH empty image")
	}

	if err := b.client.TagImage(b.state.ImageID, c.cfg.args[0]); err != nil {
		return b.state, err
	}

	image := imagename.NewFromString(c.cfg.args[0])
	artifact := imagename.Artifact{
		Name:      image,
		Pushed:    b.cfg.Push,
		Tag:       image.GetTag(),
		ImageID:   b.state.ImageID,
		BuildTime: time.Now(),
	}

	// push image and add some lines to artifacts
	if b.cfg.Push {
		digest, err := b.client.PushImage(image.String())
		if err != nil {
			return b.state, err
		}
		artifact.SetDigest(digest)
	} else {
		log.Infof("| Don't push. Pass --push flag to actually push to the registry")
	}

	// Publish artifact files
	if b.cfg.ArtifactsPath != "" {
		if err := os.MkdirAll(b.cfg.ArtifactsPath, 0755); err != nil {
			return b.state, fmt.Errorf("Failed to create directory %s for the artifacts, error: %s", b.cfg.ArtifactsPath, err)
		}

		filePath := filepath.Join(b.cfg.ArtifactsPath, artifact.GetFileName())

		artifacts := imagename.Artifacts{
			RockerArtifacts: []imagename.Artifact{artifact},
		}
		content, err := yaml.Marshal(artifacts)
		if err != nil {
			return b.state, err
		}

		if err := ioutil.WriteFile(filePath, content, 0644); err != nil {
			return b.state, fmt.Errorf("Failed to write artifact file %s, error: %s", filePath, err)
		}

		log.Infof("| Saved artifact file %s", filePath)
		log.Debugf("Artifact properties: %# v", pretty.Formatter(artifact))
	}

	return b.state, nil
}
Example #6
0
func TestBuild_LookupImage_PullAndNotExist(t *testing.T) {
	var (
		b, c = makeBuild(t, "", Config{Pull: true})
		name = "ubuntu:latest"

		remoteImages = []*imagename.ImageName{
			imagename.NewFromString("debian:7.7"),
			imagename.NewFromString("debian:latest"),
			imagename.NewFromString("ubuntu:12.04"),
			imagename.NewFromString("ubuntu:14.04"),
		}
	)

	c.On("ListImageTags", name).Return(remoteImages, nil).Once()

	_, err := b.lookupImage(name)
	assert.EqualError(t, err, "Image not found: ubuntu:latest (also checked in the remote registry)")
	c.AssertExpectations(t)
}
Example #7
0
// NewContainerFromConfig makes a single Container object from a spec Config object.
func NewContainerFromConfig(name *config.ContainerName, containerConfig *config.Container) *Container {
	container := &Container{
		Name: name,
		State: &ContainerState{
			Running: containerConfig.State.Bool(),
		},
		Config: containerConfig,
	}
	if containerConfig.Image != nil {
		container.Image = imagename.NewFromString(*containerConfig.Image)
	}
	return container
}
Example #8
0
// PullImage pulls docker image
func (c *DockerClient) PullImage(name string) error {
	image := imagename.NewFromString(name)

	// e.g. s3:bucket-name/image-name
	if image.Storage == imagename.StorageS3 {
		if isOld, warning := imagename.WarnIfOldS3ImageName(name); isOld {
			c.log.Warn(warning)
		}

		return c.s3storage.Pull(name)
	}

	var (
		pipeReader, pipeWriter = io.Pipe()
		fdOut, isTerminalOut   = term.GetFdInfo(c.log.Out)
		out                    = c.log.Out
		errch                  = make(chan error, 1)
	)

	if !isTerminalOut {
		out = c.log.Writer()
	}

	opts := docker.PullImageOptions{
		Repository:    image.NameWithRegistry(),
		Registry:      image.Registry,
		Tag:           image.GetTag(),
		OutputStream:  pipeWriter,
		RawJSONStream: true,
	}

	c.log.Infof("| Pull image %s", image)
	c.log.Debugf("Pull image %s with options: %# v", image, opts)

	go func() {
		errch <- jsonmessage.DisplayJSONMessagesStream(pipeReader, out, fdOut, isTerminalOut)
	}()

	auth, err := dockerclient.GetAuthForRegistry(c.auth, image)
	if err != nil {
		return fmt.Errorf("Failed to authenticate registry %s, error: %s", image.Registry, err)
	}

	if err := c.client.PullImage(opts, auth); err != nil {
		return err
	}

	pipeWriter.Close()
	return <-errch
}
Example #9
0
// ListImages lists all pulled images in the local docker registry
func (c *DockerClient) ListImages() (images []*imagename.ImageName, err error) {

	var dockerImages []docker.APIImages
	if dockerImages, err = c.client.ListImages(docker.ListImagesOptions{}); err != nil {
		return
	}

	images = []*imagename.ImageName{}
	for _, image := range dockerImages {
		for _, repoTag := range image.RepoTags {
			images = append(images, imagename.NewFromString(repoTag))
		}
	}

	return
}
Example #10
0
// TagImage adds tag to the image
func (c *DockerClient) TagImage(imageID, imageName string) error {
	img := imagename.NewFromString(imageName)
	if isOld, warning := imagename.WarnIfOldS3ImageName(imageName); isOld {
		c.log.Warn(warning)
	}

	c.log.Infof("| Tag %.12s -> %s", imageID, img)

	opts := docker.TagImageOptions{
		Repo:  img.NameWithRegistry(),
		Tag:   img.GetTag(),
		Force: true,
	}

	c.log.Debugf("Tag image %s with options: %# v", imageID, opts)

	return c.client.TagImage(imageID, opts)
}
Example #11
0
// GetBridgeIP gets the ip address of docker network bridge
// it is useful when you want to loose couple containers and not have tightly link them
// container A may publish port 8125 to host network and container B may access this port through
// a bridge ip address; it's a hacky solution, any better way to obtain bridge ip without ssh access
// to host machine is welcome
//
// Here we create a dummy container and look at .NetworkSettings.Gateway value
//
// TODO: maybe we don't need this anymore since docker 1.8 seem to specify all existing containers
// 			 in a /etc/hosts file of every contianer. Need to research it further.
//
// https://github.com/docker/docker/issues/1143
// https://github.com/docker/docker/issues/11247
//
func GetBridgeIP(client *docker.Client) (ip string, err error) {
	// Ensure empty image existing
	_, err = client.InspectImage(emptyImageName)
	if err != nil && err.Error() == "no such image" {
		log.Infof("Pulling image %s to obtain network bridge address", emptyImageName)
		if _, err := PullDockerImage(client, imagename.NewFromString(emptyImageName), nil); err != nil {
			return "", err
		}
	} else if err != nil {
		return "", fmt.Errorf("Failed to inspect image %s, error: %s", emptyImageName, err)
	}

	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: emptyImageName,
			Cmd:   []string{"/bin/sh", "-c", "while true; do sleep 1; done"},
		},
		HostConfig: &docker.HostConfig{},
	})
	if err != nil {
		return "", fmt.Errorf("Failed to create dummy network container, error: %s", err)
	}
	defer func() {
		removeOpts := docker.RemoveContainerOptions{
			ID:            container.ID,
			Force:         true,
			RemoveVolumes: true,
		}
		if err2 := client.RemoveContainer(removeOpts); err2 != nil && err == nil {
			err = err2
		}
	}()

	if err := client.StartContainer(container.ID, &docker.HostConfig{}); err != nil {
		return "", fmt.Errorf("Failed to start dummy network container %.12s, error: %s", container.ID, err)
	}

	inspect, err := client.InspectContainer(container.ID)
	if err != nil {
		return "", fmt.Errorf("Failed to inspect dummy network container %.12s, error: %s", container.ID, err)
	}

	return inspect.NetworkSettings.Gateway, nil
}
Example #12
0
// ListTags returns the list of parsed tags existing for given image name on S3
func (s *StorageS3) ListTags(imageName string) (images []*imagename.ImageName, err error) {
	image := imagename.NewFromString(imageName)

	params := &s3.ListObjectsInput{
		Bucket:  aws.String(image.Registry),
		MaxKeys: aws.Int64(1000),
		Prefix:  aws.String(image.Name),
	}

	resp, err := s.s3.ListObjects(params)
	if err != nil {
		return nil, err
	}

	for _, s3Obj := range resp.Contents {
		split := strings.Split(*s3Obj.Key, "/")
		if len(split) < 2 {
			continue
		}

		imgName := strings.Join(split[:len(split)-1], "/")
		imgName = fmt.Sprintf("s3.amazonaws.com/%s/%s", image.Registry, imgName)

		tag := strings.TrimSuffix(split[len(split)-1], ".tar")
		candidate := imagename.New(imgName, tag)

		if candidate.Name != image.Name {
			continue
		}

		if image.Contains(candidate) || image.Tag == candidate.Tag {
			images = append(images, candidate)
		}
	}

	return
}
Example #13
0
// pushImageInner pushes the image is the inner straightforward push without retries
func (c *DockerClient) pushImageInner(imageName string) (digest string, err error) {
	img := imagename.NewFromString(imageName)

	// Use direct S3 image pusher instead
	if img.Storage == imagename.StorageS3 {
		if isOld, warning := imagename.WarnIfOldS3ImageName(imageName); isOld {
			c.log.Warn(warning)
		}
		return c.s3storage.Push(imageName)
	}

	var (
		buf                    bytes.Buffer
		pipeReader, pipeWriter = io.Pipe()
		outStream              = io.MultiWriter(pipeWriter, &buf)
		fdOut, isTerminalOut   = term.GetFdInfo(c.log.Out)
		out                    = c.log.Out

		opts = docker.PushImageOptions{
			Name:          img.NameWithRegistry(),
			Tag:           img.GetTag(),
			Registry:      img.Registry,
			OutputStream:  outStream,
			RawJSONStream: true,
		}
		errch = make(chan error, 1)
	)

	if !isTerminalOut {
		out = c.log.Writer()
	}

	c.log.Infof("| Push %s", img)

	c.log.Debugf("Push with options: %# v", opts)

	// TODO: DisplayJSONMessagesStream may fail by client.PushImage run without errors
	go func() {
		errch <- jsonmessage.DisplayJSONMessagesStream(pipeReader, out, fdOut, isTerminalOut)
	}()

	auth, err := dockerclient.GetAuthForRegistry(c.auth, img)
	if err != nil {
		return "", fmt.Errorf("Failed to authenticate registry %s, error: %s", img.Registry, err)
	}

	if err := c.client.PushImage(opts, auth); err != nil {
		return "", err
	}
	pipeWriter.Close()

	if err := <-errch; err != nil {
		return "", fmt.Errorf("Failed to process json stream, error %s", err)
	}

	// It is the best way to have pushed image digest so far
	matches := captureDigest.FindStringSubmatch(buf.String())
	if len(matches) > 0 {
		digest = matches[1]
	}

	return digest, nil
}
Example #14
0
func TestClientClean(t *testing.T) {
	// This test involves interaction with docker
	// enable it in case you want to test Clean() functionality
	t.Skip()

	dockerCli, err := dockerclient.New()
	if err != nil {
		t.Fatal(err)
	}

	// Create number of images to test

	createdContainers := []string{}
	createdImages := []string{}

	defer func() {
		for _, id := range createdContainers {
			if err := dockerCli.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true}); err != nil {
				t.Error(err)
			}
		}
		for _, id := range createdImages {
			if err := dockerCli.RemoveImageExtended(id, docker.RemoveImageOptions{Force: true}); err != nil {
				if err.Error() == "no such image" {
					continue
				}
				t.Error(err)
			}
		}
	}()

	for i := 1; i <= 5; i++ {
		c, err := dockerCli.CreateContainer(docker.CreateContainerOptions{
			Config: &docker.Config{
				Image: "gliderlabs/alpine:3.1",
				Cmd:   []string{"true"},
			},
		})
		if err != nil {
			t.Fatal(err)
		}

		createdContainers = append(createdContainers, c.ID)

		commitOpts := docker.CommitContainerOptions{
			Container:  c.ID,
			Repository: "rocker-compose-test-image-clean",
			Tag:        fmt.Sprintf("%d", i),
		}

		img, err := dockerCli.CommitContainer(commitOpts)
		if err != nil {
			t.Fatal(err)
		}

		createdImages = append(createdImages, img.ID)

		// Make sure images have different timestamps
		time.Sleep(time.Second)
	}

	////////////////////////

	cli, err := NewClient(&DockerClient{Docker: dockerCli, KeepImages: 2})
	if err != nil {
		t.Fatal(err)
	}

	yml := `
namespace: test
containers:
  main:
    image: rocker-compose-test-image-clean:5
`

	config, err := config.ReadConfig("test.yml", strings.NewReader(yml), map[string]interface{}{}, map[string]interface{}{}, false)
	if err != nil {
		t.Fatal(err)
	}

	if err := cli.Clean(config); err != nil {
		t.Fatal(err)
	}

	// test that images left

	all, err := dockerCli.ListImages(docker.ListImagesOptions{})
	if err != nil {
		t.Fatal(err)
	}

	n := 0
	for _, image := range all {
		for _, repoTag := range image.RepoTags {
			imageName := imagename.NewFromString(repoTag)
			if imageName.Name == "rocker-compose-test-image-clean" {
				n++
			}
		}
	}

	assert.Equal(t, 2, n, "Expected images to be cleaned up")

	// test removed images list

	removed := cli.GetRemovedImages()
	assert.Equal(t, 3, len(removed), "Expected to remove a particular number of images")

	assert.EqualValues(t, "rocker-compose-test-image-clean:3", removed[0].String(), "removed wrong image")
	assert.EqualValues(t, "rocker-compose-test-image-clean:2", removed[1].String(), "removed wrong image")
	assert.EqualValues(t, "rocker-compose-test-image-clean:1", removed[2].String(), "removed wrong image")
}
Example #15
0
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

var (
	configTemplateVars = Vars{
		"mykey": "myval",
		"n":     "5",
		"data": map[string]string{
			"foo": "bar",
		},
		"RockerArtifacts": []imagename.Artifact{
			imagename.Artifact{
				Name: imagename.NewFromString("alpine:3.2"),
				Tag:  "3.2",
			},
			imagename.Artifact{
				Name:   imagename.NewFromString("golang:1.5"),
				Tag:    "1.5",
				Digest: "sha256:ead434",
			},
			imagename.Artifact{
				Name:   imagename.NewFromString("data:master"),
				Tag:    "master",
				Digest: "sha256:fafe14",
			},
			imagename.Artifact{
				Name:   imagename.NewFromString("ssh:latest"),
				Tag:    "latest",
Example #16
0
// Push pushes image tarball directly to S3
func (s *StorageS3) Push(imageName string) (digest string, err error) {
	img := imagename.NewFromString(imageName)

	if img.Storage != imagename.StorageS3 {
		return "", fmt.Errorf("Can only push images with s3 storage specified, got: %s", img)
	}

	if img.Registry == "" {
		return "", fmt.Errorf("Cannot push image to S3, missing bucket name, got: %s", img)
	}

	var image *docker.Image
	if image, err = s.client.InspectImage(img.String()); err != nil {
		return "", err
	}

	if digest, err = s.CacheGet(image.ID); err != nil {
		return "", err
	}

	var tmpf string

	defer func() {
		if tmpf != "" {
			os.Remove(tmpf)
		}
	}()

	// Not cached, make tar
	if digest == "" {
		if tmpf, digest, err = s.MakeTar(imageName); err != nil {
			return "", err
		}
	}

	var (
		ext           = ".tar"
		imgPathDigest = fmt.Sprintf("%s/%s%s", img.Name, digest, ext)
		imgPathTag    = fmt.Sprintf("%s/%s%s", img.Name, img.Tag, ext)
	)

	// Make HEAD request to s3 and check if image already uploaded
	_, headErr := s.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(img.Registry),
		Key:    aws.String(imgPathDigest),
	})

	// Object not found, need to store
	if headErr != nil {
		// Other error, raise then
		if e, ok := headErr.(awserr.RequestFailure); !ok || e.StatusCode() != 404 {
			return "", headErr
		}
		// In case we do not have archive
		if tmpf == "" {
			var digest2 string
			if tmpf, digest2, err = s.MakeTar(imageName); err != nil {
				return "", err
			}
			// Verify digest (TODO: remote this check in future?)
			if digest != digest2 {
				return "", fmt.Errorf("The new digest does no equal old one (shouldn't happen) %s != %s", digest, digest2)
			}
		}

		uploader := s3manager.NewUploaderWithClient(s.s3, func(u *s3manager.Uploader) {
			u.PartSize = 64 * 1024 * 1024 // 64MB per part
		})

		fd, err := os.Open(tmpf)
		if err != nil {
			return "", err
		}
		defer fd.Close()

		log.Infof("| Uploading image to s3.amazonaws.com/%s/%s", img.Registry, imgPathDigest)

		uploadParams := &s3manager.UploadInput{
			Bucket:      aws.String(img.Registry),
			Key:         aws.String(imgPathDigest),
			ContentType: aws.String("application/x-tar"),
			Body:        fd,
			Metadata: map[string]*string{
				"Tag":     aws.String(img.Tag),
				"ImageID": aws.String(image.ID),
				"Digest":  aws.String(digest),
			},
		}

		if err := s.retryer.Outer(func() error {
			_, err := uploader.Upload(uploadParams)
			return err
		}); err != nil {
			return "", fmt.Errorf("Failed to upload object to S3, error: %s", err)
		}
	}

	// Make a content addressable copy of an image file
	copyParams := &s3.CopyObjectInput{
		Bucket:     aws.String(img.Registry),
		CopySource: aws.String(img.Registry + "/" + imgPathDigest),
		Key:        aws.String(imgPathTag),
	}

	log.Infof("| Make alias s3.amazonaws.com/%s/%s", img.Registry, imgPathTag)

	if _, err = s.s3.CopyObject(copyParams); err != nil {
		return "", fmt.Errorf("Failed to PUT object to S3, error: %s", err)
	}

	return digest, nil
}
Example #17
0
// MakeTar makes a tar out of docker image and gives a temporary file and a digest
func (s *StorageS3) MakeTar(imageName string) (tmpfile string, digest string, err error) {
	img := imagename.NewFromString(imageName)

	var image *docker.Image
	if image, err = s.client.InspectImage(img.String()); err != nil {
		return "", "", err
	}

	tmpf, err := ioutil.TempFile("", "rocker_image_")
	if err != nil {
		return "", "", err
	}
	defer tmpf.Close()

	var (
		cleanup = func() {
			os.Remove(tmpf.Name())
		}

		humanSize = units.HumanSize(float64(image.VirtualSize))
		errch     = make(chan error, 1)

		pipeReader, pipeWriter = io.Pipe()
		tr                     = tar.NewReader(pipeReader)
		tw                     = tar.NewWriter(tmpf)
		hash                   = sha256.New()
		tarHashStream          = io.MultiWriter(tw, hash)

		exportParams = docker.ExportImageOptions{
			Name:         img.String(),
			OutputStream: pipeWriter,
		}
	)

	defer pipeWriter.Close()

	log.Infof("| Buffering image to a file %s (%s)", tmpf.Name(), humanSize)

	go func() {
		errch <- s.client.ExportImage(exportParams)
	}()

	// Iterate through the files in the archive.
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			cleanup()
			return "", "", err
		}

		// Skip "repositories" file, we will write our own
		if hdr.Name == "repositories" {
			io.Copy(ioutil.Discard, tr)
			continue
		}

		// Write any other file
		tw.WriteHeader(hdr)
		if _, err := io.Copy(tarHashStream, tr); err != nil {
			cleanup()
			return "", "", err
		}
	}

	// Write "repositories" file
	digest = fmt.Sprintf("sha256-%x", hash.Sum(nil))

	reposBody, err := json.Marshal(Repositories{
		img.NameWithRegistry(): {
			img.Tag: image.ID,
			digest:  image.ID,
		},
	})
	if err != nil {
		cleanup()
		return "", "", err
	}

	hdr := &tar.Header{
		Name: "repositories",
		Mode: 0644,
		Size: int64(len(reposBody)),
	}
	if err := tw.WriteHeader(hdr); err != nil {
		cleanup()
		return "", "", err
	}
	if _, err := tw.Write(reposBody); err != nil {
		cleanup()
		return "", "", err
	}

	// Finish tar

	if err := tw.Close(); err != nil {
		cleanup()
		return "", "", err
	}

	if err := <-errch; err != nil {
		cleanup()
		return "", "", fmt.Errorf("Failed to export docker image %s, error: %s", img, err)
	}

	// Cache digest by image ID
	if err := s.CachePut(image.ID, digest); err != nil {
		return "", "", fmt.Errorf("Failed to save digest cache, error: %s", err)
	}

	return tmpf.Name(), digest, nil
}
Example #18
0
// Clean finds the obsolete image tags from container specs that exist in docker daemon,
// skipping topN images that we want to keep (keep_images, default 5) and deletes them.
func (client *DockerClient) Clean(config *config.Config) error {
	// do not pull same image twice
	images := map[imagename.ImageName]*imagename.Tags{}
	keep := client.KeepImages

	// keep 5 latest images by default
	if keep == 0 {
		keep = 5
	}

	for _, container := range GetContainersFromConfig(config) {
		if container.Image == nil {
			continue
		}
		images[*container.Image] = &imagename.Tags{}
	}

	if len(images) == 0 {
		return nil
	}

	// Go through every image and list existing tags
	all, err := client.Docker.ListImages(docker.ListImagesOptions{})
	if err != nil {
		return fmt.Errorf("Failed to list all images, error: %s", err)
	}

	// collect tags for every image
	for _, image := range all {
		for _, repoTag := range image.RepoTags {
			imageName := imagename.NewFromString(repoTag)
			for img := range images {
				if img.IsSameKind(*imageName) {
					images[img].Items = append(images[img].Items, &imagename.Tag{
						ID:      image.ID,
						Name:    *imageName,
						Created: image.Created,
					})
				}
			}
		}
	}

	// for every image, delete obsolete tags
	for name, tags := range images {
		toDelete := tags.GetOld(keep)
		if len(toDelete) == 0 {
			continue
		}

		log.Infof("Cleanup: removing %d tags of image %s", len(toDelete), name.NameWithRegistry())
		for _, n := range toDelete {
			if name.GetTag() == n.GetTag() {
				log.Infof("Cleanup: skipping %s because it is in the spec", n)
				continue
			}

			wasRemoved := true

			log.Infof("Cleanup: remove %s", n)
			if err := client.Docker.RemoveImageExtended(n.String(), docker.RemoveImageOptions{Force: false}); err != nil {
				// 409 is conflict, which means there is a container exists running under this image
				if e, ok := err.(*docker.Error); ok && e.Status == 409 {
					log.Infof("Cleanup: skip %s because there is an existing container using it", n)
					wasRemoved = false
				} else {
					return err
				}
			}

			// cannot refer to &n because of for loop
			if wasRemoved {
				removed := n
				client.removedImages = append(client.removedImages, &removed)
			}
		}
	}

	return nil
}
Example #19
0
// lookupImage looks up for the image by name and returns *docker.Image object (result of the inspect)
// `Pull` config option defines whether we want to update the latest version of the image from the remote registry
// See build.Config struct for more details about other build config options.
//
// If `Pull` is false, it tries to lookup locally by exact matching, e.g. if the image is already
// pulled with that exact name given (no fuzzy semver matching)
//
// Then the function fetches the list of all pulled images and tries to match one of them by the given name.
//
// If `Pull` is set to true or if it cannot find the image locally, it then fetches all image
// tags from the remote registry and finds the best match for the given image name.
//
// If it cannot find the image either locally or in the remote registry, it returns `nil`
//
// In case the given image has sha256 tag, it looks for it locally and pulls if it's not found.
// No semver matching is done for sha256 tagged images.
//
// See also TestBuild_LookupImage_* test cases in build_test.go
func (b *Build) lookupImage(name string) (img *docker.Image, err error) {
	var (
		candidate, remoteCandidate *imagename.ImageName

		imgName = imagename.NewFromString(name)
		pull    = false
		hub     = b.cfg.Pull
		isSha   = imgName.TagIsSha()
	)

	// If hub is true, then there is no sense to inspect the local image
	if !hub || isSha {
		if isOld, warning := imagename.WarnIfOldS3ImageName(name); isOld {
			log.Warn(warning)
		}
		// Try to inspect image as is, without version resolution
		if img, err := b.client.InspectImage(imgName.String()); err != nil || img != nil {
			return img, err
		}
	}

	if isSha {
		// If we are still here and image not found locally, we want to pull it
		candidate = imgName
		hub = false
		pull = true
	}

	if !isSha && !hub {
		// List local images
		var localImages = []*imagename.ImageName{}
		if localImages, err = b.client.ListImages(); err != nil {
			return nil, err
		}
		// Resolve local candidate
		candidate = imgName.ResolveVersion(localImages, true)
	}

	// In case we want to include external images as well, pulling list of available
	// images from the remote registry
	if hub || candidate == nil {
		log.Debugf("Getting list of tags for %s from the registry", imgName)

		var remoteImages []*imagename.ImageName

		if remoteImages, err = b.client.ListImageTags(imgName.String()); err != nil {
			err = fmt.Errorf("Failed to list tags of image %s from the remote registry, error: %s", imgName, err)
			return
		}

		// Since we found the remote image, we want to pull it
		if remoteCandidate = imgName.ResolveVersion(remoteImages, false); remoteCandidate != nil {
			pull = true
			candidate = remoteCandidate

			// If we've found needed image on s3 it should be pulled in the same name style as lookuped image
			candidate.IsOldS3Name = imgName.IsOldS3Name
		}
	}

	// If not candidate found, it's an error
	if candidate == nil {
		err = fmt.Errorf("Image not found: %s (also checked in the remote registry)", imgName)
		return
	}

	if !isSha && imgName.GetTag() != candidate.GetTag() {
		if remoteCandidate != nil {
			log.Infof("Resolve %s --> %s (found remotely)", imgName, candidate.GetTag())
		} else {
			log.Infof("Resolve %s --> %s", imgName, candidate.GetTag())
		}
	}

	if pull {
		if err = b.client.PullImage(candidate.String()); err != nil {
			return
		}
	}

	return b.client.InspectImage(candidate.String())
}
Example #20
0
// Pull imports docker image from tar artifact stored on S3
func (s *StorageS3) Pull(name string) error {
	img := imagename.NewFromString(name)

	if img.Storage != imagename.StorageS3 {
		return fmt.Errorf("Can only pull images with s3 storage specified, got: %s", img)
	}

	if img.Registry == "" {
		return fmt.Errorf("Cannot pull image from S3, missing bucket name, got: %s", img)
	}

	// TODO: here we use tmp file, but we can stream from S3 directly to Docker
	tmpf, err := ioutil.TempFile("", "rocker_image_")
	if err != nil {
		return err
	}
	defer os.Remove(tmpf.Name())

	var (
		// Create a downloader with the s3 client and custom options
		downloader = s3manager.NewDownloaderWithClient(s.s3, func(d *s3manager.Downloader) {
			d.PartSize = 64 * 1024 * 1024 // 64MB per part
		})

		imgPath = img.Name + "/" + img.Tag + ".tar"

		downloadParams = &s3.GetObjectInput{
			Bucket: aws.String(img.Registry),
			Key:    aws.String(imgPath),
		}
	)

	log.Infof("| Import %s/%s.tar to %s", img.NameWithRegistry(), img.Tag, tmpf.Name())

	if err := s.retryer.Outer(func() error {
		_, err := downloader.Download(tmpf, downloadParams)
		return err
	}); err != nil {
		return fmt.Errorf("Failed to download object from S3, error: %s", err)
	}

	fd, err := os.Open(tmpf.Name())
	if err != nil {
		return err
	}
	defer fd.Close()

	// Read through tar reader to patch repositories file since we might
	// mave a different tag property
	var (
		pipeReader, pipeWriter = io.Pipe()
		tr                     = tar.NewReader(fd)
		tw                     = tar.NewWriter(pipeWriter)
		errch                  = make(chan error, 1)

		loadOptions = docker.LoadImageOptions{
			InputStream: pipeReader,
		}
	)

	go func() {
		errch <- s.client.LoadImage(loadOptions)
	}()

	// Iterate through the files in the archive.
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("Failed to read tar content, error: %s", err)
		}

		// Skip "repositories" file, we will write our own
		if hdr.Name == "repositories" {
			// Read repositories file and pass to JSON decoder
			r1 := Repositories{}
			data, err := ioutil.ReadAll(tr)
			if err != nil {
				return fmt.Errorf("Failed to read `repositories` file content, error: %s", err)
			}
			if err := json.Unmarshal(data, &r1); err != nil {
				return fmt.Errorf("Failed to parse `repositories` file json, error: %s", err)
			}

			var imageID string

			// Read first key from repositories
			for _, tags := range r1 {
				for _, id := range tags {
					imageID = id
					break
				}
				break
			}

			// Make a new repositories struct
			r2 := Repositories{
				img.NameWithRegistry(): {
					img.GetTag(): imageID,
				},
			}

			// Write repositories file to the stream
			reposBody, err := json.Marshal(r2)
			if err != nil {
				return fmt.Errorf("Failed to marshal `repositories` file json, error: %s", err)
			}

			hdr := &tar.Header{
				Name: "repositories",
				Mode: 0644,
				Size: int64(len(reposBody)),
			}
			if err := tw.WriteHeader(hdr); err != nil {
				return fmt.Errorf("Failed to write `repositories` file tar header, error: %s", err)
			}
			if _, err := tw.Write(reposBody); err != nil {
				return fmt.Errorf("Failed to write `repositories` file to tar, error: %s", err)
			}

			continue
		}

		// Passthrough other files as is
		if err := tw.WriteHeader(hdr); err != nil {
			return fmt.Errorf("Failed to passthough tar header, error: %s", err)
		}
		if _, err := io.Copy(tw, tr); err != nil {
			return fmt.Errorf("Failed to passthough tar content, error: %s", err)
		}
	}

	// Finish tar
	if err := tw.Close(); err != nil {
		return fmt.Errorf("Failed to close tar writer, error: %s", err)
	}

	// Close pipeWriter
	if err := pipeWriter.Close(); err != nil {
		return fmt.Errorf("Failed to close tar pipeWriter, error: %s", err)
	}

	if err := <-errch; err != nil {
		errch <- fmt.Errorf("Failed to import image, error: %s", err)
	}

	return nil
}
Example #21
0
// resolveVersions walks through the list of images and resolves their tags in case they are not strict
func (client *DockerClient) resolveVersions(local, hub bool, vars template.Vars, containers []*Container) (err error) {

	// Provide function getter of all images to fetch only once
	var available []*imagename.ImageName
	getImages := func() ([]*imagename.ImageName, error) {
		if available == nil {
			available = []*imagename.ImageName{}

			if !local {
				return available, nil
			}

			// retrieving images currently available in docker
			var dockerImages []docker.APIImages
			if dockerImages, err = client.Docker.ListImages(docker.ListImagesOptions{}); err != nil {
				return nil, err
			}

			for _, image := range dockerImages {
				for _, repoTag := range image.RepoTags {
					available = append(available, imagename.NewFromString(repoTag))
				}
			}
		}
		return available, nil
	}

	resolved := map[string]*imagename.ImageName{}

	// check images for each container
	for _, container := range containers {
		// error in configuration, fail fast
		if container.Image == nil {
			err = fmt.Errorf("Image is not specified for the container: %s", container.Name)
			return
		}

		// Version specified in variables
		var k string
		k = fmt.Sprintf("v_image_%s", container.Image.NameWithRegistry())
		if tag, ok := vars[k]; ok {
			log.Infof("Resolve %s --> %s (derived by variable %s)", container.Image, tag, k)
			container.Image.SetTag(tag.(string))
		}
		k = fmt.Sprintf("v_container_%s", container.Name.Name)
		if tag, ok := vars[k]; ok {
			log.Infof("Resolve %s --> %s (derived by variable %s)", container.Image, tag, k)
			container.Image.SetTag(tag.(string))
		}

		// Do not resolve anything if the image is strict, e.g. "redis:2.8.11" or "redis:latest"
		if container.Image.IsStrict() {
			continue
		}

		// already resolved it for other container
		if _, ok := resolved[container.Image.String()]; ok {
			container.Image = resolved[container.Image.String()]
			continue
		}

		// Override to not change the common images slice
		var images []*imagename.ImageName
		if images, err = getImages(); err != nil {
			return err
		}

		// looking locally first
		candidate := container.Image.ResolveVersion(images, true)

		// in case we want to include external images as well, pulling list of available
		// images from repository or central docker hub
		if hub || candidate == nil {
			log.Debugf("Getting list of tags for %s from the registry", container.Image)

			var remote []*imagename.ImageName

			if container.Image.Storage == imagename.StorageS3 {
				s3storage := s3.New(client.Docker, os.TempDir())
				remote, err = s3storage.ListTags(container.Image.String())
			} else {
				remote, err = dockerclient.RegistryListTags(container.Image, client.Auth)
			}

			if err != nil {
				return fmt.Errorf("Failed to list tags of image %s for container %s from the remote registry, error: %s",
					container.Image, container.Name, err)
			}

			log.Debugf("remote: %v", remote)

			// Re-Resolve having hub tags
			candidate = container.Image.ResolveVersion(append(images, remote...), false)
		}

		if candidate == nil {
			err = fmt.Errorf("Image not found: %s", container.Image)
			return
		}
		candidate.IsOldS3Name = container.Image.IsOldS3Name

		log.Infof("Resolve %s --> %s", container.Image, candidate.GetTag())

		container.Image = candidate
		resolved[container.Image.String()] = candidate
	}

	return
}
Example #22
0
// ReadConfig reads and parses the config from io.Reader stream.
// Before parsing it processes config through a template engine implemented in template.go.
func ReadConfig(configName string, reader io.Reader, vars template.Vars, funcs map[string]interface{}, print bool) (*Config, error) {
	config := &Config{}

	basedir, err := os.Getwd()
	if err != nil {
		return nil, fmt.Errorf("Failed to get working dir, error: %s", err)
	}

	if configName == "-" {
		configName = "<STDIN>"
	} else {
		// if file given, process volume paths relative to the manifest file
		basedir = filepath.Dir(configName)
	}

	data, err := template.Process(configName, reader, vars, funcs)
	if err != nil {
		return nil, fmt.Errorf("Failed to process config template, error: %s", err)
	}

	if print {
		fmt.Print(data.String())
		os.Exit(0)
	}

	if err := yaml.Unmarshal(data.Bytes(), config); err != nil {
		return nil, fmt.Errorf("Failed to parse YAML config, error: %s", err)
	}

	// empty namespace is a backward compatible docker-compose format
	// we will try to guess the namespace my parent directory name
	if config.Namespace == "" {
		parentDir := filepath.Base(basedir)
		config.Namespace = regexp.MustCompile("[^a-z0-9\\-\\_]").ReplaceAllString(parentDir, "")
	}

	// Save vars to config
	config.Vars = vars

	// Read extra data
	type ConfigExtra struct {
		Containers map[string]map[string]interface{}
	}
	extra := &ConfigExtra{}
	if err := yaml.Unmarshal(data.Bytes(), extra); err != nil {
		return nil, fmt.Errorf("Failed to parse YAML config extra properties, error: %s", err)
	}

	// Initialize YAML keys
	// Index yaml fields for better search
	yamlFields := make(map[string]bool)
	for _, v := range getYamlFields() {
		yamlFields[v] = true
	}

	// Function that gets HOME (initialize only once)
	homeMemo := ""
	getHome := func() (h string, err error) {
		if homeMemo == "" {
			if homeMemo, err = homedir.Dir(); err != nil {
				return "", err
			}
		}
		return homeMemo, nil
	}

	// Process aliases on the first run, have to do it before extends
	// because Golang randomizes maps, sometimes inherited containers
	// process earlier then dependencies; also do initial validation
	for name, container := range config.Containers {
		if container == nil {
			return nil, fmt.Errorf("Invalid specification for container `%s` in %s", name, configName)
		}
		// Handle aliases
		if container.Command != nil {
			if container.Cmd == nil {
				container.Cmd = container.Command
			}
			container.Command = nil
		}
		if container.Link != nil {
			if container.Links == nil {
				container.Links = container.Link
			}
			container.Link = nil
		}
		if container.Label != nil {
			if container.Labels == nil {
				container.Labels = container.Label
			}
			container.Label = nil
		}
		if container.Hosts != nil {
			if container.AddHost == nil {
				container.AddHost = container.Hosts
			}
			container.Hosts = nil
		}
		if container.ExtraHosts != nil {
			if container.AddHost == nil {
				container.AddHost = container.ExtraHosts
			}
			container.ExtraHosts = nil
		}
		if container.WorkingDir != nil {
			if container.Workdir == nil {
				container.Workdir = container.WorkingDir
			}
			container.WorkingDir = nil
		}
		if container.Environment != nil {
			if container.Env == nil {
				container.Env = container.Environment
			}
			container.Environment = nil
		}

		// Process extra data
		extraFields := map[string]interface{}{}
		for key, val := range extra.Containers[name] {
			if !yamlFields[key] {
				extraFields[key] = val
			}
		}
		if len(extraFields) > 0 {
			container.Extra = extraFields
		}

		// pretty.Println(name, container.Extra)
	}

	// Process extending containers configuration
	for name, container := range config.Containers {
		if container.Extends != "" {
			if container.Extends == name {
				return nil, fmt.Errorf("Container %s: cannot extend from itself", name)
			}
			if _, ok := config.Containers[container.Extends]; !ok {
				return nil, fmt.Errorf("Container %s: cannot find container %s to extend from", name, container.Extends)
			}
			// TODO: build dependency graph by extends hierarchy to allow multiple inheritance
			if config.Containers[container.Extends].Extends != "" {
				return nil, fmt.Errorf("Container %s: cannot extend from %s: multiple inheritance is not allowed yet",
					name, container.Extends)
			}
			container.ExtendFrom(config.Containers[container.Extends])
		}

		// Validate image
		if container.Image == nil {
			return nil, fmt.Errorf("Image should be specified for container: %s", name)
		}

		img := imagename.NewFromString(*container.Image)

		if !img.IsStrict() && !img.HasVersionRange() && !img.All() {
			return nil, fmt.Errorf("Image `%s` for container `%s`: image without tag is not allowed",
				*container.Image, name)
		}

		// Set namespace for all containers inside
		for k := range container.VolumesFrom {
			container.VolumesFrom[k].DefaultNamespace(config.Namespace)
		}
		for k := range container.Links {
			container.Links[k].DefaultNamespace(config.Namespace)
		}
		for k := range container.WaitFor {
			container.WaitFor[k].DefaultNamespace(config.Namespace)
		}
		if container.Net != nil && container.Net.Type == "container" {
			container.Net.Container.DefaultNamespace(config.Namespace)
		}

		// Fix exposed ports
		for k, port := range container.Expose {
			if !strings.Contains(port, "/") {
				container.Expose[k] = port + "/tcp"
			}
		}

		// Process relative paths in volumes
		for i, volume := range container.Volumes {
			split := strings.SplitN(volume, ":", 2)
			if len(split) == 1 {
				continue
			}
			if strings.HasPrefix(split[0], "~") {
				home, err := getHome()
				if err != nil {
					return nil, fmt.Errorf("Failed to get HOME path, error: %s", err)
				}
				split[0] = strings.Replace(split[0], "~", home, 1)
			}
			if !path.IsAbs(split[0]) {
				split[0] = path.Join(basedir, split[0])
			}
			container.Volumes[i] = strings.Join(split, ":")
		}
	}

	return config, nil
}