// NewContainerFromDocker converts a container object given by
// docker client to a local Container object
func NewContainerFromDocker(dockerContainer *docker.Container) (*Container, error) {
cfg, err := config.NewFromDocker(dockerContainer)
if err != nil {
if _, ok := err.(config.ErrNotRockerCompose); !ok {
return nil, err
}
}
return &Container{
ID: dockerContainer.ID,
Image: imagename.NewFromString(dockerContainer.Config.Image),
ImageID: dockerContainer.Image,
Name: config.NewContainerNameFromString(dockerContainer.Name),
Created: dockerContainer.Created,
State: &ContainerState{
Running: dockerContainer.State.Running,
Paused: dockerContainer.State.Paused,
Restarting: dockerContainer.State.Restarting,
OOMKilled: dockerContainer.State.OOMKilled,
Pid: dockerContainer.State.Pid,
ExitCode: dockerContainer.State.ExitCode,
Error: dockerContainer.State.Error,
StartedAt: dockerContainer.State.StartedAt,
FinishedAt: dockerContainer.State.FinishedAt,
},
Config: cfg,
container: dockerContainer,
}, nil
}
func TestClientResolveVersions(t *testing.T) {
t.Skip()
dockerCli, err := dockerclient.New()
if err != nil {
t.Fatal(err)
}
client, err := NewClient(&DockerClient{
Docker: dockerCli,
})
if err != nil {
t.Fatal(err)
}
containers := []*Container{
&Container{
Name: config.NewContainerName("test", "test"),
Image: imagename.NewFromString("golang:1.4.*"),
},
}
if err := client.resolveVersions(true, true, template.Vars{}, containers); err != nil {
t.Fatal(err)
}
pretty.Println(containers)
}
// NewContainerFromConfig makes a single Container object from a spec Config object.
func NewContainerFromConfig(name *config.ContainerName, containerConfig *config.Container) *Container {
container := &Container{
Name: name,
State: &ContainerState{
Running: containerConfig.State.Bool(),
},
Config: containerConfig,
}
if containerConfig.Image != nil {
container.Image = imagename.NewFromString(*containerConfig.Image)
}
return container
}
// GetBridgeIP gets the ip address of docker network bridge
// it is useful when you want to loose couple containers and not have tightly link them
// container A may publish port 8125 to host network and container B may access this port through
// a bridge ip address; it's a hacky solution, any better way to obtain bridge ip without ssh access
// to host machine is welcome
//
// Here we create a dummy container and look at .NetworkSettings.Gateway value
//
// TODO: maybe we don't need this anymore since docker 1.8 seem to specify all existing containers
// in a /etc/hosts file of every contianer. Need to research it further.
//
// https://github.com/docker/docker/issues/1143
// https://github.com/docker/docker/issues/11247
//
func GetBridgeIP(client *docker.Client) (ip string, err error) {
// Ensure empty image existing
_, err = client.InspectImage(emptyImageName)
if err != nil && err.Error() == "no such image" {
log.Infof("Pulling image %s to obtain network bridge address", emptyImageName)
if _, err := PullDockerImage(client, imagename.NewFromString(emptyImageName), &docker.AuthConfiguration{}); err != nil {
return "", err
}
} else if err != nil {
return "", fmt.Errorf("Failed to inspect image %s, error: %s", emptyImageName, err)
}
container, err := client.CreateContainer(docker.CreateContainerOptions{
Config: &docker.Config{
Image: emptyImageName,
Cmd: []string{"/bin/sh", "-c", "while true; do sleep 1; done"},
},
HostConfig: &docker.HostConfig{},
})
if err != nil {
return "", fmt.Errorf("Failed to create dummy network container, error: %s", err)
}
defer func() {
removeOpts := docker.RemoveContainerOptions{
ID: container.ID,
Force: true,
RemoveVolumes: true,
}
if err2 := client.RemoveContainer(removeOpts); err2 != nil && err == nil {
err = err2
}
}()
if err := client.StartContainer(container.ID, &docker.HostConfig{}); err != nil {
return "", fmt.Errorf("Failed to start dummy network container %.12s, error: %s", container.ID, err)
}
inspect, err := client.InspectContainer(container.ID)
if err != nil {
return "", fmt.Errorf("Failed to inspect dummy network container %.12s, error: %s", container.ID, err)
}
return inspect.NetworkSettings.Gateway, nil
}
// resolveVersions walks through the list of images and resolves their tags in case they are not strict
func (client *DockerClient) resolveVersions(local, hub bool, vars template.Vars, containers []*Container) (err error) {
// Provide function getter of all images to fetch only once
var available []*imagename.ImageName
getImages := func() ([]*imagename.ImageName, error) {
if available == nil {
available = []*imagename.ImageName{}
if !local {
return available, nil
}
// retrieving images currently available in docker
var dockerImages []docker.APIImages
if dockerImages, err = client.Docker.ListImages(docker.ListImagesOptions{}); err != nil {
return nil, err
}
for _, image := range dockerImages {
for _, repoTag := range image.RepoTags {
available = append(available, imagename.NewFromString(repoTag))
}
}
}
return available, nil
}
resolved := map[string]*imagename.ImageName{}
// check images for each container
for _, container := range containers {
// error in configuration, fail fast
if container.Image == nil {
err = fmt.Errorf("Image is not specified for the container: %s", container.Name)
return
}
// Version specified in variables
var k string
k = fmt.Sprintf("v_image_%s", container.Image.NameWithRegistry())
if tag, ok := vars[k]; ok {
log.Infof("Resolve %s --> %s (derived by variable %s)", container.Image, tag, k)
container.Image.SetTag(tag.(string))
}
k = fmt.Sprintf("v_container_%s", container.Name.Name)
if tag, ok := vars[k]; ok {
log.Infof("Resolve %s --> %s (derived by variable %s)", container.Image, tag, k)
container.Image.SetTag(tag.(string))
}
// Do not resolve anything if the image is strict, e.g. "redis:2.8.11" or "redis:latest"
if container.Image.IsStrict() {
continue
}
// already resolved it for other container
if _, ok := resolved[container.Image.String()]; ok {
container.Image = resolved[container.Image.String()]
continue
}
// Override to not change the common images slice
var images []*imagename.ImageName
if images, err = getImages(); err != nil {
return err
}
// looking locally first
candidate := container.Image.ResolveVersion(images)
// in case we want to include external images as well, pulling list of available
// images from repository or central docker hub
if hub || candidate == nil {
log.Debugf("Getting list of tags for %s from the registry", container.Image)
var remote []*imagename.ImageName
if remote, err = imagename.RegistryListTags(container.Image); err != nil {
return fmt.Errorf("Failed to list tags of image %s for container %s from the remote registry, error: %s",
container.Image, container.Name, err)
}
// Re-Resolve having hub tags
candidate = container.Image.ResolveVersion(append(images, remote...))
}
if candidate == nil {
err = fmt.Errorf("Image not found: %s", container.Image)
return
}
log.Infof("Resolve %s --> %s", container.Image, candidate.GetTag())
container.Image = candidate
resolved[container.Image.String()] = candidate
}
return
}
// Clean finds the obsolete image tags from container specs that exist in docker daemon,
// skipping topN images that we want to keep (keep_images, default 5) and deletes them.
func (client *DockerClient) Clean(config *config.Config) error {
// do not pull same image twice
images := map[imagename.ImageName]*imagename.Tags{}
keep := client.KeepImages
// keep 5 latest images by default
if keep == 0 {
keep = 5
}
for _, container := range GetContainersFromConfig(config) {
if container.Image == nil {
continue
}
images[*container.Image] = &imagename.Tags{}
}
if len(images) == 0 {
return nil
}
// Go through every image and list existing tags
all, err := client.Docker.ListImages(docker.ListImagesOptions{})
if err != nil {
return fmt.Errorf("Failed to list all images, error: %s", err)
}
// collect tags for every image
for _, image := range all {
for _, repoTag := range image.RepoTags {
imageName := imagename.NewFromString(repoTag)
for img := range images {
if img.IsSameKind(*imageName) {
images[img].Items = append(images[img].Items, &imagename.Tag{
ID: image.ID,
Name: *imageName,
Created: image.Created,
})
}
}
}
}
// for every image, delete obsolete tags
for name, tags := range images {
toDelete := tags.GetOld(keep)
if len(toDelete) == 0 {
continue
}
log.Infof("Cleanup: removing %d tags of image %s", len(toDelete), name.NameWithRegistry())
for _, n := range toDelete {
if name.GetTag() == n.GetTag() {
log.Infof("Cleanup: skipping %s because it is in the spec", n)
continue
}
wasRemoved := true
log.Infof("Cleanup: remove %s", n)
if err := client.Docker.RemoveImageExtended(n.String(), docker.RemoveImageOptions{Force: false}); err != nil {
// 409 is conflict, which means there is a container exists running under this image
if e, ok := err.(*docker.Error); ok && e.Status == 409 {
log.Infof("Cleanup: skip %s because there is an existing container using it", n)
wasRemoved = false
} else {
return err
}
}
// cannot refer to &n because of for loop
if wasRemoved {
removed := n
client.removedImages = append(client.removedImages, &removed)
}
}
}
return nil
}
// ReadConfig reads and parses the config from io.Reader stream.
// Before parsing it processes config through a template engine implemented in template.go.
func ReadConfig(configName string, reader io.Reader, vars template.Vars, funcs map[string]interface{}, print bool) (*Config, error) {
config := &Config{}
basedir, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("Failed to get working dir, error: %s", err)
}
if configName == "-" {
configName = "<STDIN>"
} else {
// if file given, process volume paths relative to the manifest file
basedir = filepath.Dir(configName)
}
data, err := template.Process(configName, reader, vars, funcs)
if err != nil {
return nil, fmt.Errorf("Failed to process config template, error: %s", err)
}
if print {
fmt.Print(data.String())
os.Exit(0)
}
if err := yaml.Unmarshal(data.Bytes(), config); err != nil {
return nil, fmt.Errorf("Failed to parse YAML config, error: %s", err)
}
// empty namespace is a backward compatible docker-compose format
// we will try to guess the namespace my parent directory name
if config.Namespace == "" {
parentDir := filepath.Base(basedir)
config.Namespace = regexp.MustCompile("[^a-z0-9\\-\\_]").ReplaceAllString(parentDir, "")
}
// Save vars to config
config.Vars = vars
// Read extra data
type ConfigExtra struct {
Containers map[string]map[string]interface{}
}
extra := &ConfigExtra{}
if err := yaml.Unmarshal(data.Bytes(), extra); err != nil {
return nil, fmt.Errorf("Failed to parse YAML config extra properties, error: %s", err)
}
// Initialize YAML keys
// Index yaml fields for better search
yamlFields := make(map[string]bool)
for _, v := range getYamlFields() {
yamlFields[v] = true
}
// Function that gets HOME (initialize only once)
homeMemo := ""
getHome := func() (h string, err error) {
if homeMemo == "" {
if homeMemo, err = homedir.Dir(); err != nil {
return "", err
}
}
return homeMemo, nil
}
// Process aliases on the first run, have to do it before extends
// because Golang randomizes maps, sometimes inherited containers
// process earlier then dependencies; also do initial validation
for name, container := range config.Containers {
if container == nil {
return nil, fmt.Errorf("Invalid specification for container `%s` in %s", name, configName)
}
// Handle aliases
if container.Command != nil {
if container.Cmd == nil {
container.Cmd = container.Command
}
container.Command = nil
}
if container.Link != nil {
if container.Links == nil {
container.Links = container.Link
}
container.Link = nil
}
if container.Label != nil {
if container.Labels == nil {
container.Labels = container.Label
}
container.Label = nil
}
if container.Hosts != nil {
if container.AddHost == nil {
container.AddHost = container.Hosts
}
container.Hosts = nil
}
if container.ExtraHosts != nil {
if container.AddHost == nil {
container.AddHost = container.ExtraHosts
}
container.ExtraHosts = nil
}
if container.WorkingDir != nil {
if container.Workdir == nil {
container.Workdir = container.WorkingDir
}
container.WorkingDir = nil
}
if container.Environment != nil {
if container.Env == nil {
container.Env = container.Environment
}
container.Environment = nil
}
// Process extra data
extraFields := map[string]interface{}{}
for key, val := range extra.Containers[name] {
if !yamlFields[key] {
extraFields[key] = val
}
}
if len(extraFields) > 0 {
container.Extra = extraFields
}
// pretty.Println(name, container.Extra)
}
// Process extending containers configuration
for name, container := range config.Containers {
if container.Extends != "" {
if container.Extends == name {
return nil, fmt.Errorf("Container %s: cannot extend from itself", name)
}
if _, ok := config.Containers[container.Extends]; !ok {
return nil, fmt.Errorf("Container %s: cannot find container %s to extend from", name, container.Extends)
}
// TODO: build dependency graph by extends hierarchy to allow multiple inheritance
if config.Containers[container.Extends].Extends != "" {
return nil, fmt.Errorf("Container %s: cannot extend from %s: multiple inheritance is not allowed yet",
name, container.Extends)
}
container.ExtendFrom(config.Containers[container.Extends])
}
// Validate image
if container.Image == nil {
return nil, fmt.Errorf("Image should be specified for container: %s", name)
}
img := imagename.NewFromString(*container.Image)
if !img.IsStrict() && !img.HasVersionRange() && !img.All() {
return nil, fmt.Errorf("Image `%s` for container `%s`: image without tag is not allowed",
*container.Image, name)
}
// Set namespace for all containers inside
for k := range container.VolumesFrom {
container.VolumesFrom[k].DefaultNamespace(config.Namespace)
}
for k := range container.Links {
container.Links[k].DefaultNamespace(config.Namespace)
}
for k := range container.WaitFor {
container.WaitFor[k].DefaultNamespace(config.Namespace)
}
if container.Net != nil && container.Net.Type == "container" {
container.Net.Container.DefaultNamespace(config.Namespace)
}
// Fix exposed ports
for k, port := range container.Expose {
if !strings.Contains(port, "/") {
container.Expose[k] = port + "/tcp"
}
}
// Process relative paths in volumes
for i, volume := range container.Volumes {
split := strings.SplitN(volume, ":", 2)
if len(split) == 1 {
continue
}
if strings.HasPrefix(split[0], "~") {
home, err := getHome()
if err != nil {
return nil, fmt.Errorf("Failed to get HOME path, error: %s", err)
}
split[0] = strings.Replace(split[0], "~", home, 1)
}
if !path.IsAbs(split[0]) {
split[0] = path.Join(basedir, split[0])
}
container.Volumes[i] = strings.Join(split, ":")
}
}
return config, nil
}
"github.com/grammarly/rocker/src/rocker/imagename"
"github.com/stretchr/testify/assert"
)
var (
configTemplateVars = Vars{
"mykey": "myval",
"n": "5",
"data": map[string]string{
"foo": "bar",
},
"RockerArtifacts": []imagename.Artifact{
imagename.Artifact{
Name: imagename.NewFromString("alpine:3.2"),
Tag: "3.2",
},
imagename.Artifact{
Name: imagename.NewFromString("golang:1.5"),
Tag: "1.5",
Digest: "sha256:ead434",
},
imagename.Artifact{
Name: imagename.NewFromString("data:master"),
Tag: "master",
Digest: "sha256:fafe14",
},
imagename.Artifact{
Name: imagename.NewFromString("ssh:latest"),
Tag: "latest",
func makeImageHelper(vars Vars) func(string, ...string) (string, error) {
// Sort artifacts so we match semver on latest item
var (
artifacts = &imagename.Artifacts{}
ok bool
)
if artifacts.RockerArtifacts, ok = vars["RockerArtifacts"].([]imagename.Artifact); !ok {
artifacts.RockerArtifacts = []imagename.Artifact{}
}
sort.Sort(artifacts)
log.Debugf("`image` helper got artifacts: %# v", pretty.Formatter(artifacts))
return func(img string, args ...string) (string, error) {
var (
matched bool
ok bool
shouldMatch bool
image = imagename.NewFromString(img)
)
if len(args) > 0 {
image = imagename.New(img, args[0])
}
for _, a := range artifacts.RockerArtifacts {
if !image.IsSameKind(*a.Name) {
continue
}
if image.HasVersionRange() {
if !image.Contains(a.Name) {
log.Debugf("Skipping artifact %s because it is not suitable for %s", a.Name, image)
continue
}
} else if image.GetTag() != a.Name.GetTag() {
log.Debugf("Skipping artifact %s because it is not suitable for %s", a.Name, image)
continue
}
if a.Digest != "" {
log.Infof("Apply artifact digest %s for image %s", a.Digest, image)
image.SetTag(a.Digest)
matched = true
break
}
if a.Name.HasTag() {
log.Infof("Apply artifact tag %s for image %s", a.Name.GetTag(), image)
image.SetTag(a.Name.GetTag())
matched = true
break
}
}
if shouldMatch, ok = vars["DemandArtifacts"].(bool); ok && shouldMatch && !matched {
return "", fmt.Errorf("Cannot find suitable artifact for image %s", image)
}
return image.String(), nil
}
}
func TestClientClean(t *testing.T) {
// This test involves interaction with docker
// enable it in case you want to test Clean() functionality
t.Skip()
dockerCli, err := dockerclient.New()
if err != nil {
t.Fatal(err)
}
// Create number of images to test
createdContainers := []string{}
createdImages := []string{}
defer func() {
for _, id := range createdContainers {
if err := dockerCli.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true}); err != nil {
t.Error(err)
}
}
for _, id := range createdImages {
if err := dockerCli.RemoveImageExtended(id, docker.RemoveImageOptions{Force: true}); err != nil {
if err.Error() == "no such image" {
continue
}
t.Error(err)
}
}
}()
for i := 1; i <= 5; i++ {
c, err := dockerCli.CreateContainer(docker.CreateContainerOptions{
Config: &docker.Config{
Image: "gliderlabs/alpine:3.1",
Cmd: []string{"true"},
},
})
if err != nil {
t.Fatal(err)
}
createdContainers = append(createdContainers, c.ID)
commitOpts := docker.CommitContainerOptions{
Container: c.ID,
Repository: "rocker-compose-test-image-clean",
Tag: fmt.Sprintf("%d", i),
}
img, err := dockerCli.CommitContainer(commitOpts)
if err != nil {
t.Fatal(err)
}
createdImages = append(createdImages, img.ID)
// Make sure images have different timestamps
time.Sleep(time.Second)
}
////////////////////////
cli, err := NewClient(&DockerClient{Docker: dockerCli, KeepImages: 2})
if err != nil {
t.Fatal(err)
}
yml := `
namespace: test
containers:
main:
image: rocker-compose-test-image-clean:5
`
config, err := config.ReadConfig("test.yml", strings.NewReader(yml), map[string]interface{}{}, map[string]interface{}{}, false)
if err != nil {
t.Fatal(err)
}
if err := cli.Clean(config); err != nil {
t.Fatal(err)
}
// test that images left
all, err := dockerCli.ListImages(docker.ListImagesOptions{})
if err != nil {
t.Fatal(err)
}
n := 0
for _, image := range all {
for _, repoTag := range image.RepoTags {
imageName := imagename.NewFromString(repoTag)
if imageName.Name == "rocker-compose-test-image-clean" {
n++
}
}
}
assert.Equal(t, 2, n, "Expected images to be cleaned up")
// test removed images list
removed := cli.GetRemovedImages()
assert.Equal(t, 3, len(removed), "Expected to remove a particular number of images")
assert.EqualValues(t, "rocker-compose-test-image-clean:3", removed[0].String(), "removed wrong image")
assert.EqualValues(t, "rocker-compose-test-image-clean:2", removed[1].String(), "removed wrong image")
assert.EqualValues(t, "rocker-compose-test-image-clean:1", removed[2].String(), "removed wrong image")
}