// NewContainerFromDocker converts a container object given by // docker client to a local Container object func NewContainerFromDocker(dockerContainer *docker.Container) (*Container, error) { cfg, err := config.NewFromDocker(dockerContainer) if err != nil { if _, ok := err.(config.ErrNotRockerCompose); !ok { return nil, err } } return &Container{ ID: dockerContainer.ID, Image: imagename.NewFromString(dockerContainer.Config.Image), ImageID: dockerContainer.Image, Name: config.NewContainerNameFromString(dockerContainer.Name), Created: dockerContainer.Created, State: &ContainerState{ Running: dockerContainer.State.Running, Paused: dockerContainer.State.Paused, Restarting: dockerContainer.State.Restarting, OOMKilled: dockerContainer.State.OOMKilled, Pid: dockerContainer.State.Pid, ExitCode: dockerContainer.State.ExitCode, Error: dockerContainer.State.Error, StartedAt: dockerContainer.State.StartedAt, FinishedAt: dockerContainer.State.FinishedAt, }, Config: cfg, container: dockerContainer, }, nil }
func TestClientResolveVersions(t *testing.T) { t.Skip() dockerCli, err := dockerclient.New() if err != nil { t.Fatal(err) } client, err := NewClient(&DockerClient{ Docker: dockerCli, }) if err != nil { t.Fatal(err) } containers := []*Container{ &Container{ Name: config.NewContainerName("test", "test"), Image: imagename.NewFromString("golang:1.4.*"), }, } if err := client.resolveVersions(true, true, template.Vars{}, containers); err != nil { t.Fatal(err) } pretty.Println(containers) }
// NewContainerFromConfig makes a single Container object from a spec Config object. func NewContainerFromConfig(name *config.ContainerName, containerConfig *config.Container) *Container { container := &Container{ Name: name, State: &ContainerState{ Running: containerConfig.State.Bool(), }, Config: containerConfig, } if containerConfig.Image != nil { container.Image = imagename.NewFromString(*containerConfig.Image) } return container }
// GetBridgeIP gets the ip address of docker network bridge // it is useful when you want to loose couple containers and not have tightly link them // container A may publish port 8125 to host network and container B may access this port through // a bridge ip address; it's a hacky solution, any better way to obtain bridge ip without ssh access // to host machine is welcome // // Here we create a dummy container and look at .NetworkSettings.Gateway value // // TODO: maybe we don't need this anymore since docker 1.8 seem to specify all existing containers // in a /etc/hosts file of every contianer. Need to research it further. // // https://github.com/docker/docker/issues/1143 // https://github.com/docker/docker/issues/11247 // func GetBridgeIP(client *docker.Client) (ip string, err error) { // Ensure empty image existing _, err = client.InspectImage(emptyImageName) if err != nil && err.Error() == "no such image" { log.Infof("Pulling image %s to obtain network bridge address", emptyImageName) if _, err := PullDockerImage(client, imagename.NewFromString(emptyImageName), &docker.AuthConfiguration{}); err != nil { return "", err } } else if err != nil { return "", fmt.Errorf("Failed to inspect image %s, error: %s", emptyImageName, err) } container, err := client.CreateContainer(docker.CreateContainerOptions{ Config: &docker.Config{ Image: emptyImageName, Cmd: []string{"/bin/sh", "-c", "while true; do sleep 1; done"}, }, HostConfig: &docker.HostConfig{}, }) if err != nil { return "", fmt.Errorf("Failed to create dummy network container, error: %s", err) } defer func() { removeOpts := docker.RemoveContainerOptions{ ID: container.ID, Force: true, RemoveVolumes: true, } if err2 := client.RemoveContainer(removeOpts); err2 != nil && err == nil { err = err2 } }() if err := client.StartContainer(container.ID, &docker.HostConfig{}); err != nil { return "", fmt.Errorf("Failed to start dummy network container %.12s, error: %s", container.ID, err) } inspect, err := client.InspectContainer(container.ID) if err != nil { return "", fmt.Errorf("Failed to inspect dummy network container %.12s, error: %s", container.ID, err) } return inspect.NetworkSettings.Gateway, nil }
// resolveVersions walks through the list of images and resolves their tags in case they are not strict func (client *DockerClient) resolveVersions(local, hub bool, vars template.Vars, containers []*Container) (err error) { // Provide function getter of all images to fetch only once var available []*imagename.ImageName getImages := func() ([]*imagename.ImageName, error) { if available == nil { available = []*imagename.ImageName{} if !local { return available, nil } // retrieving images currently available in docker var dockerImages []docker.APIImages if dockerImages, err = client.Docker.ListImages(docker.ListImagesOptions{}); err != nil { return nil, err } for _, image := range dockerImages { for _, repoTag := range image.RepoTags { available = append(available, imagename.NewFromString(repoTag)) } } } return available, nil } resolved := map[string]*imagename.ImageName{} // check images for each container for _, container := range containers { // error in configuration, fail fast if container.Image == nil { err = fmt.Errorf("Image is not specified for the container: %s", container.Name) return } // Version specified in variables var k string k = fmt.Sprintf("v_image_%s", container.Image.NameWithRegistry()) if tag, ok := vars[k]; ok { log.Infof("Resolve %s --> %s (derived by variable %s)", container.Image, tag, k) container.Image.SetTag(tag.(string)) } k = fmt.Sprintf("v_container_%s", container.Name.Name) if tag, ok := vars[k]; ok { log.Infof("Resolve %s --> %s (derived by variable %s)", container.Image, tag, k) container.Image.SetTag(tag.(string)) } // Do not resolve anything if the image is strict, e.g. "redis:2.8.11" or "redis:latest" if container.Image.IsStrict() { continue } // already resolved it for other container if _, ok := resolved[container.Image.String()]; ok { container.Image = resolved[container.Image.String()] continue } // Override to not change the common images slice var images []*imagename.ImageName if images, err = getImages(); err != nil { return err } // looking locally first candidate := container.Image.ResolveVersion(images) // in case we want to include external images as well, pulling list of available // images from repository or central docker hub if hub || candidate == nil { log.Debugf("Getting list of tags for %s from the registry", container.Image) var remote []*imagename.ImageName if remote, err = imagename.RegistryListTags(container.Image); err != nil { return fmt.Errorf("Failed to list tags of image %s for container %s from the remote registry, error: %s", container.Image, container.Name, err) } // Re-Resolve having hub tags candidate = container.Image.ResolveVersion(append(images, remote...)) } if candidate == nil { err = fmt.Errorf("Image not found: %s", container.Image) return } log.Infof("Resolve %s --> %s", container.Image, candidate.GetTag()) container.Image = candidate resolved[container.Image.String()] = candidate } return }
// Clean finds the obsolete image tags from container specs that exist in docker daemon, // skipping topN images that we want to keep (keep_images, default 5) and deletes them. func (client *DockerClient) Clean(config *config.Config) error { // do not pull same image twice images := map[imagename.ImageName]*imagename.Tags{} keep := client.KeepImages // keep 5 latest images by default if keep == 0 { keep = 5 } for _, container := range GetContainersFromConfig(config) { if container.Image == nil { continue } images[*container.Image] = &imagename.Tags{} } if len(images) == 0 { return nil } // Go through every image and list existing tags all, err := client.Docker.ListImages(docker.ListImagesOptions{}) if err != nil { return fmt.Errorf("Failed to list all images, error: %s", err) } // collect tags for every image for _, image := range all { for _, repoTag := range image.RepoTags { imageName := imagename.NewFromString(repoTag) for img := range images { if img.IsSameKind(*imageName) { images[img].Items = append(images[img].Items, &imagename.Tag{ ID: image.ID, Name: *imageName, Created: image.Created, }) } } } } // for every image, delete obsolete tags for name, tags := range images { toDelete := tags.GetOld(keep) if len(toDelete) == 0 { continue } log.Infof("Cleanup: removing %d tags of image %s", len(toDelete), name.NameWithRegistry()) for _, n := range toDelete { if name.GetTag() == n.GetTag() { log.Infof("Cleanup: skipping %s because it is in the spec", n) continue } wasRemoved := true log.Infof("Cleanup: remove %s", n) if err := client.Docker.RemoveImageExtended(n.String(), docker.RemoveImageOptions{Force: false}); err != nil { // 409 is conflict, which means there is a container exists running under this image if e, ok := err.(*docker.Error); ok && e.Status == 409 { log.Infof("Cleanup: skip %s because there is an existing container using it", n) wasRemoved = false } else { return err } } // cannot refer to &n because of for loop if wasRemoved { removed := n client.removedImages = append(client.removedImages, &removed) } } } return nil }
// ReadConfig reads and parses the config from io.Reader stream. // Before parsing it processes config through a template engine implemented in template.go. func ReadConfig(configName string, reader io.Reader, vars template.Vars, funcs map[string]interface{}, print bool) (*Config, error) { config := &Config{} basedir, err := os.Getwd() if err != nil { return nil, fmt.Errorf("Failed to get working dir, error: %s", err) } if configName == "-" { configName = "<STDIN>" } else { // if file given, process volume paths relative to the manifest file basedir = filepath.Dir(configName) } data, err := template.Process(configName, reader, vars, funcs) if err != nil { return nil, fmt.Errorf("Failed to process config template, error: %s", err) } if print { fmt.Print(data.String()) os.Exit(0) } if err := yaml.Unmarshal(data.Bytes(), config); err != nil { return nil, fmt.Errorf("Failed to parse YAML config, error: %s", err) } // empty namespace is a backward compatible docker-compose format // we will try to guess the namespace my parent directory name if config.Namespace == "" { parentDir := filepath.Base(basedir) config.Namespace = regexp.MustCompile("[^a-z0-9\\-\\_]").ReplaceAllString(parentDir, "") } // Save vars to config config.Vars = vars // Read extra data type ConfigExtra struct { Containers map[string]map[string]interface{} } extra := &ConfigExtra{} if err := yaml.Unmarshal(data.Bytes(), extra); err != nil { return nil, fmt.Errorf("Failed to parse YAML config extra properties, error: %s", err) } // Initialize YAML keys // Index yaml fields for better search yamlFields := make(map[string]bool) for _, v := range getYamlFields() { yamlFields[v] = true } // Function that gets HOME (initialize only once) homeMemo := "" getHome := func() (h string, err error) { if homeMemo == "" { if homeMemo, err = homedir.Dir(); err != nil { return "", err } } return homeMemo, nil } // Process aliases on the first run, have to do it before extends // because Golang randomizes maps, sometimes inherited containers // process earlier then dependencies; also do initial validation for name, container := range config.Containers { if container == nil { return nil, fmt.Errorf("Invalid specification for container `%s` in %s", name, configName) } // Handle aliases if container.Command != nil { if container.Cmd == nil { container.Cmd = container.Command } container.Command = nil } if container.Link != nil { if container.Links == nil { container.Links = container.Link } container.Link = nil } if container.Label != nil { if container.Labels == nil { container.Labels = container.Label } container.Label = nil } if container.Hosts != nil { if container.AddHost == nil { container.AddHost = container.Hosts } container.Hosts = nil } if container.ExtraHosts != nil { if container.AddHost == nil { container.AddHost = container.ExtraHosts } container.ExtraHosts = nil } if container.WorkingDir != nil { if container.Workdir == nil { container.Workdir = container.WorkingDir } container.WorkingDir = nil } if container.Environment != nil { if container.Env == nil { container.Env = container.Environment } container.Environment = nil } // Process extra data extraFields := map[string]interface{}{} for key, val := range extra.Containers[name] { if !yamlFields[key] { extraFields[key] = val } } if len(extraFields) > 0 { container.Extra = extraFields } // pretty.Println(name, container.Extra) } // Process extending containers configuration for name, container := range config.Containers { if container.Extends != "" { if container.Extends == name { return nil, fmt.Errorf("Container %s: cannot extend from itself", name) } if _, ok := config.Containers[container.Extends]; !ok { return nil, fmt.Errorf("Container %s: cannot find container %s to extend from", name, container.Extends) } // TODO: build dependency graph by extends hierarchy to allow multiple inheritance if config.Containers[container.Extends].Extends != "" { return nil, fmt.Errorf("Container %s: cannot extend from %s: multiple inheritance is not allowed yet", name, container.Extends) } container.ExtendFrom(config.Containers[container.Extends]) } // Validate image if container.Image == nil { return nil, fmt.Errorf("Image should be specified for container: %s", name) } img := imagename.NewFromString(*container.Image) if !img.IsStrict() && !img.HasVersionRange() && !img.All() { return nil, fmt.Errorf("Image `%s` for container `%s`: image without tag is not allowed", *container.Image, name) } // Set namespace for all containers inside for k := range container.VolumesFrom { container.VolumesFrom[k].DefaultNamespace(config.Namespace) } for k := range container.Links { container.Links[k].DefaultNamespace(config.Namespace) } for k := range container.WaitFor { container.WaitFor[k].DefaultNamespace(config.Namespace) } if container.Net != nil && container.Net.Type == "container" { container.Net.Container.DefaultNamespace(config.Namespace) } // Fix exposed ports for k, port := range container.Expose { if !strings.Contains(port, "/") { container.Expose[k] = port + "/tcp" } } // Process relative paths in volumes for i, volume := range container.Volumes { split := strings.SplitN(volume, ":", 2) if len(split) == 1 { continue } if strings.HasPrefix(split[0], "~") { home, err := getHome() if err != nil { return nil, fmt.Errorf("Failed to get HOME path, error: %s", err) } split[0] = strings.Replace(split[0], "~", home, 1) } if !path.IsAbs(split[0]) { split[0] = path.Join(basedir, split[0]) } container.Volumes[i] = strings.Join(split, ":") } } return config, nil }
"github.com/grammarly/rocker/src/rocker/imagename" "github.com/stretchr/testify/assert" ) var ( configTemplateVars = Vars{ "mykey": "myval", "n": "5", "data": map[string]string{ "foo": "bar", }, "RockerArtifacts": []imagename.Artifact{ imagename.Artifact{ Name: imagename.NewFromString("alpine:3.2"), Tag: "3.2", }, imagename.Artifact{ Name: imagename.NewFromString("golang:1.5"), Tag: "1.5", Digest: "sha256:ead434", }, imagename.Artifact{ Name: imagename.NewFromString("data:master"), Tag: "master", Digest: "sha256:fafe14", }, imagename.Artifact{ Name: imagename.NewFromString("ssh:latest"), Tag: "latest",
func makeImageHelper(vars Vars) func(string, ...string) (string, error) { // Sort artifacts so we match semver on latest item var ( artifacts = &imagename.Artifacts{} ok bool ) if artifacts.RockerArtifacts, ok = vars["RockerArtifacts"].([]imagename.Artifact); !ok { artifacts.RockerArtifacts = []imagename.Artifact{} } sort.Sort(artifacts) log.Debugf("`image` helper got artifacts: %# v", pretty.Formatter(artifacts)) return func(img string, args ...string) (string, error) { var ( matched bool ok bool shouldMatch bool image = imagename.NewFromString(img) ) if len(args) > 0 { image = imagename.New(img, args[0]) } for _, a := range artifacts.RockerArtifacts { if !image.IsSameKind(*a.Name) { continue } if image.HasVersionRange() { if !image.Contains(a.Name) { log.Debugf("Skipping artifact %s because it is not suitable for %s", a.Name, image) continue } } else if image.GetTag() != a.Name.GetTag() { log.Debugf("Skipping artifact %s because it is not suitable for %s", a.Name, image) continue } if a.Digest != "" { log.Infof("Apply artifact digest %s for image %s", a.Digest, image) image.SetTag(a.Digest) matched = true break } if a.Name.HasTag() { log.Infof("Apply artifact tag %s for image %s", a.Name.GetTag(), image) image.SetTag(a.Name.GetTag()) matched = true break } } if shouldMatch, ok = vars["DemandArtifacts"].(bool); ok && shouldMatch && !matched { return "", fmt.Errorf("Cannot find suitable artifact for image %s", image) } return image.String(), nil } }
func TestClientClean(t *testing.T) { // This test involves interaction with docker // enable it in case you want to test Clean() functionality t.Skip() dockerCli, err := dockerclient.New() if err != nil { t.Fatal(err) } // Create number of images to test createdContainers := []string{} createdImages := []string{} defer func() { for _, id := range createdContainers { if err := dockerCli.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true}); err != nil { t.Error(err) } } for _, id := range createdImages { if err := dockerCli.RemoveImageExtended(id, docker.RemoveImageOptions{Force: true}); err != nil { if err.Error() == "no such image" { continue } t.Error(err) } } }() for i := 1; i <= 5; i++ { c, err := dockerCli.CreateContainer(docker.CreateContainerOptions{ Config: &docker.Config{ Image: "gliderlabs/alpine:3.1", Cmd: []string{"true"}, }, }) if err != nil { t.Fatal(err) } createdContainers = append(createdContainers, c.ID) commitOpts := docker.CommitContainerOptions{ Container: c.ID, Repository: "rocker-compose-test-image-clean", Tag: fmt.Sprintf("%d", i), } img, err := dockerCli.CommitContainer(commitOpts) if err != nil { t.Fatal(err) } createdImages = append(createdImages, img.ID) // Make sure images have different timestamps time.Sleep(time.Second) } //////////////////////// cli, err := NewClient(&DockerClient{Docker: dockerCli, KeepImages: 2}) if err != nil { t.Fatal(err) } yml := ` namespace: test containers: main: image: rocker-compose-test-image-clean:5 ` config, err := config.ReadConfig("test.yml", strings.NewReader(yml), map[string]interface{}{}, map[string]interface{}{}, false) if err != nil { t.Fatal(err) } if err := cli.Clean(config); err != nil { t.Fatal(err) } // test that images left all, err := dockerCli.ListImages(docker.ListImagesOptions{}) if err != nil { t.Fatal(err) } n := 0 for _, image := range all { for _, repoTag := range image.RepoTags { imageName := imagename.NewFromString(repoTag) if imageName.Name == "rocker-compose-test-image-clean" { n++ } } } assert.Equal(t, 2, n, "Expected images to be cleaned up") // test removed images list removed := cli.GetRemovedImages() assert.Equal(t, 3, len(removed), "Expected to remove a particular number of images") assert.EqualValues(t, "rocker-compose-test-image-clean:3", removed[0].String(), "removed wrong image") assert.EqualValues(t, "rocker-compose-test-image-clean:2", removed[1].String(), "removed wrong image") assert.EqualValues(t, "rocker-compose-test-image-clean:1", removed[2].String(), "removed wrong image") }