func Boot(client *docker.Client, opt *docker.CreateContainerOptions,
exitCh chan error) (*docker.Container, error) {
log.Debugf("Creating container for image %s", opt.Config.Image)
container, err := client.CreateContainer(*opt)
if err != nil {
return container, err
}
log.Debugf("Starting container %s", container.ID)
go func() {
exitCh <- dockerpty.Start(client, container, opt.HostConfig)
}()
trial := 0
for {
container, err = client.InspectContainer(container.ID)
if container.State.StartedAt.Unix() > 0 {
break
}
if trial > 30 {
return container, fmt.Errorf("container %s seems not started. state=%#v", container.ID, container.State)
}
trial += 1
time.Sleep(time.Duration(trial*100) * time.Millisecond)
}
log.Debugf("container state=%#v", container.State)
return container, nil
}
func launchContainer(client *docker.Client, containerImageName, name string) (*docker.Container, error) {
images, err := client.ListImages(docker.ListImagesOptions{Filter: containerImageName})
if err != nil {
return nil, err
}
targetImageId := images[0].ID
container, err := client.CreateContainer(docker.CreateContainerOptions{
Name: name,
Config: &docker.Config{
Image: targetImageId,
},
})
if err != nil {
return nil, err
}
if err := client.StartContainer(container.ID, &docker.HostConfig{PublishAllPorts: true}); err != nil {
return nil, err
}
return client.InspectContainer(container.ID)
}
// imageFSMetadata creates a container and reads the filesystem metadata out of the archive.
func imageFSMetadata(c *docker.Client, name string) (map[string]*tar.Header, error) {
container, err := c.CreateContainer(docker.CreateContainerOptions{Name: name + "-export", Config: &docker.Config{Image: name}})
if err != nil {
return nil, err
}
defer c.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID, RemoveVolumes: true, Force: true})
ch := make(chan struct{})
result := make(map[string]*tar.Header)
r, w := io.Pipe()
go func() {
defer close(ch)
out := tar.NewReader(r)
for {
h, err := out.Next()
if err != nil {
if err == io.EOF {
w.Close()
} else {
w.CloseWithError(err)
}
break
}
result[h.Name] = h
}
}()
if err := c.ExportContainer(docker.ExportContainerOptions{ID: container.ID, OutputStream: w}); err != nil {
return nil, err
}
<-ch
return result, nil
}
// createGremlin creates the gremlin container using the docker client.
// It is used only in the test code.
func createGremlin(client *docker.Client) (*docker.Container, error) {
container, err := client.CreateContainer(docker.CreateContainerOptions{
Config: &docker.Config{
Image: testImageName,
},
})
return container, err
}
func runContainer(client *dockerapi.Client, createOpts dockerapi.CreateContainerOptions, startConfig *dockerapi.HostConfig) (string, error) {
container, err := client.CreateContainer(createOpts)
if err != nil {
return "", err
}
err = client.StartContainer(container.ID, startConfig)
// return container ID even if there is an error, so caller can clean up container if desired
return container.ID, err
}
func createContainer(client *docker.Client, opts docker.CreateContainerOptions) (c *container, err error) {
var dockerInfo *docker.Container
dockerInfo, err = client.CreateContainer(opts)
if err == nil {
c = &container{
client: client,
id: dockerInfo.ID,
}
}
return
}
func createContainers(client *docker.Client, num int, labels map[string]string) {
for i := 0; i < num; i++ {
name := "test_container_" + strconv.FormatInt(time.Now().UnixNano(), 10)
dockerOpts := docker.CreateContainerOptions{
Name: name,
Config: &docker.Config{
Image: "ubuntu",
Labels: labels,
},
}
client.CreateContainer(dockerOpts)
}
}
// Run runs cmd in the given image using the docker client cl. It mounts cwd into containerMount in the running container and sends on the following channels:
//
// - rmContainerCh: a function closure that the receiver should call, after they receive on errCh or exitCodeCh, to remove the container. this is commonly done with a 'defer'
// - stdOut: all logs from STDOUT in the container. this may never receive
// - stdErr: all logs from STDERR in the container. this may never receive
// - exitCodeCh: the exit code of the container
// - errCh: any error in setting up or running the container. if errCh receives, exitCodeCh may not receive
func Run(
cl *docker.Client,
image *Image,
taskName,
cwd,
containerMount,
cmd string,
env []string,
rmContainerCh chan<- func(),
stdOut chan<- Log,
stdErr chan<- Log,
exitCodeCh chan<- int,
errCh chan<- error,
) {
mounts := []docker.Mount{
{Name: "pwd", Source: cwd, Destination: containerMount, Mode: "rxw"},
}
cmdSpl := strings.Split(cmd, " ")
containerName := NewContainerName(taskName, cwd)
createContainerOpts, hostConfig := CreateAndStartContainerOpts(image.String(), containerName, cmdSpl, env, mounts, containerMount)
if err := EnsureImage(cl, image.String(), func() (io.Writer, error) {
return os.Stdout, nil
}); err != nil {
errCh <- err
return
}
container, err := cl.CreateContainer(createContainerOpts)
if err != nil {
errCh <- err
}
rmContainerCh <- func() {
if err := cl.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID, Force: true}); err != nil {
log.Warn("Error removing container %s (%s)", container.ID, err)
}
}
log.Debug(CmdStr(createContainerOpts, hostConfig))
attachOpts := AttachToContainerOpts(container.ID, NewChanWriter(stdOut), NewChanWriter(stdErr))
// attach before the container starts, so we get all the logs etc...
go AttachAndWait(cl, container.ID, attachOpts, exitCodeCh, errCh)
if startErr := cl.StartContainer(container.ID, &hostConfig); startErr != nil {
errCh <- err
return
}
}
func CreateContainer(client *docker.Client, config *Config, image string, standby int, env string) (*docker.Container, error) {
id, _ := randomHex(20)
volumePath := fmt.Sprintf("%s/%s", config.SharedPath, id)
name := fmt.Sprintf("bitrun-%v", time.Now().UnixNano())
if err := os.Mkdir(volumePath, 0777); err != nil {
return nil, err
}
opts := docker.CreateContainerOptions{
Name: name,
HostConfig: &docker.HostConfig{
Binds: []string{
volumePath + ":/code",
volumePath + ":/tmp",
},
ReadonlyRootfs: true,
Memory: config.MemoryLimit,
MemorySwap: 0,
},
Config: &docker.Config{
Hostname: "bitrun",
Image: image,
Labels: map[string]string{"id": id},
AttachStdout: false,
AttachStderr: false,
AttachStdin: false,
Tty: false,
NetworkDisabled: config.NetworkDisabled,
WorkingDir: "/code",
Cmd: []string{"sleep", fmt.Sprintf("%v", standby)},
Env: strings.Split(env, "\n"),
},
}
container, err := client.CreateContainer(opts)
if err == nil {
container.Config = opts.Config
}
return container, err
}
func CreateAndRemoveContainers(client *docker.Client) string {
name := newContainerName()
dockerOpts := docker.CreateContainerOptions{
Name: name,
Config: &docker.Config{
Image: "ubuntu",
},
}
container, err := client.CreateContainer(dockerOpts)
if err != nil {
panic(fmt.Sprintf("Error create containers: %v", err))
}
removeOpts := docker.RemoveContainerOptions{
ID: container.ID,
}
if err := client.RemoveContainer(removeOpts); err != nil {
panic(fmt.Sprintf("Error remove containers: %v", err))
}
return container.ID
}
func CreateContainers(client *docker.Client, num int) []string {
ids := []string{}
for i := 0; i < num; i++ {
name := newContainerName()
dockerOpts := docker.CreateContainerOptions{
Name: name,
Config: &docker.Config{
AttachStderr: false,
AttachStdin: false,
AttachStdout: false,
Tty: true,
Cmd: []string{"/bin/bash"},
Image: "ubuntu",
},
}
container, err := client.CreateContainer(dockerOpts)
if err != nil {
panic(fmt.Sprintf("Error create containers: %v", err))
}
ids = append(ids, container.ID)
}
return ids
}
// createAndStartRouterContainer is responsible for deploying the router image in docker. It assumes that all router images
// will use a command line flag that can take --master which points to the master url
func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp string) (containerId string, err error) {
ports := []string{"80", "443"}
portBindings := make(map[dockerClient.Port][]dockerClient.PortBinding)
exposedPorts := map[dockerClient.Port]struct{}{}
for _, p := range ports {
dockerPort := dockerClient.Port(p + "/tcp")
portBindings[dockerPort] = []dockerClient.PortBinding{
{
HostPort: p,
},
}
exposedPorts[dockerPort] = struct{}{}
}
containerOpts := dockerClient.CreateContainerOptions{
Config: &dockerClient.Config{
Image: getRouterImage(),
Cmd: []string{"--master=" + masterIp, "--loglevel=4"},
ExposedPorts: exposedPorts,
},
}
container, err := dockerCli.CreateContainer(containerOpts)
if err != nil {
return "", err
}
dockerHostCfg := &dockerClient.HostConfig{NetworkMode: "host", PortBindings: portBindings}
err = dockerCli.StartContainer(container.ID, dockerHostCfg)
if err != nil {
return "", err
}
running := false
//wait for it to start
for i := 0; i < dockerRetries; i++ {
c, err := dockerCli.InspectContainer(container.ID)
if err != nil {
return "", err
}
if c.State.Running {
running = true
break
}
time.Sleep(time.Second * dockerWaitSeconds)
}
if !running {
return "", errors.New("Container did not start after 3 tries!")
}
return container.ID, nil
}
// StartDockerContainer starts a new Lever container for the specified
// environment and service.
func StartDockerContainer(
docker *dockerapi.Client, environment string, service string,
instanceID string, codeVersion int64, isAdmin bool,
leverConfig *core.LeverConfig) (
containerID string, node string, err error) {
codeDir := HostCodeDirPath(environment, service, codeVersion)
binds := []string{codeDir + ":/leveros/custcode:ro,Z"}
env := []string{
"LEVEROS_ENVIRONMENT=" + environment,
"LEVEROS_SERVICE=" + service,
"LEVEROS_INSTANCE_ID=" + instanceID,
"LEVEROS_CODE_VERSION=" + strconv.Itoa(int(codeVersion)),
"LEVEROS_INTERNAL_ENV_SUFFIX=" +
core.InternalEnvironmentSuffixFlag.Get(),
}
if isAdmin {
// This is used by admin to make deployments.
binds = append(
binds, LeverCodeHostDirFlag.Get()+":/leveros/custcodetree:Z")
}
// Configure logging.
var logConfig dockerapi.LogConfig
if devlogger.DisableFlag.Get() {
logConfig.Type = "none"
} else {
// TODO: Should use scale.Dereference... to get IP of syslog server
// and shard by env+service.
tag := fmt.Sprintf(
"%s/%s/%d/%s", environment, service, codeVersion, instanceID)
logConfig.Type = "syslog"
logConfig.Config = map[string]string{
"syslog-address": "tcp://127.0.0.1:6514",
"syslog-facility": "user",
"tag": tag,
"syslog-format": "rfc5424",
}
}
memoryBytes := int64(leverConfig.InstanceMemoryMB) * 1000 * 1000
memAndSwapBytes := memoryBytes // No swap.
kernelMemBytes := memoryBytes / 10 // Allow 10% memory for kernel.
// Entry point.
entry := leverConfig.EntryPoint
if leverConfig.JSEntryPoint != "" {
// Trigger GC in node when garbage reaches 90% of memory.
maxGarbage := strconv.Itoa(
int(float32(leverConfig.InstanceMemoryMB) * 0.9))
// Set entry point for node.
entry = []string{
"node", "--optimize_for_size", "--max_old_space_size=" + maxGarbage,
"--gc_interval=100",
"/leveros/js/leveros-server/compiled/lib/serve.js",
leverConfig.JSEntryPoint,
}
}
container, err := docker.CreateContainer(dockerapi.CreateContainerOptions{
Name: "leveros_" + instanceID,
Config: &dockerapi.Config{
Image: "leveros/levercontainer:latest",
Cmd: entry,
Env: env,
KernelMemory: kernelMemBytes,
Labels: map[string]string{
"com.leveros.environment": environment,
"com.leveros.service": service,
"com.leveros.instanceid": instanceID,
"com.leveros.codeversion": strconv.Itoa(int(codeVersion)),
},
},
// TODO: Documentation for these here:
// https://docs.docker.com/engine/reference/api/docker_remote_api_v1.23/#create-a-container
// TODO: Also check if need to set limits on IO operations (blkio).
// TODO: Should allow to write to disk, but limit amount of disk space.
HostConfig: &dockerapi.HostConfig{
ReadonlyRootfs: true,
Binds: binds,
CapDrop: []string{"all"},
NetworkMode: "none",
Ulimits: []dockerapi.ULimit{}, // TODO
SecurityOpt: []string{"no-new-privileges"}, // TODO
LogConfig: logConfig,
Memory: memoryBytes,
MemorySwap: memAndSwapBytes,
MemorySwappiness: 0,
CPUShares: 0, // TODO
CPUPeriod: 0, // TODO
CPUQuota: 0, // TODO
},
})
if err != nil {
logger.WithFields("err", err).Debug(
"Error trying to create container")
return "", "", err
}
// Get info about the node it was allocated to by Docker Swarm.
container, err = docker.InspectContainer(container.ID)
if err != nil {
removeErr := RemoveDockerContainer(docker, container.ID)
if removeErr != nil {
logger.WithFields(
"containerID", containerID,
"err", removeErr,
).Error("Error trying to remove container after previous error")
}
return "", "", err
}
if container.Node != nil {
node = container.Node.Name
} else {
// In a dev/testing (non-swarm) environment.
logger.Warning(
"Using non-swarm node. " +
"YOU SHOULD NEVER SEE THIS IN PRODUCTION.")
node = "leverosconsul"
}
// Start the container.
err = docker.StartContainer(container.ID, nil)
if err != nil {
removeErr := RemoveDockerContainer(docker, container.ID)
if removeErr != nil {
logger.WithFields(
"containerID", containerID,
"err", removeErr,
).Error("Error trying to remove container after failed to start")
}
return "", "", err
}
// Need to disconnect it from the "none" network before being able to
// connect it to its local environment network.
err = DisconnectFromDockerEnvNetwork(docker, container.ID, "none")
if err != nil {
removeErr := RemoveDockerContainer(docker, container.ID)
if removeErr != nil {
logger.WithFields(
"containerID", containerID,
"err", removeErr,
).Error("Error trying to remove container after previous error")
}
return "", "", err
}
return container.ID, node, nil
}
// createAndStartRouterContainer is responsible for deploying the router image in docker. It assumes that all router images
// will use a command line flag that can take --master which points to the master url
func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp string, routerStatsPort int, reloadInterval int) (containerId string, err error) {
ports := []string{"80", "443"}
if routerStatsPort > 0 {
ports = append(ports, fmt.Sprintf("%d", routerStatsPort))
}
portBindings := make(map[dockerClient.Port][]dockerClient.PortBinding)
exposedPorts := map[dockerClient.Port]struct{}{}
for _, p := range ports {
dockerPort := dockerClient.Port(p + "/tcp")
portBindings[dockerPort] = []dockerClient.PortBinding{
{
HostPort: p,
},
}
exposedPorts[dockerPort] = struct{}{}
}
copyEnv := []string{
"ROUTER_EXTERNAL_HOST_HOSTNAME",
"ROUTER_EXTERNAL_HOST_USERNAME",
"ROUTER_EXTERNAL_HOST_PASSWORD",
"ROUTER_EXTERNAL_HOST_HTTP_VSERVER",
"ROUTER_EXTERNAL_HOST_HTTPS_VSERVER",
"ROUTER_EXTERNAL_HOST_INSECURE",
"ROUTER_EXTERNAL_HOST_PRIVKEY",
}
env := []string{
fmt.Sprintf("STATS_PORT=%d", routerStatsPort),
fmt.Sprintf("STATS_USERNAME=%s", statsUser),
fmt.Sprintf("STATS_PASSWORD=%s", statsPassword),
fmt.Sprintf("DEFAULT_CERTIFICATE=%s", defaultCert),
}
reloadIntVar := fmt.Sprintf("RELOAD_INTERVAL=%ds", reloadInterval)
env = append(env, reloadIntVar)
for _, name := range copyEnv {
val := os.Getenv(name)
if len(val) > 0 {
env = append(env, name+"="+val)
}
}
vols := ""
hostVols := []string{}
privkeyFilename := os.Getenv("ROUTER_EXTERNAL_HOST_PRIVKEY")
if len(privkeyFilename) != 0 {
vols = privkeyFilename
privkeyBindmount := fmt.Sprintf("%[1]s:%[1]s", privkeyFilename)
hostVols = append(hostVols, privkeyBindmount)
}
binary := os.Getenv("ROUTER_OPENSHIFT_BINARY")
if len(binary) != 0 {
hostVols = append(hostVols, fmt.Sprintf("%[1]s:/usr/bin/openshift", binary))
}
containerOpts := dockerClient.CreateContainerOptions{
Config: &dockerClient.Config{
Image: getRouterImage(),
Cmd: []string{"--master=" + masterIp, "--loglevel=4"},
Env: env,
ExposedPorts: exposedPorts,
VolumesFrom: vols,
},
HostConfig: &dockerClient.HostConfig{
Binds: hostVols,
},
}
container, err := dockerCli.CreateContainer(containerOpts)
if err != nil {
return "", err
}
dockerHostCfg := &dockerClient.HostConfig{NetworkMode: "host", PortBindings: portBindings}
err = dockerCli.StartContainer(container.ID, dockerHostCfg)
if err != nil {
return "", err
}
//wait for it to start
if err := wait.Poll(time.Millisecond*100, time.Second*30, func() (bool, error) {
c, err := dockerCli.InspectContainer(container.ID)
if err != nil {
return false, err
}
return c.State.Running, nil
}); err != nil {
return "", err
}
return container.ID, nil
}
// createAndStartRouterContainer is responsible for deploying the router image in docker. It assumes that all router images
// will use a command line flag that can take --master which points to the master url
func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp string, routerStatsPort int) (containerId string, err error) {
ports := []string{"80", "443"}
if routerStatsPort > 0 {
ports = append(ports, fmt.Sprintf("%d", routerStatsPort))
}
portBindings := make(map[dockerClient.Port][]dockerClient.PortBinding)
exposedPorts := map[dockerClient.Port]struct{}{}
for _, p := range ports {
dockerPort := dockerClient.Port(p + "/tcp")
portBindings[dockerPort] = []dockerClient.PortBinding{
{
HostPort: p,
},
}
exposedPorts[dockerPort] = struct{}{}
}
copyEnv := []string{
"ROUTER_EXTERNAL_HOST_HOSTNAME",
"ROUTER_EXTERNAL_HOST_USERNAME",
"ROUTER_EXTERNAL_HOST_PASSWORD",
"ROUTER_EXTERNAL_HOST_HTTP_VSERVER",
"ROUTER_EXTERNAL_HOST_HTTPS_VSERVER",
"ROUTER_EXTERNAL_HOST_INSECURE",
"ROUTER_EXTERNAL_HOST_PRIVKEY",
}
env := []string{
fmt.Sprintf("STATS_PORT=%d", routerStatsPort),
fmt.Sprintf("STATS_USERNAME=%s", statsUser),
fmt.Sprintf("STATS_PASSWORD=%s", statsPassword),
}
for _, name := range copyEnv {
val := os.Getenv(name)
if len(val) > 0 {
env = append(env, name+"="+val)
}
}
vols := ""
hostVols := []string{}
privkeyFilename := os.Getenv("ROUTER_EXTERNAL_HOST_PRIVKEY")
if len(privkeyFilename) != 0 {
vols = privkeyFilename
privkeyBindmount := fmt.Sprintf("%[1]s:%[1]s", privkeyFilename)
hostVols = append(hostVols, privkeyBindmount)
}
binary := os.Getenv("ROUTER_OPENSHIFT_BINARY")
if len(binary) != 0 {
hostVols = append(hostVols, fmt.Sprintf("%[1]s:/usr/bin/openshift", binary))
}
containerOpts := dockerClient.CreateContainerOptions{
Config: &dockerClient.Config{
Image: getRouterImage(),
Cmd: []string{"--master=" + masterIp, "--loglevel=4"},
Env: env,
ExposedPorts: exposedPorts,
VolumesFrom: vols,
},
HostConfig: &dockerClient.HostConfig{
Binds: hostVols,
},
}
container, err := dockerCli.CreateContainer(containerOpts)
if err != nil {
return "", err
}
dockerHostCfg := &dockerClient.HostConfig{NetworkMode: "host", PortBindings: portBindings}
err = dockerCli.StartContainer(container.ID, dockerHostCfg)
if err != nil {
return "", err
}
running := false
//wait for it to start
for i := 0; i < dockerRetries; i++ {
time.Sleep(time.Second * dockerWaitSeconds)
c, err := dockerCli.InspectContainer(container.ID)
if err != nil {
return "", err
}
if c.State.Running {
running = true
break
}
}
if !running {
return "", errors.New("Container did not start after 3 tries!")
}
return container.ID, nil
}
// createAndExtractImage creates a docker container based on the option's image with containerName.
// It will then insepct the container and image and then attempt to extract the image to
// option's destination path. If the destination path is empty it will write to a temp directory
// and update the option's destination path with a /var/tmp directory. /var/tmp is used to
// try and ensure it is a non-in-memory tmpfs.
func (i *defaultImageInspector) createAndExtractImage(client *docker.Client, containerName string) (*docker.Image, error) {
container, err := client.CreateContainer(docker.CreateContainerOptions{
Name: containerName,
Config: &docker.Config{
Image: i.opts.Image,
// For security purpose we don't define any entrypoint and command
Entrypoint: []string{""},
Cmd: []string{""},
},
})
if err != nil {
return nil, fmt.Errorf("Unable to create docker container: %v\n", err)
}
// delete the container when we are done extracting it
defer func() {
client.RemoveContainer(docker.RemoveContainerOptions{
ID: container.ID,
})
}()
containerMetadata, err := client.InspectContainer(container.ID)
if err != nil {
return nil, fmt.Errorf("Unable to get docker container information: %v\n", err)
}
imageMetadata, err := client.InspectImage(containerMetadata.Image)
if err != nil {
return imageMetadata, fmt.Errorf("Unable to get docker image information: %v\n", err)
}
if i.opts.DstPath, err = createOutputDir(i.opts.DstPath, "image-inspector-"); err != nil {
return imageMetadata, err
}
reader, writer := io.Pipe()
// handle closing the reader/writer in the method that creates them
defer writer.Close()
defer reader.Close()
log.Printf("Extracting image %s to %s", i.opts.Image, i.opts.DstPath)
// start the copy function first which will block after the first write while waiting for
// the reader to read.
errorChannel := make(chan error)
go func() {
errorChannel <- client.CopyFromContainer(docker.CopyFromContainerOptions{
Container: container.ID,
OutputStream: writer,
Resource: "/",
})
}()
// block on handling the reads here so we ensure both the write and the reader are finished
// (read waits until an EOF or error occurs).
handleTarStream(reader, i.opts.DstPath)
// capture any error from the copy, ensures both the handleTarStream and CopyFromContainer
// are done.
err = <-errorChannel
if err != nil {
return imageMetadata, fmt.Errorf("Unable to extract container: %v\n", err)
}
return imageMetadata, nil
}
