func launchContainer(client *docker.Client, containerImageName, name string) (*docker.Container, error) {
images, err := client.ListImages(docker.ListImagesOptions{Filter: containerImageName})
if err != nil {
return nil, err
}
targetImageId := images[0].ID
container, err := client.CreateContainer(docker.CreateContainerOptions{
Name: name,
Config: &docker.Config{
Image: targetImageId,
},
})
if err != nil {
return nil, err
}
if err := client.StartContainer(container.ID, &docker.HostConfig{PublishAllPorts: true}); err != nil {
return nil, err
}
return client.InspectContainer(container.ID)
}
func main() {
fmt.Println("dockerapi started...")
time.Sleep(time.Millisecond * 5000)
fmt.Println("action: " + ACTION)
fmt.Println("containers: " + CONTAINERS)
var containers = strings.Split(CONTAINERS, ",")
var docker *dockerapi.Client
var err error
docker, err = dockerapi.NewClient(DOCKER_HOST)
if err != nil {
fmt.Println(err.Error())
}
err = docker.Ping()
for i := range containers {
fmt.Println(ACTION + " issued for " + containers[i])
switch ACTION {
case "stop":
err = docker.StopContainer(containers[i], 5)
case "start":
err = docker.StartContainer(containers[i], nil)
default:
fmt.Println(ACTION + " unsupported action")
}
if err != nil {
fmt.Println(err.Error())
}
time.Sleep(time.Millisecond * 2000)
}
}
func runContainer(client *dockerapi.Client, createOpts dockerapi.CreateContainerOptions, startConfig *dockerapi.HostConfig) (string, error) {
container, err := client.CreateContainer(createOpts)
if err != nil {
return "", err
}
err = client.StartContainer(container.ID, startConfig)
// return container ID even if there is an error, so caller can clean up container if desired
return container.ID, err
}
// Run runs cmd in the given image using the docker client cl. It mounts cwd into containerMount in the running container and sends on the following channels:
//
// - rmContainerCh: a function closure that the receiver should call, after they receive on errCh or exitCodeCh, to remove the container. this is commonly done with a 'defer'
// - stdOut: all logs from STDOUT in the container. this may never receive
// - stdErr: all logs from STDERR in the container. this may never receive
// - exitCodeCh: the exit code of the container
// - errCh: any error in setting up or running the container. if errCh receives, exitCodeCh may not receive
func Run(
cl *docker.Client,
image *Image,
taskName,
cwd,
containerMount,
cmd string,
env []string,
rmContainerCh chan<- func(),
stdOut chan<- Log,
stdErr chan<- Log,
exitCodeCh chan<- int,
errCh chan<- error,
) {
mounts := []docker.Mount{
{Name: "pwd", Source: cwd, Destination: containerMount, Mode: "rxw"},
}
cmdSpl := strings.Split(cmd, " ")
containerName := NewContainerName(taskName, cwd)
createContainerOpts, hostConfig := CreateAndStartContainerOpts(image.String(), containerName, cmdSpl, env, mounts, containerMount)
if err := EnsureImage(cl, image.String(), func() (io.Writer, error) {
return os.Stdout, nil
}); err != nil {
errCh <- err
return
}
container, err := cl.CreateContainer(createContainerOpts)
if err != nil {
errCh <- err
}
rmContainerCh <- func() {
if err := cl.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID, Force: true}); err != nil {
log.Warn("Error removing container %s (%s)", container.ID, err)
}
}
log.Debug(CmdStr(createContainerOpts, hostConfig))
attachOpts := AttachToContainerOpts(container.ID, NewChanWriter(stdOut), NewChanWriter(stdErr))
// attach before the container starts, so we get all the logs etc...
go AttachAndWait(cl, container.ID, attachOpts, exitCodeCh, errCh)
if startErr := cl.StartContainer(container.ID, &hostConfig); startErr != nil {
errCh <- err
return
}
}
// createAndStartRouterContainer is responsible for deploying the router image in docker. It assumes that all router images
// will use a command line flag that can take --master which points to the master url
func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp string) (containerId string, err error) {
ports := []string{"80", "443"}
portBindings := make(map[dockerClient.Port][]dockerClient.PortBinding)
exposedPorts := map[dockerClient.Port]struct{}{}
for _, p := range ports {
dockerPort := dockerClient.Port(p + "/tcp")
portBindings[dockerPort] = []dockerClient.PortBinding{
{
HostPort: p,
},
}
exposedPorts[dockerPort] = struct{}{}
}
containerOpts := dockerClient.CreateContainerOptions{
Config: &dockerClient.Config{
Image: getRouterImage(),
Cmd: []string{"--master=" + masterIp, "--loglevel=4"},
ExposedPorts: exposedPorts,
},
}
container, err := dockerCli.CreateContainer(containerOpts)
if err != nil {
return "", err
}
dockerHostCfg := &dockerClient.HostConfig{NetworkMode: "host", PortBindings: portBindings}
err = dockerCli.StartContainer(container.ID, dockerHostCfg)
if err != nil {
return "", err
}
running := false
//wait for it to start
for i := 0; i < dockerRetries; i++ {
c, err := dockerCli.InspectContainer(container.ID)
if err != nil {
return "", err
}
if c.State.Running {
running = true
break
}
time.Sleep(time.Second * dockerWaitSeconds)
}
if !running {
return "", errors.New("Container did not start after 3 tries!")
}
return container.ID, nil
}
// StartDockerContainer starts a new Lever container for the specified
// environment and service.
func StartDockerContainer(
docker *dockerapi.Client, environment string, service string,
instanceID string, codeVersion int64, isAdmin bool,
leverConfig *core.LeverConfig) (
containerID string, node string, err error) {
codeDir := HostCodeDirPath(environment, service, codeVersion)
binds := []string{codeDir + ":/leveros/custcode:ro,Z"}
env := []string{
"LEVEROS_ENVIRONMENT=" + environment,
"LEVEROS_SERVICE=" + service,
"LEVEROS_INSTANCE_ID=" + instanceID,
"LEVEROS_CODE_VERSION=" + strconv.Itoa(int(codeVersion)),
"LEVEROS_INTERNAL_ENV_SUFFIX=" +
core.InternalEnvironmentSuffixFlag.Get(),
}
if isAdmin {
// This is used by admin to make deployments.
binds = append(
binds, LeverCodeHostDirFlag.Get()+":/leveros/custcodetree:Z")
}
// Configure logging.
var logConfig dockerapi.LogConfig
if devlogger.DisableFlag.Get() {
logConfig.Type = "none"
} else {
// TODO: Should use scale.Dereference... to get IP of syslog server
// and shard by env+service.
tag := fmt.Sprintf(
"%s/%s/%d/%s", environment, service, codeVersion, instanceID)
logConfig.Type = "syslog"
logConfig.Config = map[string]string{
"syslog-address": "tcp://127.0.0.1:6514",
"syslog-facility": "user",
"tag": tag,
"syslog-format": "rfc5424",
}
}
memoryBytes := int64(leverConfig.InstanceMemoryMB) * 1000 * 1000
memAndSwapBytes := memoryBytes // No swap.
kernelMemBytes := memoryBytes / 10 // Allow 10% memory for kernel.
// Entry point.
entry := leverConfig.EntryPoint
if leverConfig.JSEntryPoint != "" {
// Trigger GC in node when garbage reaches 90% of memory.
maxGarbage := strconv.Itoa(
int(float32(leverConfig.InstanceMemoryMB) * 0.9))
// Set entry point for node.
entry = []string{
"node", "--optimize_for_size", "--max_old_space_size=" + maxGarbage,
"--gc_interval=100",
"/leveros/js/leveros-server/compiled/lib/serve.js",
leverConfig.JSEntryPoint,
}
}
container, err := docker.CreateContainer(dockerapi.CreateContainerOptions{
Name: "leveros_" + instanceID,
Config: &dockerapi.Config{
Image: "leveros/levercontainer:latest",
Cmd: entry,
Env: env,
KernelMemory: kernelMemBytes,
Labels: map[string]string{
"com.leveros.environment": environment,
"com.leveros.service": service,
"com.leveros.instanceid": instanceID,
"com.leveros.codeversion": strconv.Itoa(int(codeVersion)),
},
},
// TODO: Documentation for these here:
// https://docs.docker.com/engine/reference/api/docker_remote_api_v1.23/#create-a-container
// TODO: Also check if need to set limits on IO operations (blkio).
// TODO: Should allow to write to disk, but limit amount of disk space.
HostConfig: &dockerapi.HostConfig{
ReadonlyRootfs: true,
Binds: binds,
CapDrop: []string{"all"},
NetworkMode: "none",
Ulimits: []dockerapi.ULimit{}, // TODO
SecurityOpt: []string{"no-new-privileges"}, // TODO
LogConfig: logConfig,
Memory: memoryBytes,
MemorySwap: memAndSwapBytes,
MemorySwappiness: 0,
CPUShares: 0, // TODO
CPUPeriod: 0, // TODO
CPUQuota: 0, // TODO
},
})
if err != nil {
logger.WithFields("err", err).Debug(
"Error trying to create container")
return "", "", err
}
// Get info about the node it was allocated to by Docker Swarm.
container, err = docker.InspectContainer(container.ID)
if err != nil {
removeErr := RemoveDockerContainer(docker, container.ID)
if removeErr != nil {
logger.WithFields(
"containerID", containerID,
"err", removeErr,
).Error("Error trying to remove container after previous error")
}
return "", "", err
}
if container.Node != nil {
node = container.Node.Name
} else {
// In a dev/testing (non-swarm) environment.
logger.Warning(
"Using non-swarm node. " +
"YOU SHOULD NEVER SEE THIS IN PRODUCTION.")
node = "leverosconsul"
}
// Start the container.
err = docker.StartContainer(container.ID, nil)
if err != nil {
removeErr := RemoveDockerContainer(docker, container.ID)
if removeErr != nil {
logger.WithFields(
"containerID", containerID,
"err", removeErr,
).Error("Error trying to remove container after failed to start")
}
return "", "", err
}
// Need to disconnect it from the "none" network before being able to
// connect it to its local environment network.
err = DisconnectFromDockerEnvNetwork(docker, container.ID, "none")
if err != nil {
removeErr := RemoveDockerContainer(docker, container.ID)
if removeErr != nil {
logger.WithFields(
"containerID", containerID,
"err", removeErr,
).Error("Error trying to remove container after previous error")
}
return "", "", err
}
return container.ID, node, nil
}
// createAndStartRouterContainer is responsible for deploying the router image in docker. It assumes that all router images
// will use a command line flag that can take --master which points to the master url
func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp string, routerStatsPort int, reloadInterval int) (containerId string, err error) {
ports := []string{"80", "443"}
if routerStatsPort > 0 {
ports = append(ports, fmt.Sprintf("%d", routerStatsPort))
}
portBindings := make(map[dockerClient.Port][]dockerClient.PortBinding)
exposedPorts := map[dockerClient.Port]struct{}{}
for _, p := range ports {
dockerPort := dockerClient.Port(p + "/tcp")
portBindings[dockerPort] = []dockerClient.PortBinding{
{
HostPort: p,
},
}
exposedPorts[dockerPort] = struct{}{}
}
copyEnv := []string{
"ROUTER_EXTERNAL_HOST_HOSTNAME",
"ROUTER_EXTERNAL_HOST_USERNAME",
"ROUTER_EXTERNAL_HOST_PASSWORD",
"ROUTER_EXTERNAL_HOST_HTTP_VSERVER",
"ROUTER_EXTERNAL_HOST_HTTPS_VSERVER",
"ROUTER_EXTERNAL_HOST_INSECURE",
"ROUTER_EXTERNAL_HOST_PRIVKEY",
}
env := []string{
fmt.Sprintf("STATS_PORT=%d", routerStatsPort),
fmt.Sprintf("STATS_USERNAME=%s", statsUser),
fmt.Sprintf("STATS_PASSWORD=%s", statsPassword),
fmt.Sprintf("DEFAULT_CERTIFICATE=%s", defaultCert),
}
reloadIntVar := fmt.Sprintf("RELOAD_INTERVAL=%ds", reloadInterval)
env = append(env, reloadIntVar)
for _, name := range copyEnv {
val := os.Getenv(name)
if len(val) > 0 {
env = append(env, name+"="+val)
}
}
vols := ""
hostVols := []string{}
privkeyFilename := os.Getenv("ROUTER_EXTERNAL_HOST_PRIVKEY")
if len(privkeyFilename) != 0 {
vols = privkeyFilename
privkeyBindmount := fmt.Sprintf("%[1]s:%[1]s", privkeyFilename)
hostVols = append(hostVols, privkeyBindmount)
}
binary := os.Getenv("ROUTER_OPENSHIFT_BINARY")
if len(binary) != 0 {
hostVols = append(hostVols, fmt.Sprintf("%[1]s:/usr/bin/openshift", binary))
}
containerOpts := dockerClient.CreateContainerOptions{
Config: &dockerClient.Config{
Image: getRouterImage(),
Cmd: []string{"--master=" + masterIp, "--loglevel=4"},
Env: env,
ExposedPorts: exposedPorts,
VolumesFrom: vols,
},
HostConfig: &dockerClient.HostConfig{
Binds: hostVols,
},
}
container, err := dockerCli.CreateContainer(containerOpts)
if err != nil {
return "", err
}
dockerHostCfg := &dockerClient.HostConfig{NetworkMode: "host", PortBindings: portBindings}
err = dockerCli.StartContainer(container.ID, dockerHostCfg)
if err != nil {
return "", err
}
//wait for it to start
if err := wait.Poll(time.Millisecond*100, time.Second*30, func() (bool, error) {
c, err := dockerCli.InspectContainer(container.ID)
if err != nil {
return false, err
}
return c.State.Running, nil
}); err != nil {
return "", err
}
return container.ID, nil
}
// StartContainers starts all the containers in ids slice
func StartContainers(client *docker.Client, ids []string) {
for _, id := range ids {
client.StartContainer(id, &docker.HostConfig{})
}
}
// createAndStartRouterContainer is responsible for deploying the router image in docker. It assumes that all router images
// will use a command line flag that can take --master which points to the master url
func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp string, routerStatsPort int) (containerId string, err error) {
ports := []string{"80", "443"}
if routerStatsPort > 0 {
ports = append(ports, fmt.Sprintf("%d", routerStatsPort))
}
portBindings := make(map[dockerClient.Port][]dockerClient.PortBinding)
exposedPorts := map[dockerClient.Port]struct{}{}
for _, p := range ports {
dockerPort := dockerClient.Port(p + "/tcp")
portBindings[dockerPort] = []dockerClient.PortBinding{
{
HostPort: p,
},
}
exposedPorts[dockerPort] = struct{}{}
}
copyEnv := []string{
"ROUTER_EXTERNAL_HOST_HOSTNAME",
"ROUTER_EXTERNAL_HOST_USERNAME",
"ROUTER_EXTERNAL_HOST_PASSWORD",
"ROUTER_EXTERNAL_HOST_HTTP_VSERVER",
"ROUTER_EXTERNAL_HOST_HTTPS_VSERVER",
"ROUTER_EXTERNAL_HOST_INSECURE",
"ROUTER_EXTERNAL_HOST_PRIVKEY",
}
env := []string{
fmt.Sprintf("STATS_PORT=%d", routerStatsPort),
fmt.Sprintf("STATS_USERNAME=%s", statsUser),
fmt.Sprintf("STATS_PASSWORD=%s", statsPassword),
}
for _, name := range copyEnv {
val := os.Getenv(name)
if len(val) > 0 {
env = append(env, name+"="+val)
}
}
vols := ""
hostVols := []string{}
privkeyFilename := os.Getenv("ROUTER_EXTERNAL_HOST_PRIVKEY")
if len(privkeyFilename) != 0 {
vols = privkeyFilename
privkeyBindmount := fmt.Sprintf("%[1]s:%[1]s", privkeyFilename)
hostVols = append(hostVols, privkeyBindmount)
}
binary := os.Getenv("ROUTER_OPENSHIFT_BINARY")
if len(binary) != 0 {
hostVols = append(hostVols, fmt.Sprintf("%[1]s:/usr/bin/openshift", binary))
}
containerOpts := dockerClient.CreateContainerOptions{
Config: &dockerClient.Config{
Image: getRouterImage(),
Cmd: []string{"--master=" + masterIp, "--loglevel=4"},
Env: env,
ExposedPorts: exposedPorts,
VolumesFrom: vols,
},
HostConfig: &dockerClient.HostConfig{
Binds: hostVols,
},
}
container, err := dockerCli.CreateContainer(containerOpts)
if err != nil {
return "", err
}
dockerHostCfg := &dockerClient.HostConfig{NetworkMode: "host", PortBindings: portBindings}
err = dockerCli.StartContainer(container.ID, dockerHostCfg)
if err != nil {
return "", err
}
running := false
//wait for it to start
for i := 0; i < dockerRetries; i++ {
time.Sleep(time.Second * dockerWaitSeconds)
c, err := dockerCli.InspectContainer(container.ID)
if err != nil {
return "", err
}
if c.State.Running {
running = true
break
}
}
if !running {
return "", errors.New("Container did not start after 3 tries!")
}
return container.ID, nil
}
