// makeSandboxDockerConfig returns dockertypes.ContainerCreateConfig based on runtimeapi.PodSandboxConfig.
func (ds *dockerService) makeSandboxDockerConfig(c *runtimeapi.PodSandboxConfig, image string) (*dockertypes.ContainerCreateConfig, error) {
// Merge annotations and labels because docker supports only labels.
labels := makeLabels(c.GetLabels(), c.GetAnnotations())
// Apply a label to distinguish sandboxes from regular containers.
labels[containerTypeLabelKey] = containerTypeLabelSandbox
// Apply a container name label for infra container. This is used in summary v1.
// TODO(random-liu): Deprecate this label once container metrics is directly got from CRI.
labels[types.KubernetesContainerNameLabel] = sandboxContainerName
hc := &dockercontainer.HostConfig{}
createConfig := &dockertypes.ContainerCreateConfig{
Name: makeSandboxName(c),
Config: &dockercontainer.Config{
Hostname: c.GetHostname(),
// TODO: Handle environment variables.
Image: image,
Labels: labels,
},
HostConfig: hc,
}
// Set sysctls if requested
sysctls, err := getSysctlsFromAnnotations(c.Annotations)
if err != nil {
return nil, fmt.Errorf("failed to get sysctls from annotations %v for sandbox %q: %v", c.Annotations, c.Metadata.GetName(), err)
}
hc.Sysctls = sysctls
// Apply linux-specific options.
if lc := c.GetLinux(); lc != nil {
if err := ds.applySandboxLinuxOptions(hc, lc, createConfig, image); err != nil {
return nil, err
}
}
// Set port mappings.
exposedPorts, portBindings := makePortsAndBindings(c.GetPortMappings())
createConfig.Config.ExposedPorts = exposedPorts
hc.PortBindings = portBindings
// Set DNS options.
if dnsConfig := c.GetDnsConfig(); dnsConfig != nil {
hc.DNS = dnsConfig.GetServers()
hc.DNSSearch = dnsConfig.GetSearches()
hc.DNSOptions = dnsConfig.GetOptions()
}
// Apply resource options.
setSandboxResources(hc)
// Set security options.
securityOpts, err := getSandboxSecurityOpts(c, ds.seccompProfileRoot)
if err != nil {
return nil, fmt.Errorf("failed to generate sandbox security options for sandbox %q: %v", c.Metadata.GetName(), err)
}
hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...)
return createConfig, nil
}
// RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
// the sandbox is in ready state.
// For docker, PodSandbox is implemented by a container holding the network
// namespace for the pod.
// Note: docker doesn't use LogDirectory (yet).
func (ds *dockerService) RunPodSandbox(config *runtimeapi.PodSandboxConfig) (string, error) {
// Step 1: Pull the image for the sandbox.
image := defaultSandboxImage
podSandboxImage := ds.podSandboxImage
if len(podSandboxImage) != 0 {
image = podSandboxImage
}
// NOTE: To use a custom sandbox image in a private repository, users need to configure the nodes with credentials properly.
// see: http://kubernetes.io/docs/user-guide/images/#configuring-nodes-to-authenticate-to-a-private-repository
if err := ds.client.PullImage(image, dockertypes.AuthConfig{}, dockertypes.ImagePullOptions{}); err != nil {
return "", fmt.Errorf("unable to pull image for the sandbox container: %v", err)
}
// Step 2: Create the sandbox container.
createConfig, err := ds.makeSandboxDockerConfig(config, image)
if err != nil {
return "", fmt.Errorf("failed to make sandbox docker config for pod %q: %v", config.Metadata.GetName(), err)
}
createResp, err := ds.client.CreateContainer(*createConfig)
recoverFromConflictIfNeeded(ds.client, err)
if err != nil || createResp == nil {
return "", fmt.Errorf("failed to create a sandbox for pod %q: %v", config.Metadata.GetName(), err)
}
// Step 3: Start the sandbox container.
// Assume kubelet's garbage collector would remove the sandbox later, if
// startContainer failed.
err = ds.client.StartContainer(createResp.ID)
if err != nil {
return createResp.ID, fmt.Errorf("failed to start sandbox container for pod %q: %v", config.Metadata.GetName(), err)
}
if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetHostNetwork() {
return createResp.ID, nil
}
// Step 4: Setup networking for the sandbox.
// All pod networking is setup by a CNI plugin discovered at startup time.
// This plugin assigns the pod ip, sets up routes inside the sandbox,
// creates interfaces etc. In theory, its jurisdiction ends with pod
// sandbox networking, but it might insert iptables rules or open ports
// on the host as well, to satisfy parts of the pod spec that aren't
// recognized by the CNI standard yet.
cID := kubecontainer.BuildContainerID(runtimeName, createResp.ID)
err = ds.networkPlugin.SetUpPod(config.GetMetadata().GetNamespace(), config.GetMetadata().GetName(), cID)
// TODO: Do we need to teardown on failure or can we rely on a StopPodSandbox call with the given ID?
return createResp.ID, err
}
func makeSandboxDockerConfig(c *runtimeApi.PodSandboxConfig, image string) *dockertypes.ContainerCreateConfig {
// Merge annotations and labels because docker supports only labels.
labels := makeLabels(c.GetLabels(), c.GetAnnotations())
// Apply a label to distinguish sandboxes from regular containers.
labels[containerTypeLabelKey] = containerTypeLabelSandbox
hc := &dockercontainer.HostConfig{}
createConfig := &dockertypes.ContainerCreateConfig{
Name: c.GetName(),
Config: &dockercontainer.Config{
Hostname: c.GetHostname(),
// TODO: Handle environment variables.
Image: image,
Labels: labels,
},
HostConfig: hc,
}
// Apply linux-specific options.
if lc := c.GetLinux(); lc != nil {
// Apply Cgroup options.
// TODO: Check if this works with per-pod cgroups.
hc.CgroupParent = lc.GetCgroupParent()
// Apply namespace options.
hc.NetworkMode, hc.UTSMode, hc.PidMode = "", "", ""
nsOpts := lc.GetNamespaceOptions()
if nsOpts != nil {
if nsOpts.GetHostNetwork() {
hc.NetworkMode = namespaceModeHost
} else {
// Assume kubelet uses either the cni or the kubenet plugin.
// TODO: support docker networking.
hc.NetworkMode = "none"
}
if nsOpts.GetHostIpc() {
hc.IpcMode = namespaceModeHost
}
if nsOpts.GetHostPid() {
hc.PidMode = namespaceModeHost
}
}
}
// Set port mappings.
exposedPorts, portBindings := makePortsAndBindings(c.GetPortMappings())
createConfig.Config.ExposedPorts = exposedPorts
hc.PortBindings = portBindings
// Set DNS options.
if dnsOpts := c.GetDnsOptions(); dnsOpts != nil {
hc.DNS = dnsOpts.GetServers()
hc.DNSSearch = dnsOpts.GetSearches()
}
// Apply resource options.
setSandboxResources(c.GetResources(), hc)
// Set security options.
hc.SecurityOpt = []string{getSeccompOpts()}
return createConfig
}
// CreateContainer creates a new container in the given PodSandbox
// Note: docker doesn't use LogPath yet.
// TODO: check if the default values returned by the runtime API are ok.
func (ds *dockerService) CreateContainer(podSandboxID string, config *runtimeApi.ContainerConfig, sandboxConfig *runtimeApi.PodSandboxConfig) (string, error) {
if config == nil {
return "", fmt.Errorf("container config is nil")
}
if sandboxConfig == nil {
return "", fmt.Errorf("sandbox config is nil for container %q", config.Metadata.GetName())
}
labels := makeLabels(config.GetLabels(), config.GetAnnotations())
// Apply a the container type label.
labels[containerTypeLabelKey] = containerTypeLabelContainer
// Write the sandbox ID in the labels.
labels[sandboxIDLabelKey] = podSandboxID
image := ""
if iSpec := config.GetImage(); iSpec != nil {
image = iSpec.GetImage()
}
createConfig := dockertypes.ContainerCreateConfig{
Name: makeContainerName(sandboxConfig, config),
Config: &dockercontainer.Config{
// TODO: set User.
Entrypoint: dockerstrslice.StrSlice(config.GetCommand()),
Cmd: dockerstrslice.StrSlice(config.GetArgs()),
Env: generateEnvList(config.GetEnvs()),
Image: image,
WorkingDir: config.GetWorkingDir(),
Labels: labels,
// Interactive containers:
OpenStdin: config.GetStdin(),
StdinOnce: config.GetStdinOnce(),
Tty: config.GetTty(),
},
}
// Fill the HostConfig.
hc := &dockercontainer.HostConfig{
Binds: generateMountBindings(config.GetMounts()),
ReadonlyRootfs: config.GetReadonlyRootfs(),
Privileged: config.GetPrivileged(),
}
// Apply options derived from the sandbox config.
if lc := sandboxConfig.GetLinux(); lc != nil {
// Apply Cgroup options.
// TODO: Check if this works with per-pod cgroups.
hc.CgroupParent = lc.GetCgroupParent()
// Apply namespace options.
sandboxNSMode := fmt.Sprintf("container:%v", podSandboxID)
hc.NetworkMode = dockercontainer.NetworkMode(sandboxNSMode)
hc.IpcMode = dockercontainer.IpcMode(sandboxNSMode)
hc.UTSMode = ""
hc.PidMode = ""
nsOpts := lc.GetNamespaceOptions()
if nsOpts != nil {
if nsOpts.GetHostNetwork() {
hc.UTSMode = namespaceModeHost
}
if nsOpts.GetHostPid() {
hc.PidMode = namespaceModeHost
}
}
}
// Apply Linux-specific options if applicable.
if lc := config.GetLinux(); lc != nil {
// Apply resource options.
// TODO: Check if the units are correct.
// TODO: Can we assume the defaults are sane?
rOpts := lc.GetResources()
if rOpts != nil {
hc.Resources = dockercontainer.Resources{
Memory: rOpts.GetMemoryLimitInBytes(),
MemorySwap: -1, // Always disable memory swap.
CPUShares: rOpts.GetCpuShares(),
CPUQuota: rOpts.GetCpuQuota(),
CPUPeriod: rOpts.GetCpuPeriod(),
// TODO: Need to set devices.
}
hc.OomScoreAdj = int(rOpts.GetOomScoreAdj())
}
// Note: ShmSize is handled in kube_docker_client.go
}
var err error
hc.SecurityOpt, err = getContainerSecurityOpts(config.Metadata.GetName(), sandboxConfig, ds.seccompProfileRoot)
if err != nil {
return "", fmt.Errorf("failed to generate container security options for container %q: %v", config.Metadata.GetName(), err)
}
// TODO: Add or drop capabilities.
createConfig.HostConfig = hc
createResp, err := ds.client.CreateContainer(createConfig)
if createResp != nil {
return createResp.ID, err
}
return "", err
}
func (ds *dockerService) makeSandboxDockerConfig(c *runtimeApi.PodSandboxConfig, image string) (*dockertypes.ContainerCreateConfig, error) {
// Merge annotations and labels because docker supports only labels.
labels := makeLabels(c.GetLabels(), c.GetAnnotations())
// Apply a label to distinguish sandboxes from regular containers.
labels[containerTypeLabelKey] = containerTypeLabelSandbox
// Apply a container name label for infra container. This is used in summary api.
// TODO(random-liu): Deprecate this label once container metrics is directly got from CRI.
labels[types.KubernetesContainerNameLabel] = sandboxContainerName
hc := &dockercontainer.HostConfig{}
createConfig := &dockertypes.ContainerCreateConfig{
Name: makeSandboxName(c),
Config: &dockercontainer.Config{
Hostname: c.GetHostname(),
// TODO: Handle environment variables.
Image: image,
Labels: labels,
},
HostConfig: hc,
}
// Apply linux-specific options.
if lc := c.GetLinux(); lc != nil {
// Apply Cgroup options.
// TODO: Check if this works with per-pod cgroups.
hc.CgroupParent = lc.GetCgroupParent()
// Apply namespace options.
hc.NetworkMode, hc.UTSMode, hc.PidMode = "", "", ""
nsOpts := lc.GetNamespaceOptions()
if nsOpts != nil {
if nsOpts.GetHostNetwork() {
hc.NetworkMode = namespaceModeHost
} else {
// Assume kubelet uses either the cni or the kubenet plugin.
// TODO: support docker networking.
hc.NetworkMode = "none"
}
if nsOpts.GetHostIpc() {
hc.IpcMode = namespaceModeHost
}
if nsOpts.GetHostPid() {
hc.PidMode = namespaceModeHost
}
}
}
// Set port mappings.
exposedPorts, portBindings := makePortsAndBindings(c.GetPortMappings())
createConfig.Config.ExposedPorts = exposedPorts
hc.PortBindings = portBindings
// Set DNS options.
if dnsConfig := c.GetDnsConfig(); dnsConfig != nil {
hc.DNS = dnsConfig.GetServers()
hc.DNSSearch = dnsConfig.GetSearches()
hc.DNSOptions = dnsConfig.GetOptions()
}
// Apply resource options.
setSandboxResources(hc)
// Set security options.
var err error
hc.SecurityOpt, err = getSandboxSecurityOpts(c, ds.seccompProfileRoot)
if err != nil {
return nil, fmt.Errorf("failed to generate sandbox security options for sandbox %q: %v", c.Metadata.GetName(), err)
}
return createConfig, nil
}
// CreateContainer creates a new container in the given PodSandbox
// Note: docker doesn't use LogPath yet.
// TODO: check if the default values returned by the runtime API are ok.
func (ds *dockerService) CreateContainer(podSandboxID string, config *runtimeApi.ContainerConfig, sandboxConfig *runtimeApi.PodSandboxConfig) (string, error) {
if config == nil {
return "", fmt.Errorf("container config is nil")
}
if sandboxConfig == nil {
return "", fmt.Errorf("sandbox config is nil for container %q", config.Metadata.GetName())
}
labels := makeLabels(config.GetLabels(), config.GetAnnotations())
// Apply a the container type label.
labels[containerTypeLabelKey] = containerTypeLabelContainer
// Write the container log path in the labels.
labels[containerLogPathLabelKey] = filepath.Join(sandboxConfig.GetLogDirectory(), config.GetLogPath())
// Write the sandbox ID in the labels.
labels[sandboxIDLabelKey] = podSandboxID
image := ""
if iSpec := config.GetImage(); iSpec != nil {
image = iSpec.GetImage()
}
createConfig := dockertypes.ContainerCreateConfig{
Name: makeContainerName(sandboxConfig, config),
Config: &dockercontainer.Config{
// TODO: set User.
Entrypoint: dockerstrslice.StrSlice(config.GetCommand()),
Cmd: dockerstrslice.StrSlice(config.GetArgs()),
Env: generateEnvList(config.GetEnvs()),
Image: image,
WorkingDir: config.GetWorkingDir(),
Labels: labels,
// Interactive containers:
OpenStdin: config.GetStdin(),
StdinOnce: config.GetStdinOnce(),
Tty: config.GetTty(),
},
}
// Fill the HostConfig.
hc := &dockercontainer.HostConfig{
Binds: generateMountBindings(config.GetMounts()),
}
// Apply cgroupsParent derived from the sandbox config.
if lc := sandboxConfig.GetLinux(); lc != nil {
// Apply Cgroup options.
// TODO: Check if this works with per-pod cgroups.
// TODO: we need to pass the cgroup in syntax expected by cgroup driver but shim does not use docker info yet...
hc.CgroupParent = lc.GetCgroupParent()
}
// Apply Linux-specific options if applicable.
if lc := config.GetLinux(); lc != nil {
// Apply resource options.
// TODO: Check if the units are correct.
// TODO: Can we assume the defaults are sane?
rOpts := lc.GetResources()
if rOpts != nil {
hc.Resources = dockercontainer.Resources{
Memory: rOpts.GetMemoryLimitInBytes(),
MemorySwap: -1, // Always disable memory swap.
CPUShares: rOpts.GetCpuShares(),
CPUQuota: rOpts.GetCpuQuota(),
CPUPeriod: rOpts.GetCpuPeriod(),
}
hc.OomScoreAdj = int(rOpts.GetOomScoreAdj())
}
// Note: ShmSize is handled in kube_docker_client.go
// Apply security context.
applyContainerSecurityContext(lc, podSandboxID, createConfig.Config, hc)
}
// Set devices for container.
devices := make([]dockercontainer.DeviceMapping, len(config.Devices))
for i, device := range config.Devices {
devices[i] = dockercontainer.DeviceMapping{
PathOnHost: device.GetHostPath(),
PathInContainer: device.GetContainerPath(),
CgroupPermissions: device.GetPermissions(),
}
}
hc.Resources.Devices = devices
// Apply appArmor and seccomp options.
securityOpts, err := getContainerSecurityOpts(config.Metadata.GetName(), sandboxConfig, ds.seccompProfileRoot)
if err != nil {
return "", fmt.Errorf("failed to generate container security options for container %q: %v", config.Metadata.GetName(), err)
}
hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...)
createConfig.HostConfig = hc
createResp, err := ds.client.CreateContainer(createConfig)
if createResp != nil {
return createResp.ID, err
}
return "", err
}
// CreateContainer creates a new container in the given PodSandbox
// Note: docker doesn't use LogPath yet.
// TODO: check if the default values returned by the runtime API are ok.
func (ds *dockerService) CreateContainer(podSandboxID string, config *runtimeApi.ContainerConfig, sandboxConfig *runtimeApi.PodSandboxConfig) (string, error) {
if config == nil {
return "", fmt.Errorf("container config is nil")
}
if sandboxConfig == nil {
return "", fmt.Errorf("sandbox config is nil for container %q", config.GetName())
}
// Merge annotations and labels because docker supports only labels.
// TODO: add a prefix to annotations so that we can distinguish labels and
// annotations when reading back them from the docker container.
// TODO: should we apply docker-specific labels?
labels := config.GetLabels()
for k, v := range config.GetAnnotations() {
if _, ok := labels[k]; !ok {
// Only write to labels if the key doesn't exist.
labels[k] = v
}
}
image := ""
if iSpec := config.GetImage(); iSpec != nil {
image = iSpec.GetImage()
}
createConfig := dockertypes.ContainerCreateConfig{
Name: config.GetName(),
Config: &dockercontainer.Config{
// TODO: set User.
Hostname: sandboxConfig.GetHostname(),
Entrypoint: dockerstrslice.StrSlice(config.GetCommand()),
Cmd: dockerstrslice.StrSlice(config.GetArgs()),
Env: generateEnvList(config.GetEnvs()),
Image: image,
WorkingDir: config.GetWorkingDir(),
Labels: labels,
// Interactive containers:
OpenStdin: config.GetStdin(),
StdinOnce: config.GetStdinOnce(),
Tty: config.GetTty(),
},
}
// Fill the HostConfig.
hc := &dockercontainer.HostConfig{
Binds: generateMountBindings(config.GetMounts()),
ReadonlyRootfs: config.GetReadonlyRootfs(),
Privileged: config.GetPrivileged(),
}
// Apply options derived from the sandbox config.
if lc := sandboxConfig.GetLinux(); lc != nil {
// Apply Cgroup options.
// TODO: Check if this works with per-pod cgroups.
hc.CgroupParent = lc.GetCgroupParent()
// Apply namespace options.
sandboxNSMode := fmt.Sprintf("container:%v", podSandboxID)
hc.NetworkMode = dockercontainer.NetworkMode(sandboxNSMode)
hc.IpcMode = dockercontainer.IpcMode(sandboxNSMode)
hc.UTSMode = ""
hc.PidMode = ""
nsOpts := lc.GetNamespaceOptions()
if nsOpts != nil {
if nsOpts.GetHostNetwork() {
hc.UTSMode = namespaceModeHost
}
if nsOpts.GetHostPid() {
hc.PidMode = namespaceModeHost
}
}
}
// Apply Linux-specific options if applicable.
if lc := config.GetLinux(); lc != nil {
// Apply resource options.
// TODO: Check if the units are correct.
// TODO: Can we assume the defaults are sane?
rOpts := lc.GetResources()
if rOpts != nil {
hc.Resources = dockercontainer.Resources{
Memory: rOpts.GetMemoryLimitInBytes(),
MemorySwap: -1, // Always disable memory swap.
CPUShares: rOpts.GetCpuShares(),
CPUQuota: rOpts.GetCpuQuota(),
CPUPeriod: rOpts.GetCpuPeriod(),
// TODO: Need to set devices.
}
hc.OomScoreAdj = int(rOpts.GetOomScoreAdj())
}
// Note: ShmSize is handled in kube_docker_client.go
}
// TODO: Seccomp support. Need to figure out how to pass seccomp options
// through the runtime API (annotations?).See dockerManager.getSecurityOpts()
// for the details. Always set the default seccomp profile for now.
hc.SecurityOpt = []string{fmt.Sprintf("%s=%s", "seccomp", defaultSeccompProfile)}
// TODO: Add or drop capabilities.
createConfig.HostConfig = hc
createResp, err := ds.client.CreateContainer(createConfig)
if createResp != nil {
return createResp.ID, err
}
return "", err
}