// setApp merges the container spec with the image's manifest.
func setApp(app *appctypes.App, c *api.Container, opts *kubecontainer.RunContainerOptions, ctx *api.SecurityContext, podCtx *api.PodSecurityContext) error {
// TODO(yifan): If ENTRYPOINT and CMD are both specified in the image,
// we cannot override just one of these at this point as they are already mixed.
command, args := kubecontainer.ExpandContainerCommandAndArgs(c, opts.Envs)
exec := append(command, args...)
if len(exec) > 0 {
app.Exec = exec
}
// Set UID and GIDs.
if err := verifyNonRoot(app, ctx); err != nil {
return err
}
if ctx != nil && ctx.RunAsUser != nil {
app.User = strconv.Itoa(int(*ctx.RunAsUser))
}
setSupplementaryGIDs(app, podCtx)
// Set working directory.
if len(c.WorkingDir) > 0 {
app.WorkingDirectory = c.WorkingDir
}
// Notes that we don't create Mounts section in the pod manifest here,
// as Mounts will be automatically generated by rkt.
mergeMounts(app, opts.Mounts)
mergeEnv(app, opts.Envs)
mergePortMappings(app, opts.PortMappings)
return setIsolators(app, c, ctx)
}
// generateContainerConfig generates container config for kubelet runtime v1.
func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP string) (*runtimeapi.ContainerConfig, error) {
opts, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
if err != nil {
return nil, err
}
uid, username, err := m.getImageUser(container.Image)
if err != nil {
return nil, err
}
if uid != nil {
// Verify RunAsNonRoot. Non-root verification only supports numeric user.
if err := verifyRunAsNonRoot(pod, container, *uid); err != nil {
return nil, err
}
} else {
glog.Warningf("Non-root verification doesn't support non-numeric user (%s)", *username)
}
command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs)
containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
restartCountUint32 := uint32(restartCount)
config := &runtimeapi.ContainerConfig{
Metadata: &runtimeapi.ContainerMetadata{
Name: &container.Name,
Attempt: &restartCountUint32,
},
Image: &runtimeapi.ImageSpec{Image: &container.Image},
Command: command,
Args: args,
WorkingDir: &container.WorkingDir,
Labels: newContainerLabels(container, pod),
Annotations: newContainerAnnotations(container, pod, restartCount),
Devices: makeDevices(opts),
Mounts: m.makeMounts(opts, container),
LogPath: &containerLogsPath,
Stdin: &container.Stdin,
StdinOnce: &container.StdinOnce,
Tty: &container.TTY,
Linux: m.generateLinuxContainerConfig(container, pod, uid, username),
}
// set environment variables
envs := make([]*runtimeapi.KeyValue, len(opts.Envs))
for idx := range opts.Envs {
e := opts.Envs[idx]
envs[idx] = &runtimeapi.KeyValue{
Key: &e.Name,
Value: &e.Value,
}
}
config.Envs = envs
return config, nil
}
// generateContainerConfig generates container config for kubelet runtime api.
func (m *kubeGenericRuntimeManager) generateContainerConfig(container *api.Container, pod *api.Pod, restartCount int, podIP string) (*runtimeApi.ContainerConfig, error) {
opts, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
if err != nil {
return nil, err
}
command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs)
containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
podHasSELinuxLabel := pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.SELinuxOptions != nil
restartCountUint32 := uint32(restartCount)
config := &runtimeApi.ContainerConfig{
Metadata: &runtimeApi.ContainerMetadata{
Name: &container.Name,
Attempt: &restartCountUint32,
},
Image: &runtimeApi.ImageSpec{Image: &container.Image},
Command: command,
Args: args,
WorkingDir: &container.WorkingDir,
Labels: newContainerLabels(container, pod),
Annotations: newContainerAnnotations(container, pod, restartCount),
Mounts: m.makeMounts(opts, container, podHasSELinuxLabel),
Devices: makeDevices(opts),
LogPath: &containerLogsPath,
Stdin: &container.Stdin,
StdinOnce: &container.StdinOnce,
Tty: &container.TTY,
Linux: m.generateLinuxContainerConfig(container, pod),
}
// set privileged and readonlyRootfs
if container.SecurityContext != nil {
securityContext := container.SecurityContext
if securityContext.Privileged != nil {
config.Privileged = securityContext.Privileged
}
if securityContext.ReadOnlyRootFilesystem != nil {
config.ReadonlyRootfs = securityContext.ReadOnlyRootFilesystem
}
}
// set environment variables
envs := make([]*runtimeApi.KeyValue, len(opts.Envs))
for idx := range opts.Envs {
e := opts.Envs[idx]
envs[idx] = &runtimeApi.KeyValue{
Key: &e.Name,
Value: &e.Value,
}
}
config.Envs = envs
return config, nil
}
// setApp merges the container spec with the image's manifest.
func setApp(imgManifest *appcschema.ImageManifest, c *api.Container, opts *kubecontainer.RunContainerOptions, ctx *api.SecurityContext, podCtx *api.PodSecurityContext) error {
app := imgManifest.App
// Set up Exec.
var command, args []string
cmd, ok := imgManifest.Annotations.Get(appcDockerEntrypoint)
if ok {
err := json.Unmarshal([]byte(cmd), &command)
if err != nil {
return fmt.Errorf("cannot unmarshal ENTRYPOINT %q: %v", cmd, err)
}
}
ag, ok := imgManifest.Annotations.Get(appcDockerCmd)
if ok {
err := json.Unmarshal([]byte(ag), &args)
if err != nil {
return fmt.Errorf("cannot unmarshal CMD %q: %v", ag, err)
}
}
userCommand, userArgs := kubecontainer.ExpandContainerCommandAndArgs(c, opts.Envs)
if len(userCommand) > 0 {
command = userCommand
args = nil // If 'command' is specified, then drop the default args.
}
if len(userArgs) > 0 {
args = userArgs
}
exec := append(command, args...)
if len(exec) > 0 {
app.Exec = exec
}
// Set UID and GIDs.
if err := verifyNonRoot(app, ctx); err != nil {
return err
}
if ctx != nil && ctx.RunAsUser != nil {
app.User = strconv.Itoa(int(*ctx.RunAsUser))
}
setSupplementaryGIDs(app, podCtx)
// If 'User' or 'Group' are still empty at this point,
// then apply the root UID and GID.
// TODO(yifan): Instead of using root GID, we should use
// the GID which the user is in.
if app.User == "" {
app.User = "******"
}
if app.Group == "" {
app.Group = "0"
}
// Set working directory.
if len(c.WorkingDir) > 0 {
app.WorkingDirectory = c.WorkingDir
}
// Notes that we don't create Mounts section in the pod manifest here,
// as Mounts will be automatically generated by rkt.
mergeMounts(app, opts.Mounts)
mergeEnv(app, opts.Envs)
mergePortMappings(app, opts.PortMappings)
return setIsolators(app, c, ctx)
}
func (r *runtime) buildHyperPod(pod *api.Pod, restartCount int, pullSecrets []api.Secret) ([]byte, error) {
// check and pull image
for _, c := range pod.Spec.Containers {
if err, _ := r.imagePuller.PullImage(pod, &c, pullSecrets); err != nil {
return nil, err
}
}
// build hyper volume spec
specMap := make(map[string]interface{})
volumes := make([]map[string]interface{}, 0, 1)
volumeMap, found := r.runtimeHelper.ListVolumesForPod(pod.UID)
if found {
// process rbd volume globally
for name, mounter := range volumeMap {
glog.V(4).Infof("Hyper: volume %s, path %s, meta %s", name, mounter.GetPath(), mounter.GetMetaData())
v := make(map[string]interface{})
v[KEY_NAME] = name
// Process rbd volume
metadata := mounter.GetMetaData()
if metadata != nil && metadata["volume_type"].(string) == "rbd" {
v[KEY_VOLUME_DRIVE] = metadata["volume_type"]
v["source"] = "rbd:" + metadata["name"].(string)
monitors := make([]string, 0, 1)
for _, host := range metadata["hosts"].([]interface{}) {
for _, port := range metadata["ports"].([]interface{}) {
monitors = append(monitors, fmt.Sprintf("%s:%s", host.(string), port.(string)))
}
}
v["option"] = map[string]interface{}{
"user": metadata["auth_username"],
"keyring": metadata["keyring"],
"monitors": monitors,
}
} else {
glog.V(4).Infof("Hyper: volume %s %s", name, mounter.GetPath())
v[KEY_VOLUME_DRIVE] = VOLUME_TYPE_VFS
v[KEY_VOLUME_SOURCE] = mounter.GetPath()
}
volumes = append(volumes, v)
}
glog.V(4).Infof("Hyper volumes: %v", volumes)
}
if !r.disableHyperInternalService {
services := r.buildHyperPodServices(pod)
if services == nil {
// services can't be null for kubernetes, so fake one if it is null
services = []grpctypes.UserService{
{
ServiceIP: "127.0.0.2",
ServicePort: 65534,
},
}
}
specMap["services"] = services
}
// build hyper containers spec
var containers []map[string]interface{}
var k8sHostNeeded = true
dnsServers := make(map[string]string)
terminationMsgLabels := make(map[string]string)
for _, container := range pod.Spec.Containers {
c := make(map[string]interface{})
c[KEY_NAME] = r.buildHyperContainerFullName(
string(pod.UID),
string(pod.Name),
string(pod.Namespace),
container.Name,
restartCount,
container)
c[KEY_IMAGE] = container.Image
c[KEY_TTY] = container.TTY
if container.WorkingDir != "" {
c[KEY_WORKDIR] = container.WorkingDir
}
opts, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &container, "")
if err != nil {
return nil, err
}
command, args := kubecontainer.ExpandContainerCommandAndArgs(&container, opts.Envs)
if len(command) > 0 {
c[KEY_ENTRYPOINT] = command
}
if len(args) > 0 {
c[KEY_COMMAND] = args
}
// dns
for _, dns := range opts.DNS {
dnsServers[dns] = dns
}
// envs
envs := make([]map[string]string, 0, 1)
for _, e := range opts.Envs {
envs = append(envs, map[string]string{
"env": e.Name,
"value": e.Value,
})
}
c[KEY_ENVS] = envs
// port-mappings
var ports []map[string]interface{}
for _, mapping := range opts.PortMappings {
p := make(map[string]interface{})
p[KEY_CONTAINER_PORT] = mapping.ContainerPort
if mapping.HostPort != 0 {
p[KEY_HOST_PORT] = mapping.HostPort
}
p[KEY_PROTOCOL] = mapping.Protocol
ports = append(ports, p)
}
c[KEY_PORTS] = ports
// NOTE: PodContainerDir is from TerminationMessagePath, TerminationMessagePath is default to /dev/termination-log
if opts.PodContainerDir != "" && container.TerminationMessagePath != "" {
// In docker runtime, the container log path contains the container ID.
// However, for hyper runtime, we cannot get the container ID before the
// the container is launched, so here we generate a random uuid to enable
// us to map a container's termination message path to an unique log file
// on the disk.
randomUID := util.NewUUID()
containerLogPath := path.Join(opts.PodContainerDir, string(randomUID))
fs, err := os.Create(containerLogPath)
if err != nil {
return nil, err
}
if err := fs.Close(); err != nil {
return nil, err
}
mnt := &kubecontainer.Mount{
// Use a random name for the termination message mount, so that
// when a container restarts, it will not overwrite the old termination
// message.
Name: fmt.Sprintf("termination-message-%s", randomUID),
ContainerPath: container.TerminationMessagePath,
HostPath: containerLogPath,
ReadOnly: false,
}
opts.Mounts = append(opts.Mounts, *mnt)
// set termination msg labels with host path
terminationMsgLabels[container.Name] = mnt.HostPath
}
// volumes
if len(opts.Mounts) > 0 {
var containerVolumes []map[string]interface{}
for _, volume := range opts.Mounts {
v := make(map[string]interface{})
v[KEY_MOUNTPATH] = volume.ContainerPath
v[KEY_VOLUME] = volume.Name
v[KEY_READONLY] = volume.ReadOnly
containerVolumes = append(containerVolumes, v)
if k8sHostNeeded {
// Setup global hosts volume
if volume.Name == "k8s-managed-etc-hosts" {
k8sHostNeeded = false
volumes = append(volumes, map[string]interface{}{
KEY_NAME: volume.Name,
KEY_VOLUME_DRIVE: VOLUME_TYPE_VFS,
KEY_VOLUME_SOURCE: volume.HostPath,
})
}
// Setup global termination msg volume
if strings.HasPrefix(volume.Name, "termination-message") {
k8sHostNeeded = false
volumes = append(volumes, map[string]interface{}{
KEY_NAME: volume.Name,
KEY_VOLUME_DRIVE: VOLUME_TYPE_VFS,
KEY_VOLUME_SOURCE: volume.HostPath,
})
}
}
}
c[KEY_VOLUMES] = containerVolumes
}
containers = append(containers, c)
}
specMap[KEY_CONTAINERS] = containers
specMap[KEY_VOLUMES] = volumes
// dns
if len(dnsServers) > 0 {
dns := []string{}
for d := range dnsServers {
dns = append(dns, d)
}
specMap[KEY_DNS] = dns
}
// build hyper pod resources spec
var podCPULimit, podMemLimit int64
var labels map[string]string
podResource := make(map[string]int64)
for _, container := range pod.Spec.Containers {
resource := container.Resources.Limits
var containerCPULimit, containerMemLimit int64
for name, limit := range resource {
switch name {
case api.ResourceCPU:
containerCPULimit = limit.MilliValue()
case api.ResourceMemory:
containerMemLimit = limit.MilliValue()
}
}
if containerCPULimit == 0 {
containerCPULimit = hyperDefaultContainerCPU
}
if containerMemLimit == 0 {
containerMemLimit = hyperDefaultContainerMem * 1024 * 1024 * 1000
}
podCPULimit += containerCPULimit
podMemLimit += containerMemLimit
// generate heapster needed labels
// TODO: keep these labels up to date if the pod changes
labels = newLabels(&container, pod, restartCount, false)
}
podResource[KEY_VCPU] = (podCPULimit + 999) / 1000
podResource[KEY_MEMORY] = ((podMemLimit) / 1000 / 1024) / 1024
specMap[KEY_RESOURCE] = podResource
glog.V(5).Infof("Hyper: pod limit vcpu=%v mem=%vMiB", podResource[KEY_VCPU], podResource[KEY_MEMORY])
// Setup labels
podLabels := map[string]string{KEY_API_POD_UID: string(pod.UID)}
for k, v := range pod.Labels {
podLabels[k] = v
}
// append heapster needed labels
// NOTE(harryz): this only works for one pod one container model for now.
for k, v := range labels {
podLabels[k] = v
}
// append termination message label
for k, v := range terminationMsgLabels {
podLabels[k] = v
}
specMap[KEY_LABELS] = podLabels
// other params required
specMap[KEY_ID] = kubecontainer.BuildPodFullName(pod.Name, pod.Namespace)
// Cap hostname at 63 chars (specification is 64bytes which is 63 chars and the null terminating char).
const hostnameMaxLen = 63
podHostname := pod.Name
if len(podHostname) > hostnameMaxLen {
podHostname = podHostname[:hostnameMaxLen]
}
specMap[KEY_HOSTNAME] = podHostname
podData, err := json.Marshal(specMap)
if err != nil {
return nil, err
}
return podData, nil
}
func (r *runtime) buildHyperPod(pod *api.Pod, restartCount int, pullSecrets []api.Secret) ([]byte, error) {
// check and pull image
for _, c := range pod.Spec.Containers {
if err, _ := r.imagePuller.PullImage(pod, &c, pullSecrets); err != nil {
return nil, err
}
}
// build hyper volume spec
specMap := make(map[string]interface{})
volumeMap, ok := r.volumeGetter.GetVolumes(pod.UID)
if !ok {
return nil, fmt.Errorf("cannot get the volumes for pod %q", kubecontainer.GetPodFullName(pod))
}
volumes := make([]map[string]interface{}, 0, 1)
for name, volume := range volumeMap {
glog.V(4).Infof("Hyper: volume %s, path %s, meta %s", name, volume.Builder.GetPath(), volume.Builder.GetMetaData())
v := make(map[string]interface{})
v[KEY_NAME] = name
// Process rbd volume
metadata := volume.Builder.GetMetaData()
if metadata != nil && metadata["volume_type"].(string) == "rbd" {
v[KEY_VOLUME_DRIVE] = metadata["volume_type"]
v["source"] = "rbd:" + metadata["name"].(string)
monitors := make([]string, 0, 1)
for _, host := range metadata["hosts"].([]interface{}) {
for _, port := range metadata["ports"].([]interface{}) {
monitors = append(monitors, fmt.Sprintf("%s:%s", host.(string), port.(string)))
}
}
v["option"] = map[string]interface{}{
"user": metadata["auth_username"],
"keyring": metadata["keyring"],
"monitors": monitors,
}
} else {
glog.V(4).Infof("Hyper: volume %s %s", name, volume.Builder.GetPath())
v[KEY_VOLUME_DRIVE] = VOLUME_TYPE_VFS
v[KEY_VOLUME_SOURCE] = volume.Builder.GetPath()
}
volumes = append(volumes, v)
}
glog.V(4).Infof("Hyper volumes: %v", volumes)
if !r.disableHyperInternalService {
services := r.buildHyperPodServices(pod)
if services == nil {
// services can't be null for kubernetes, so fake one if it is null
services = []HyperService{
{
ServiceIP: "127.0.0.2",
ServicePort: 65534,
},
}
}
specMap["services"] = services
}
// build hyper containers spec
var containers []map[string]interface{}
var k8sHostNeeded = true
dnsServers := make(map[string]string)
for _, container := range pod.Spec.Containers {
c := make(map[string]interface{})
c[KEY_NAME] = r.buildHyperContainerFullName(
string(pod.UID),
string(pod.Name),
string(pod.Namespace),
container.Name,
restartCount,
container)
c[KEY_IMAGE] = container.Image
c[KEY_TTY] = container.TTY
if container.WorkingDir != "" {
c[KEY_WORKDIR] = container.WorkingDir
}
opts, err := r.generator.GenerateRunContainerOptions(pod, &container)
if err != nil {
return nil, err
}
command, args := kubecontainer.ExpandContainerCommandAndArgs(&container, opts.Envs)
if len(command) > 0 {
c[KEY_ENTRYPOINT] = command
}
if len(args) > 0 {
c[KEY_COMMAND] = args
}
// dns
for _, dns := range opts.DNS {
dnsServers[dns] = dns
}
// envs
envs := make([]map[string]string, 0, 1)
for _, e := range opts.Envs {
envs = append(envs, map[string]string{
"env": e.Name,
"value": e.Value,
})
}
c[KEY_ENVS] = envs
// port-mappings
var ports []map[string]interface{}
for _, mapping := range opts.PortMappings {
p := make(map[string]interface{})
p[KEY_CONTAINER_PORT] = mapping.ContainerPort
if mapping.HostPort != 0 {
p[KEY_HOST_PORT] = mapping.HostPort
}
p[KEY_PROTOCOL] = mapping.Protocol
ports = append(ports, p)
}
c[KEY_PORTS] = ports
// volumes
if len(opts.Mounts) > 0 {
var containerVolumes []map[string]interface{}
for _, volume := range opts.Mounts {
v := make(map[string]interface{})
v[KEY_MOUNTPATH] = volume.ContainerPath
v[KEY_VOLUME] = volume.Name
v[KEY_READONLY] = volume.ReadOnly
containerVolumes = append(containerVolumes, v)
// Setup global hosts volume
if volume.Name == "k8s-managed-etc-hosts" && k8sHostNeeded {
k8sHostNeeded = false
volumes = append(volumes, map[string]interface{}{
KEY_NAME: volume.Name,
KEY_VOLUME_DRIVE: VOLUME_TYPE_VFS,
KEY_VOLUME_SOURCE: volume.HostPath,
})
}
}
c[KEY_VOLUMES] = containerVolumes
}
containers = append(containers, c)
}
specMap[KEY_CONTAINERS] = containers
specMap[KEY_VOLUMES] = volumes
// dns
if len(dnsServers) > 0 {
dns := []string{}
for d := range dnsServers {
dns = append(dns, d)
}
specMap[KEY_DNS] = dns
}
// build hyper pod resources spec
var podCPULimit, podMemLimit int64
podResource := make(map[string]int64)
for _, container := range pod.Spec.Containers {
resource := container.Resources.Limits
var containerCPULimit, containerMemLimit int64
for name, limit := range resource {
switch name {
case api.ResourceCPU:
containerCPULimit = limit.MilliValue()
case api.ResourceMemory:
containerMemLimit = limit.MilliValue()
}
}
if containerCPULimit == 0 {
containerCPULimit = hyperDefaultContainerCPU
}
if containerMemLimit == 0 {
containerMemLimit = hyperDefaultContainerMem * 1024 * 1024 * 1000
}
podCPULimit += containerCPULimit
podMemLimit += containerMemLimit
}
podResource[KEY_VCPU] = (podCPULimit + 999) / 1000
podResource[KEY_MEMORY] = ((podMemLimit) / 1000 / 1024) / 1024
specMap[KEY_RESOURCE] = podResource
glog.V(5).Infof("Hyper: pod limit vcpu=%v mem=%vMiB", podResource[KEY_VCPU], podResource[KEY_MEMORY])
// Setup labels
podLabels := map[string]string{KEY_API_POD_UID: string(pod.UID)}
for k, v := range pod.Labels {
podLabels[k] = v
}
specMap[KEY_LABELS] = podLabels
// other params required
specMap[KEY_ID] = kubecontainer.BuildPodFullName(pod.Name, pod.Namespace)
specMap[KEY_TTY] = false
// Cap hostname at 63 chars (specification is 64bytes which is 63 chars and the null terminating char).
const hostnameMaxLen = 63
podHostname := pod.Name
if len(podHostname) > hostnameMaxLen {
podHostname = podHostname[:hostnameMaxLen]
}
specMap[KEY_HOSTNAME] = podHostname
podData, err := json.Marshal(specMap)
if err != nil {
return nil, err
}
return podData, nil
}