// createNetworkContainer starts the network container for a pod. Returns the docker container ID of the newly created container.
func (kl *Kubelet) createNetworkContainer(pod *api.BoundPod) (dockertools.DockerID, error) {
var ports []api.Port
// Docker only exports ports from the network container. Let's
// collect all of the relevant ports and export them.
for _, container := range pod.Spec.Containers {
ports = append(ports, container.Ports...)
}
container := &api.Container{
Name: networkContainerName,
Image: kl.networkContainerImage,
Ports: ports,
}
ref, err := containerRef(pod, container)
if err != nil {
glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err)
}
// TODO: make this a TTL based pull (if image older than X policy, pull)
ok, err := kl.dockerPuller.IsImagePresent(container.Image)
if err != nil {
if ref != nil {
record.Eventf(ref, "failed", "failed", "Failed to inspect image %q", container.Image)
}
return "", err
}
if !ok {
if err := kl.pullImage(container.Image, ref); err != nil {
return "", err
}
}
if ref != nil {
record.Eventf(ref, "waiting", "pulled", "Successfully pulled image %q", container.Image)
}
return kl.runContainer(pod, container, nil, "")
}
func (kl *Kubelet) pullImage(img string, ref *api.ObjectReference) error {
kl.pullLock.RLock()
defer kl.pullLock.RUnlock()
if err := kl.dockerPuller.Pull(img); err != nil {
if ref != nil {
record.Eventf(ref, "failed", "failed", "Failed to pull image %q", img)
}
return err
}
if ref != nil {
record.Eventf(ref, "waiting", "pulled", "Successfully pulled image %q", img)
}
return nil
}
func filterInvalidPods(pods []api.BoundPod, source string) (filtered []*api.BoundPod) {
names := util.StringSet{}
for i := range pods {
pod := &pods[i]
var errlist []error
if errs := validation.ValidateBoundPod(pod); len(errs) != 0 {
errlist = append(errlist, errs...)
// If validation fails, don't trust it any further -
// even Name could be bad.
} else {
name := podUniqueName(pod)
if names.Has(name) {
errlist = append(errlist, apierrs.NewFieldDuplicate("name", pod.Name))
} else {
names.Insert(name)
}
}
if len(errlist) > 0 {
name := bestPodIdentString(pod)
err := utilerrors.NewAggregate(errlist)
glog.Warningf("Pod[%d] (%s) from %s failed validation, ignoring: %v", i+1, name, source, err)
record.Eventf(pod, "failedValidation", "Error validating pod %s from %s, ignoring: %v", name, source, err)
continue
}
filtered = append(filtered, pod)
}
return
}
// BirthCry sends an event that the kubelet has started up.
func (kl *Kubelet) BirthCry() {
// Make an event that kubelet restarted.
// TODO: get the real minion object of ourself,
// and use the real minion name and UID.
ref := &api.ObjectReference{
Kind: "Minion",
Name: kl.hostname,
UID: kl.hostname,
}
record.Eventf(ref, "", "starting", "Starting kubelet.")
}
func (s *Scheduler) scheduleOne() {
pod := s.config.NextPod()
glog.V(3).Infof("Attempting to schedule: %v", pod)
dest, err := s.config.Algorithm.Schedule(*pod, s.config.MinionLister)
if err != nil {
glog.Warningf("Failed to schedule pod (times:%d): %v, err:%v", pod.Status.SchedulerFailureCount, pod, err)
if pod.Status.SchedulerFailureCount < s.config.MaxRetryTimes {
s.config.Error(pod, err)
} else {
record.Eventf(pod, string(api.PodPending), "failedScheduling", "Error scheduling: %v", err)
pod.Status.Phase = api.PodFailed
if err := s.config.Status.UpdatePodStatus(pod); err != nil {
glog.Warningf("Failed to update pod (%s): %v", pod.Name, err)
}
}
pod.Status.SchedulerFailureCount++
return
}
b := &api.Binding{
ObjectMeta: api.ObjectMeta{Namespace: pod.Namespace, Annotations: pod.Annotations},
PodID: pod.Name,
Host: dest.Name,
Network: dest.Network,
CpuSet: dest.CpuSet,
}
if err := s.config.Binder.Bind(b); err != nil {
glog.Warningf("Failed to bind pod: %v", err)
record.Eventf(pod, string(api.PodPending), "failedScheduling", "Binding rejected: %v", err)
s.config.Error(pod, err)
return
}
// check pod has scheduled
for {
ok, _ := s.config.Algorithm.CheckScheduledPod(pod.Name)
if ok {
break
}
time.Sleep(10 * time.Millisecond)
}
record.Eventf(pod, string(api.PodPending), "scheduled", "Successfully assigned %v to %#v", pod.Name, dest)
}
func (s *Scheduler) scheduleOne() {
pod := s.config.NextPod()
glog.V(3).Infof("Attempting to schedule: %v", pod)
dest, err := s.config.Algorithm.Schedule(*pod, s.config.MinionLister)
if err != nil {
glog.V(1).Infof("Failed to schedule: %v", pod)
record.Eventf(pod, string(api.PodPending), "failedScheduling", "Error scheduling: %v", err)
s.config.Error(pod, err)
return
}
b := &api.Binding{
ObjectMeta: api.ObjectMeta{Namespace: pod.Namespace},
PodID: pod.Name,
Host: dest,
}
if err := s.config.Binder.Bind(b); err != nil {
glog.V(1).Infof("Failed to bind pod: %v", err)
record.Eventf(pod, string(api.PodPending), "failedScheduling", "Binding rejected: %v", err)
s.config.Error(pod, err)
return
}
record.Eventf(pod, string(api.PodPending), "scheduled", "Successfully assigned %v to %v", pod.Name, dest)
}
func (kl *Kubelet) killContainerByID(ID, name string) error {
glog.V(2).Infof("Killing container with id %q and name %q", ID, name)
err := kl.dockerClient.StopContainer(ID, 10)
if len(name) == 0 {
return err
}
ref, ok := kl.getRef(dockertools.DockerID(ID))
if !ok {
glog.Warningf("No ref for pod '%v' - '%v'", ID, name)
} else {
// TODO: pass reason down here, and state, or move this call up the stack.
record.Eventf(ref, "terminated", "killing", "Killing %v - %v", ID, name)
}
return err
}
func (kl *Kubelet) syncPod(pod *api.BoundPod, dockerContainers dockertools.DockerContainers) error {
podFullName := GetPodFullName(pod)
uuid := pod.UID
containersToKeep := make(map[dockertools.DockerID]empty)
killedContainers := make(map[dockertools.DockerID]empty)
glog.V(4).Infof("Syncing Pod, podFullName: %q, uuid: %q", podFullName, uuid)
// Make sure we have a network container
var netID dockertools.DockerID
if netDockerContainer, found, _ := dockerContainers.FindPodContainer(podFullName, uuid, networkContainerName); found {
netID = dockertools.DockerID(netDockerContainer.ID)
} else {
glog.V(2).Infof("Network container doesn't exist for pod %q, killing and re-creating the pod", podFullName)
count, err := kl.killContainersInPod(pod, dockerContainers)
if err != nil {
return err
}
netID, err = kl.createNetworkContainer(pod)
if err != nil {
glog.Errorf("Failed to introspect network container: %v; Skipping pod %q", err, podFullName)
return err
}
if count > 0 {
// Re-list everything, otherwise we'll think we're ok.
dockerContainers, err = dockertools.GetKubeletDockerContainers(kl.dockerClient, false)
if err != nil {
glog.Errorf("Error listing containers %#v", dockerContainers)
return err
}
}
}
containersToKeep[netID] = empty{}
podVolumes, err := kl.mountExternalVolumes(pod)
if err != nil {
glog.Errorf("Unable to mount volumes for pod %q: %v; skipping pod", podFullName, err)
return err
}
podStatus := api.PodStatus{}
info, err := kl.GetPodInfo(podFullName, uuid)
if err != nil {
glog.Errorf("Unable to get pod with name %q and uuid %q info, health checks may be invalid", podFullName, uuid)
}
netInfo, found := info[networkContainerName]
if found {
podStatus.PodIP = netInfo.PodIP
}
for _, container := range pod.Spec.Containers {
expectedHash := dockertools.HashContainer(&container)
if dockerContainer, found, hash := dockerContainers.FindPodContainer(podFullName, uuid, container.Name); found {
containerID := dockertools.DockerID(dockerContainer.ID)
glog.V(3).Infof("pod %q container %q exists as %v", podFullName, container.Name, containerID)
// look for changes in the container.
if hash == 0 || hash == expectedHash {
// TODO: This should probably be separated out into a separate goroutine.
healthy, err := kl.healthy(podFullName, uuid, podStatus, container, dockerContainer)
if err != nil {
glog.V(1).Infof("health check errored: %v", err)
containersToKeep[containerID] = empty{}
continue
}
if healthy == health.Healthy {
containersToKeep[containerID] = empty{}
continue
}
glog.V(1).Infof("pod %q container %q is unhealthy. Container will be killed and re-created.", podFullName, container.Name, healthy)
} else {
glog.V(1).Infof("pod %q container %q hash changed (%d vs %d). Container will be killed and re-created.", podFullName, container.Name, hash, expectedHash)
}
if err := kl.killContainer(dockerContainer); err != nil {
glog.V(1).Infof("Failed to kill container %q: %v", dockerContainer.ID, err)
continue
}
killedContainers[containerID] = empty{}
// Also kill associated network container
if netContainer, found, _ := dockerContainers.FindPodContainer(podFullName, uuid, networkContainerName); found {
if err := kl.killContainer(netContainer); err != nil {
glog.V(1).Infof("Failed to kill network container %q: %v", netContainer.ID, err)
continue
}
}
}
// Check RestartPolicy for container
recentContainers, err := dockertools.GetRecentDockerContainersWithNameAndUUID(kl.dockerClient, podFullName, uuid, container.Name)
if err != nil {
glog.Errorf("Error listing recent containers with name and uuid:%s--%s--%s", podFullName, uuid, container.Name)
// TODO(dawnchen): error handling here?
}
if len(recentContainers) > 0 && pod.Spec.RestartPolicy.Always == nil {
if pod.Spec.RestartPolicy.Never != nil {
glog.V(3).Infof("Already ran container with name %s--%s--%s, do nothing",
podFullName, uuid, container.Name)
continue
}
if pod.Spec.RestartPolicy.OnFailure != nil {
// Check the exit code of last run
if recentContainers[0].State.ExitCode == 0 {
glog.V(3).Infof("Already successfully ran container with name %s--%s--%s, do nothing",
podFullName, uuid, container.Name)
continue
}
}
}
glog.V(3).Infof("Container with name %s--%s--%s doesn't exist, creating %#v", podFullName, uuid, container.Name, container)
ref, err := containerRef(pod, &container)
if err != nil {
glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err)
}
if !api.IsPullNever(container.ImagePullPolicy) {
present, err := kl.dockerPuller.IsImagePresent(container.Image)
latest := dockertools.RequireLatestImage(container.Image)
if err != nil {
if ref != nil {
record.Eventf(ref, "failed", "failed", "Failed to inspect image %q", container.Image)
}
glog.Errorf("Failed to inspect image %q: %v; skipping pod %q container %q", container.Image, err, podFullName, container.Name)
continue
}
if api.IsPullAlways(container.ImagePullPolicy) ||
(api.IsPullIfNotPresent(container.ImagePullPolicy) && (!present || latest)) {
if err := kl.pullImage(container.Image, ref); err != nil {
continue
}
}
}
// TODO(dawnchen): Check RestartPolicy.DelaySeconds before restart a container
containerID, err := kl.runContainer(pod, &container, podVolumes, "container:"+string(netID))
if err != nil {
// TODO(bburns) : Perhaps blacklist a container after N failures?
glog.Errorf("Error running pod %q container %q: %v", podFullName, container.Name, err)
continue
}
containersToKeep[containerID] = empty{}
}
// Kill any containers in this pod which were not identified above (guards against duplicates).
for id, container := range dockerContainers {
curPodFullName, curUUID, _, _ := dockertools.ParseDockerName(container.Names[0])
if curPodFullName == podFullName && curUUID == uuid {
// Don't kill containers we want to keep or those we already killed.
_, keep := containersToKeep[id]
_, killed := killedContainers[id]
if !keep && !killed {
glog.V(1).Infof("Killing unwanted container in pod %q: %+v", curUUID, container)
err = kl.killContainer(container)
if err != nil {
glog.Errorf("Error killing container: %v", err)
}
}
}
}
return nil
}
// Run a single container from a pod. Returns the docker container ID
func (kl *Kubelet) runContainer(pod *api.BoundPod, container *api.Container, podVolumes volumeMap, netMode string) (id dockertools.DockerID, err error) {
ref, err := containerRef(pod, container)
if err != nil {
glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err)
}
envVariables := makeEnvironmentVariables(container)
binds := makeBinds(pod, container, podVolumes)
exposedPorts, portBindings := makePortsAndBindings(container)
opts := docker.CreateContainerOptions{
Name: dockertools.BuildDockerName(pod.UID, GetPodFullName(pod), container),
Config: &docker.Config{
Cmd: container.Command,
Env: envVariables,
ExposedPorts: exposedPorts,
Hostname: pod.Name,
Image: container.Image,
Memory: container.Memory.Value(),
CPUShares: milliCPUToShares(container.CPU.MilliValue()),
WorkingDir: container.WorkingDir,
},
}
dockerContainer, err := kl.dockerClient.CreateContainer(opts)
if err != nil {
if ref != nil {
record.Eventf(ref, "failed", "failed",
"Failed to create docker container with error: %v", err)
}
return "", err
}
// Remember this reference so we can report events about this container
if ref != nil {
kl.setRef(dockertools.DockerID(dockerContainer.ID), ref)
record.Eventf(ref, "waiting", "created", "Created with docker id %v", dockerContainer.ID)
}
if len(container.TerminationMessagePath) != 0 {
p := kl.GetPodContainerDir(pod.UID, container.Name)
if err := os.MkdirAll(p, 0750); err != nil {
glog.Errorf("Error on creating %q: %v", p, err)
} else {
containerLogPath := path.Join(p, dockerContainer.ID)
fs, err := os.Create(containerLogPath)
if err != nil {
glog.Errorf("Error on creating termination-log file %q: %v", containerLogPath, err)
}
defer fs.Close()
b := fmt.Sprintf("%s:%s", containerLogPath, container.TerminationMessagePath)
binds = append(binds, b)
}
}
privileged := false
if capabilities.Get().AllowPrivileged {
privileged = container.Privileged
} else if container.Privileged {
return "", fmt.Errorf("container requested privileged mode, but it is disallowed globally.")
}
hc := &docker.HostConfig{
PortBindings: portBindings,
Binds: binds,
NetworkMode: netMode,
Privileged: privileged,
}
if pod.Spec.DNSPolicy == api.DNSClusterFirst {
if err := kl.applyClusterDNS(hc, pod); err != nil {
return "", err
}
}
err = kl.dockerClient.StartContainer(dockerContainer.ID, hc)
if err != nil {
if ref != nil {
record.Eventf(ref, "failed", "failed",
"Failed to start with docker id %v with error: %v", dockerContainer.ID, err)
}
return "", err
}
if ref != nil {
record.Eventf(ref, "running", "started", "Started with docker id %v", dockerContainer.ID)
}
if container.Lifecycle != nil && container.Lifecycle.PostStart != nil {
handlerErr := kl.runHandler(GetPodFullName(pod), pod.UID, container, container.Lifecycle.PostStart)
if handlerErr != nil {
kl.killContainerByID(dockerContainer.ID, "")
return dockertools.DockerID(""), fmt.Errorf("failed to call event handler: %v", handlerErr)
}
}
return dockertools.DockerID(dockerContainer.ID), err
}
// SyncPods synchronizes the configured list of pods (desired state) with the host current state.
func (kl *Kubelet) SyncPods(pods []api.BoundPod) error {
glog.V(4).Infof("Desired: %#v", pods)
var err error
desiredContainers := make(map[podContainer]empty)
desiredPods := make(map[string]empty)
dockerContainers, err := dockertools.GetKubeletDockerContainers(kl.dockerClient, false)
if err != nil {
glog.Errorf("Error listing containers: %#v", dockerContainers)
return err
}
// Check for any containers that need starting
for ix := range pods {
pod := &pods[ix]
podFullName := GetPodFullName(pod)
uuid := pod.UID
desiredPods[uuid] = empty{}
// Add all containers (including net) to the map.
desiredContainers[podContainer{podFullName, uuid, networkContainerName}] = empty{}
for _, cont := range pod.Spec.Containers {
desiredContainers[podContainer{podFullName, uuid, cont.Name}] = empty{}
}
// Run the sync in an async manifest worker.
kl.podWorkers.Run(podFullName, func() {
if err := kl.syncPod(pod, dockerContainers); err != nil {
glog.Errorf("Error syncing pod, skipping: %v", err)
record.Eventf(pod, "", "failedSync", "Error syncing pod, skipping: %v", err)
}
})
}
if !kl.sourcesReady() {
// If the sources aren't ready, skip deletion, as we may accidentally delete pods
// for sources that haven't reported yet.
glog.V(4).Infof("Skipping deletes, sources aren't ready yet.")
return nil
}
// Kill any containers we don't need.
for _, container := range dockerContainers {
// Don't kill containers that are in the desired pods.
podFullName, uuid, containerName, _ := dockertools.ParseDockerName(container.Names[0])
if _, found := desiredPods[uuid]; found {
// syncPod() will handle this one.
continue
}
pc := podContainer{podFullName, uuid, containerName}
if _, ok := desiredContainers[pc]; !ok {
glog.V(1).Infof("Killing unwanted container %+v", pc)
err = kl.killContainer(container)
if err != nil {
glog.Errorf("Error killing container %+v: %v", pc, err)
}
}
}
// Remove any orphaned volumes.
kl.reconcileVolumes(pods)
return err
}
func TestEventf(t *testing.T) {
testPod := &api.Pod{
ObjectMeta: api.ObjectMeta{
SelfLink: "/api/v1beta1/pods/foo",
Name: "foo",
Namespace: "baz",
UID: "bar",
},
}
testRef, err := api.GetPartialReference(testPod, "desiredState.manifest.containers[2]")
if err != nil {
t.Fatal(err)
}
table := []struct {
obj runtime.Object
status, reason string
messageFmt string
elements []interface{}
expect *api.Event
expectLog string
}{
{
obj: testRef,
status: "running",
reason: "started",
messageFmt: "some verbose message: %v",
elements: []interface{}{1},
expect: &api.Event{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: "baz",
},
InvolvedObject: api.ObjectReference{
Kind: "Pod",
Name: "foo",
Namespace: "baz",
UID: "bar",
APIVersion: "v1beta1",
FieldPath: "desiredState.manifest.containers[2]",
},
Status: "running",
Reason: "started",
Message: "some verbose message: 1",
Source: "eventTest",
},
expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"v1beta1", ResourceVersion:"", FieldPath:"desiredState.manifest.containers[2]"}): status: 'running', reason: 'started' some verbose message: 1`,
},
{
obj: testPod,
status: "running",
reason: "started",
messageFmt: "some verbose message: %v",
elements: []interface{}{1},
expect: &api.Event{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: "baz",
},
InvolvedObject: api.ObjectReference{
Kind: "Pod",
Name: "foo",
Namespace: "baz",
UID: "bar",
APIVersion: "v1beta1",
},
Status: "running",
Reason: "started",
Message: "some verbose message: 1",
Source: "eventTest",
},
expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"v1beta1", ResourceVersion:"", FieldPath:""}): status: 'running', reason: 'started' some verbose message: 1`,
},
}
for _, item := range table {
called := make(chan struct{})
testEvents := testEventRecorder{
OnEvent: func(event *api.Event) (*api.Event, error) {
a := *event
// Just check that the timestamp was set.
if a.Timestamp.IsZero() {
t.Errorf("timestamp wasn't set")
}
a.Timestamp = item.expect.Timestamp
// Check that name has the right prefix.
if n, en := a.Name, item.expect.Name; !strings.HasPrefix(n, en) {
t.Errorf("Name '%v' does not contain prefix '%v'", n, en)
}
a.Name = item.expect.Name
if e, a := item.expect, &a; !reflect.DeepEqual(e, a) {
t.Errorf("diff: %s", util.ObjectDiff(e, a))
}
called <- struct{}{}
return event, nil
},
}
recorder := record.StartRecording(&testEvents, "eventTest")
logger := record.StartLogging(t.Logf) // Prove that it is useful
logger2 := record.StartLogging(func(formatter string, args ...interface{}) {
if e, a := item.expectLog, fmt.Sprintf(formatter, args...); e != a {
t.Errorf("Expected '%v', got '%v'", e, a)
}
called <- struct{}{}
})
record.Eventf(item.obj, item.status, item.reason, item.messageFmt, item.elements...)
<-called
<-called
recorder.Stop()
logger.Stop()
logger2.Stop()
}
}
