// createNetworkContainer starts the network container for a pod. Returns the docker container ID of the newly created container. func (kl *Kubelet) createNetworkContainer(pod *api.BoundPod) (dockertools.DockerID, error) { var ports []api.Port // Docker only exports ports from the network container. Let's // collect all of the relevant ports and export them. for _, container := range pod.Spec.Containers { ports = append(ports, container.Ports...) } container := &api.Container{ Name: networkContainerName, Image: kl.networkContainerImage, Ports: ports, } ref, err := containerRef(pod, container) if err != nil { glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err) } // TODO: make this a TTL based pull (if image older than X policy, pull) ok, err := kl.dockerPuller.IsImagePresent(container.Image) if err != nil { if ref != nil { record.Eventf(ref, "failed", "failed", "Failed to inspect image %q", container.Image) } return "", err } if !ok { if err := kl.pullImage(container.Image, ref); err != nil { return "", err } } if ref != nil { record.Eventf(ref, "waiting", "pulled", "Successfully pulled image %q", container.Image) } return kl.runContainer(pod, container, nil, "") }
func (kl *Kubelet) pullImage(img string, ref *api.ObjectReference) error { kl.pullLock.RLock() defer kl.pullLock.RUnlock() if err := kl.dockerPuller.Pull(img); err != nil { if ref != nil { record.Eventf(ref, "failed", "failed", "Failed to pull image %q", img) } return err } if ref != nil { record.Eventf(ref, "waiting", "pulled", "Successfully pulled image %q", img) } return nil }
func filterInvalidPods(pods []api.BoundPod, source string) (filtered []*api.BoundPod) { names := util.StringSet{} for i := range pods { pod := &pods[i] var errlist []error if errs := validation.ValidateBoundPod(pod); len(errs) != 0 { errlist = append(errlist, errs...) // If validation fails, don't trust it any further - // even Name could be bad. } else { name := podUniqueName(pod) if names.Has(name) { errlist = append(errlist, apierrs.NewFieldDuplicate("name", pod.Name)) } else { names.Insert(name) } } if len(errlist) > 0 { name := bestPodIdentString(pod) err := utilerrors.NewAggregate(errlist) glog.Warningf("Pod[%d] (%s) from %s failed validation, ignoring: %v", i+1, name, source, err) record.Eventf(pod, "failedValidation", "Error validating pod %s from %s, ignoring: %v", name, source, err) continue } filtered = append(filtered, pod) } return }
// BirthCry sends an event that the kubelet has started up. func (kl *Kubelet) BirthCry() { // Make an event that kubelet restarted. // TODO: get the real minion object of ourself, // and use the real minion name and UID. ref := &api.ObjectReference{ Kind: "Minion", Name: kl.hostname, UID: kl.hostname, } record.Eventf(ref, "", "starting", "Starting kubelet.") }
func (s *Scheduler) scheduleOne() { pod := s.config.NextPod() glog.V(3).Infof("Attempting to schedule: %v", pod) dest, err := s.config.Algorithm.Schedule(*pod, s.config.MinionLister) if err != nil { glog.Warningf("Failed to schedule pod (times:%d): %v, err:%v", pod.Status.SchedulerFailureCount, pod, err) if pod.Status.SchedulerFailureCount < s.config.MaxRetryTimes { s.config.Error(pod, err) } else { record.Eventf(pod, string(api.PodPending), "failedScheduling", "Error scheduling: %v", err) pod.Status.Phase = api.PodFailed if err := s.config.Status.UpdatePodStatus(pod); err != nil { glog.Warningf("Failed to update pod (%s): %v", pod.Name, err) } } pod.Status.SchedulerFailureCount++ return } b := &api.Binding{ ObjectMeta: api.ObjectMeta{Namespace: pod.Namespace, Annotations: pod.Annotations}, PodID: pod.Name, Host: dest.Name, Network: dest.Network, CpuSet: dest.CpuSet, } if err := s.config.Binder.Bind(b); err != nil { glog.Warningf("Failed to bind pod: %v", err) record.Eventf(pod, string(api.PodPending), "failedScheduling", "Binding rejected: %v", err) s.config.Error(pod, err) return } // check pod has scheduled for { ok, _ := s.config.Algorithm.CheckScheduledPod(pod.Name) if ok { break } time.Sleep(10 * time.Millisecond) } record.Eventf(pod, string(api.PodPending), "scheduled", "Successfully assigned %v to %#v", pod.Name, dest) }
func (s *Scheduler) scheduleOne() { pod := s.config.NextPod() glog.V(3).Infof("Attempting to schedule: %v", pod) dest, err := s.config.Algorithm.Schedule(*pod, s.config.MinionLister) if err != nil { glog.V(1).Infof("Failed to schedule: %v", pod) record.Eventf(pod, string(api.PodPending), "failedScheduling", "Error scheduling: %v", err) s.config.Error(pod, err) return } b := &api.Binding{ ObjectMeta: api.ObjectMeta{Namespace: pod.Namespace}, PodID: pod.Name, Host: dest, } if err := s.config.Binder.Bind(b); err != nil { glog.V(1).Infof("Failed to bind pod: %v", err) record.Eventf(pod, string(api.PodPending), "failedScheduling", "Binding rejected: %v", err) s.config.Error(pod, err) return } record.Eventf(pod, string(api.PodPending), "scheduled", "Successfully assigned %v to %v", pod.Name, dest) }
func (kl *Kubelet) killContainerByID(ID, name string) error { glog.V(2).Infof("Killing container with id %q and name %q", ID, name) err := kl.dockerClient.StopContainer(ID, 10) if len(name) == 0 { return err } ref, ok := kl.getRef(dockertools.DockerID(ID)) if !ok { glog.Warningf("No ref for pod '%v' - '%v'", ID, name) } else { // TODO: pass reason down here, and state, or move this call up the stack. record.Eventf(ref, "terminated", "killing", "Killing %v - %v", ID, name) } return err }
func (kl *Kubelet) syncPod(pod *api.BoundPod, dockerContainers dockertools.DockerContainers) error { podFullName := GetPodFullName(pod) uuid := pod.UID containersToKeep := make(map[dockertools.DockerID]empty) killedContainers := make(map[dockertools.DockerID]empty) glog.V(4).Infof("Syncing Pod, podFullName: %q, uuid: %q", podFullName, uuid) // Make sure we have a network container var netID dockertools.DockerID if netDockerContainer, found, _ := dockerContainers.FindPodContainer(podFullName, uuid, networkContainerName); found { netID = dockertools.DockerID(netDockerContainer.ID) } else { glog.V(2).Infof("Network container doesn't exist for pod %q, killing and re-creating the pod", podFullName) count, err := kl.killContainersInPod(pod, dockerContainers) if err != nil { return err } netID, err = kl.createNetworkContainer(pod) if err != nil { glog.Errorf("Failed to introspect network container: %v; Skipping pod %q", err, podFullName) return err } if count > 0 { // Re-list everything, otherwise we'll think we're ok. dockerContainers, err = dockertools.GetKubeletDockerContainers(kl.dockerClient, false) if err != nil { glog.Errorf("Error listing containers %#v", dockerContainers) return err } } } containersToKeep[netID] = empty{} podVolumes, err := kl.mountExternalVolumes(pod) if err != nil { glog.Errorf("Unable to mount volumes for pod %q: %v; skipping pod", podFullName, err) return err } podStatus := api.PodStatus{} info, err := kl.GetPodInfo(podFullName, uuid) if err != nil { glog.Errorf("Unable to get pod with name %q and uuid %q info, health checks may be invalid", podFullName, uuid) } netInfo, found := info[networkContainerName] if found { podStatus.PodIP = netInfo.PodIP } for _, container := range pod.Spec.Containers { expectedHash := dockertools.HashContainer(&container) if dockerContainer, found, hash := dockerContainers.FindPodContainer(podFullName, uuid, container.Name); found { containerID := dockertools.DockerID(dockerContainer.ID) glog.V(3).Infof("pod %q container %q exists as %v", podFullName, container.Name, containerID) // look for changes in the container. if hash == 0 || hash == expectedHash { // TODO: This should probably be separated out into a separate goroutine. healthy, err := kl.healthy(podFullName, uuid, podStatus, container, dockerContainer) if err != nil { glog.V(1).Infof("health check errored: %v", err) containersToKeep[containerID] = empty{} continue } if healthy == health.Healthy { containersToKeep[containerID] = empty{} continue } glog.V(1).Infof("pod %q container %q is unhealthy. Container will be killed and re-created.", podFullName, container.Name, healthy) } else { glog.V(1).Infof("pod %q container %q hash changed (%d vs %d). Container will be killed and re-created.", podFullName, container.Name, hash, expectedHash) } if err := kl.killContainer(dockerContainer); err != nil { glog.V(1).Infof("Failed to kill container %q: %v", dockerContainer.ID, err) continue } killedContainers[containerID] = empty{} // Also kill associated network container if netContainer, found, _ := dockerContainers.FindPodContainer(podFullName, uuid, networkContainerName); found { if err := kl.killContainer(netContainer); err != nil { glog.V(1).Infof("Failed to kill network container %q: %v", netContainer.ID, err) continue } } } // Check RestartPolicy for container recentContainers, err := dockertools.GetRecentDockerContainersWithNameAndUUID(kl.dockerClient, podFullName, uuid, container.Name) if err != nil { glog.Errorf("Error listing recent containers with name and uuid:%s--%s--%s", podFullName, uuid, container.Name) // TODO(dawnchen): error handling here? } if len(recentContainers) > 0 && pod.Spec.RestartPolicy.Always == nil { if pod.Spec.RestartPolicy.Never != nil { glog.V(3).Infof("Already ran container with name %s--%s--%s, do nothing", podFullName, uuid, container.Name) continue } if pod.Spec.RestartPolicy.OnFailure != nil { // Check the exit code of last run if recentContainers[0].State.ExitCode == 0 { glog.V(3).Infof("Already successfully ran container with name %s--%s--%s, do nothing", podFullName, uuid, container.Name) continue } } } glog.V(3).Infof("Container with name %s--%s--%s doesn't exist, creating %#v", podFullName, uuid, container.Name, container) ref, err := containerRef(pod, &container) if err != nil { glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err) } if !api.IsPullNever(container.ImagePullPolicy) { present, err := kl.dockerPuller.IsImagePresent(container.Image) latest := dockertools.RequireLatestImage(container.Image) if err != nil { if ref != nil { record.Eventf(ref, "failed", "failed", "Failed to inspect image %q", container.Image) } glog.Errorf("Failed to inspect image %q: %v; skipping pod %q container %q", container.Image, err, podFullName, container.Name) continue } if api.IsPullAlways(container.ImagePullPolicy) || (api.IsPullIfNotPresent(container.ImagePullPolicy) && (!present || latest)) { if err := kl.pullImage(container.Image, ref); err != nil { continue } } } // TODO(dawnchen): Check RestartPolicy.DelaySeconds before restart a container containerID, err := kl.runContainer(pod, &container, podVolumes, "container:"+string(netID)) if err != nil { // TODO(bburns) : Perhaps blacklist a container after N failures? glog.Errorf("Error running pod %q container %q: %v", podFullName, container.Name, err) continue } containersToKeep[containerID] = empty{} } // Kill any containers in this pod which were not identified above (guards against duplicates). for id, container := range dockerContainers { curPodFullName, curUUID, _, _ := dockertools.ParseDockerName(container.Names[0]) if curPodFullName == podFullName && curUUID == uuid { // Don't kill containers we want to keep or those we already killed. _, keep := containersToKeep[id] _, killed := killedContainers[id] if !keep && !killed { glog.V(1).Infof("Killing unwanted container in pod %q: %+v", curUUID, container) err = kl.killContainer(container) if err != nil { glog.Errorf("Error killing container: %v", err) } } } } return nil }
// Run a single container from a pod. Returns the docker container ID func (kl *Kubelet) runContainer(pod *api.BoundPod, container *api.Container, podVolumes volumeMap, netMode string) (id dockertools.DockerID, err error) { ref, err := containerRef(pod, container) if err != nil { glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err) } envVariables := makeEnvironmentVariables(container) binds := makeBinds(pod, container, podVolumes) exposedPorts, portBindings := makePortsAndBindings(container) opts := docker.CreateContainerOptions{ Name: dockertools.BuildDockerName(pod.UID, GetPodFullName(pod), container), Config: &docker.Config{ Cmd: container.Command, Env: envVariables, ExposedPorts: exposedPorts, Hostname: pod.Name, Image: container.Image, Memory: container.Memory.Value(), CPUShares: milliCPUToShares(container.CPU.MilliValue()), WorkingDir: container.WorkingDir, }, } dockerContainer, err := kl.dockerClient.CreateContainer(opts) if err != nil { if ref != nil { record.Eventf(ref, "failed", "failed", "Failed to create docker container with error: %v", err) } return "", err } // Remember this reference so we can report events about this container if ref != nil { kl.setRef(dockertools.DockerID(dockerContainer.ID), ref) record.Eventf(ref, "waiting", "created", "Created with docker id %v", dockerContainer.ID) } if len(container.TerminationMessagePath) != 0 { p := kl.GetPodContainerDir(pod.UID, container.Name) if err := os.MkdirAll(p, 0750); err != nil { glog.Errorf("Error on creating %q: %v", p, err) } else { containerLogPath := path.Join(p, dockerContainer.ID) fs, err := os.Create(containerLogPath) if err != nil { glog.Errorf("Error on creating termination-log file %q: %v", containerLogPath, err) } defer fs.Close() b := fmt.Sprintf("%s:%s", containerLogPath, container.TerminationMessagePath) binds = append(binds, b) } } privileged := false if capabilities.Get().AllowPrivileged { privileged = container.Privileged } else if container.Privileged { return "", fmt.Errorf("container requested privileged mode, but it is disallowed globally.") } hc := &docker.HostConfig{ PortBindings: portBindings, Binds: binds, NetworkMode: netMode, Privileged: privileged, } if pod.Spec.DNSPolicy == api.DNSClusterFirst { if err := kl.applyClusterDNS(hc, pod); err != nil { return "", err } } err = kl.dockerClient.StartContainer(dockerContainer.ID, hc) if err != nil { if ref != nil { record.Eventf(ref, "failed", "failed", "Failed to start with docker id %v with error: %v", dockerContainer.ID, err) } return "", err } if ref != nil { record.Eventf(ref, "running", "started", "Started with docker id %v", dockerContainer.ID) } if container.Lifecycle != nil && container.Lifecycle.PostStart != nil { handlerErr := kl.runHandler(GetPodFullName(pod), pod.UID, container, container.Lifecycle.PostStart) if handlerErr != nil { kl.killContainerByID(dockerContainer.ID, "") return dockertools.DockerID(""), fmt.Errorf("failed to call event handler: %v", handlerErr) } } return dockertools.DockerID(dockerContainer.ID), err }
// SyncPods synchronizes the configured list of pods (desired state) with the host current state. func (kl *Kubelet) SyncPods(pods []api.BoundPod) error { glog.V(4).Infof("Desired: %#v", pods) var err error desiredContainers := make(map[podContainer]empty) desiredPods := make(map[string]empty) dockerContainers, err := dockertools.GetKubeletDockerContainers(kl.dockerClient, false) if err != nil { glog.Errorf("Error listing containers: %#v", dockerContainers) return err } // Check for any containers that need starting for ix := range pods { pod := &pods[ix] podFullName := GetPodFullName(pod) uuid := pod.UID desiredPods[uuid] = empty{} // Add all containers (including net) to the map. desiredContainers[podContainer{podFullName, uuid, networkContainerName}] = empty{} for _, cont := range pod.Spec.Containers { desiredContainers[podContainer{podFullName, uuid, cont.Name}] = empty{} } // Run the sync in an async manifest worker. kl.podWorkers.Run(podFullName, func() { if err := kl.syncPod(pod, dockerContainers); err != nil { glog.Errorf("Error syncing pod, skipping: %v", err) record.Eventf(pod, "", "failedSync", "Error syncing pod, skipping: %v", err) } }) } if !kl.sourcesReady() { // If the sources aren't ready, skip deletion, as we may accidentally delete pods // for sources that haven't reported yet. glog.V(4).Infof("Skipping deletes, sources aren't ready yet.") return nil } // Kill any containers we don't need. for _, container := range dockerContainers { // Don't kill containers that are in the desired pods. podFullName, uuid, containerName, _ := dockertools.ParseDockerName(container.Names[0]) if _, found := desiredPods[uuid]; found { // syncPod() will handle this one. continue } pc := podContainer{podFullName, uuid, containerName} if _, ok := desiredContainers[pc]; !ok { glog.V(1).Infof("Killing unwanted container %+v", pc) err = kl.killContainer(container) if err != nil { glog.Errorf("Error killing container %+v: %v", pc, err) } } } // Remove any orphaned volumes. kl.reconcileVolumes(pods) return err }
func TestEventf(t *testing.T) { testPod := &api.Pod{ ObjectMeta: api.ObjectMeta{ SelfLink: "/api/v1beta1/pods/foo", Name: "foo", Namespace: "baz", UID: "bar", }, } testRef, err := api.GetPartialReference(testPod, "desiredState.manifest.containers[2]") if err != nil { t.Fatal(err) } table := []struct { obj runtime.Object status, reason string messageFmt string elements []interface{} expect *api.Event expectLog string }{ { obj: testRef, status: "running", reason: "started", messageFmt: "some verbose message: %v", elements: []interface{}{1}, expect: &api.Event{ ObjectMeta: api.ObjectMeta{ Name: "foo", Namespace: "baz", }, InvolvedObject: api.ObjectReference{ Kind: "Pod", Name: "foo", Namespace: "baz", UID: "bar", APIVersion: "v1beta1", FieldPath: "desiredState.manifest.containers[2]", }, Status: "running", Reason: "started", Message: "some verbose message: 1", Source: "eventTest", }, expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"v1beta1", ResourceVersion:"", FieldPath:"desiredState.manifest.containers[2]"}): status: 'running', reason: 'started' some verbose message: 1`, }, { obj: testPod, status: "running", reason: "started", messageFmt: "some verbose message: %v", elements: []interface{}{1}, expect: &api.Event{ ObjectMeta: api.ObjectMeta{ Name: "foo", Namespace: "baz", }, InvolvedObject: api.ObjectReference{ Kind: "Pod", Name: "foo", Namespace: "baz", UID: "bar", APIVersion: "v1beta1", }, Status: "running", Reason: "started", Message: "some verbose message: 1", Source: "eventTest", }, expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"v1beta1", ResourceVersion:"", FieldPath:""}): status: 'running', reason: 'started' some verbose message: 1`, }, } for _, item := range table { called := make(chan struct{}) testEvents := testEventRecorder{ OnEvent: func(event *api.Event) (*api.Event, error) { a := *event // Just check that the timestamp was set. if a.Timestamp.IsZero() { t.Errorf("timestamp wasn't set") } a.Timestamp = item.expect.Timestamp // Check that name has the right prefix. if n, en := a.Name, item.expect.Name; !strings.HasPrefix(n, en) { t.Errorf("Name '%v' does not contain prefix '%v'", n, en) } a.Name = item.expect.Name if e, a := item.expect, &a; !reflect.DeepEqual(e, a) { t.Errorf("diff: %s", util.ObjectDiff(e, a)) } called <- struct{}{} return event, nil }, } recorder := record.StartRecording(&testEvents, "eventTest") logger := record.StartLogging(t.Logf) // Prove that it is useful logger2 := record.StartLogging(func(formatter string, args ...interface{}) { if e, a := item.expectLog, fmt.Sprintf(formatter, args...); e != a { t.Errorf("Expected '%v', got '%v'", e, a) } called <- struct{}{} }) record.Eventf(item.obj, item.status, item.reason, item.messageFmt, item.elements...) <-called <-called recorder.Stop() logger.Stop() logger2.Stop() } }