func (s *statusManager) SetPodStatus(pod *api.Pod, status api.PodStatus) { podFullName := kubecontainer.GetPodFullName(pod) s.podStatusesLock.Lock() defer s.podStatusesLock.Unlock() oldStatus, found := s.podStatuses[podFullName] // ensure that the start time does not change across updates. if found && oldStatus.StartTime != nil { status.StartTime = oldStatus.StartTime } // if the status has no start time, we need to set an initial time // TODO(yujuhong): Consider setting StartTime when generating the pod // status instead, which would allow statusManager to become a simple cache // again. if status.StartTime.IsZero() { if pod.Status.StartTime.IsZero() { // the pod did not have a previously recorded value so set to now now := util.Now() status.StartTime = &now } else { // the pod had a recorded value, but the kubelet restarted so we need to rebuild cache // based on last observed value status.StartTime = pod.Status.StartTime } } if !found || !reflect.DeepEqual(oldStatus, status) { s.podStatuses[podFullName] = status s.podStatusChannel <- podStatusSyncRequest{pod, status} } else { glog.V(3).Infof("Ignoring same pod status for %s - old: %s new: %s", podFullName, oldStatus, status) } }
// toPodStatus converts a podInfo type into an api.PodStatus type. func (p *podInfo) toPodStatus(pod *kubecontainer.Pod) api.PodStatus { var status api.PodStatus status.PodIP = p.ip // For now just make every container's state the same as the pod. for _, container := range pod.Containers { status.ContainerStatuses = append(status.ContainerStatuses, p.getContainerStatus(container)) } return status }
// GetPodStatus returns information from Docker about the containers in a pod func (kl *Kubelet) GetPodStatus(podFullName string, uid types.UID) (api.PodStatus, error) { var manifest api.PodSpec for _, pod := range kl.pods { if GetPodFullName(&pod) == podFullName { manifest = pod.Spec break } } info, err := dockertools.GetDockerPodInfo(kl.dockerClient, manifest, podFullName, uid) // TODO(dchen1107): Determine PodPhase here var podStatus api.PodStatus podStatus.Info = info return podStatus, err }
func (s *statusManager) SetPodStatus(pod *api.Pod, status api.PodStatus) { podFullName := kubecontainer.GetPodFullName(pod) s.podStatusesLock.Lock() defer s.podStatusesLock.Unlock() oldStatus, found := s.podStatuses[podFullName] // ensure that the start time does not change across updates. if found && oldStatus.StartTime != nil { status.StartTime = oldStatus.StartTime } // if the status has no start time, we need to set an initial time // TODO(yujuhong): Consider setting StartTime when generating the pod // status instead, which would allow statusManager to become a simple cache // again. if status.StartTime.IsZero() { if pod.Status.StartTime.IsZero() { // the pod did not have a previously recorded value so set to now now := util.Now() status.StartTime = &now } else { // the pod had a recorded value, but the kubelet restarted so we need to rebuild cache // based on last observed value status.StartTime = pod.Status.StartTime } } // TODO: Holding a lock during blocking operations is dangerous. Refactor so this isn't necessary. // The intent here is to prevent concurrent updates to a pod's status from // clobbering each other so the phase of a pod progresses monotonically. // Currently this routine is not called for the same pod from multiple // workers and/or the kubelet but dropping the lock before sending the // status down the channel feels like an easy way to get a bullet in foot. if !found || !isStatusEqual(&oldStatus, &status) { s.podStatuses[podFullName] = status s.podStatusChannel <- podStatusSyncRequest{pod, status} } else { glog.V(3).Infof("Ignoring same pod status for %s - old: %s new: %s", podFullName, oldStatus, status) } }
func (kl *Kubelet) syncPod(pod *api.BoundPod, dockerContainers dockertools.DockerContainers) error { podFullName := GetPodFullName(pod) uuid := pod.UID containersToKeep := make(map[dockertools.DockerID]empty) killedContainers := make(map[dockertools.DockerID]empty) glog.V(4).Infof("Syncing Pod, podFullName: %q, uuid: %q", podFullName, uuid) // Make sure we have a network container var netID dockertools.DockerID if netDockerContainer, found, _ := dockerContainers.FindPodContainer(podFullName, uuid, networkContainerName); found { netID = dockertools.DockerID(netDockerContainer.ID) } else { glog.V(2).Infof("Network container doesn't exist for pod %q, killing and re-creating the pod", podFullName) count, err := kl.killContainersInPod(pod, dockerContainers) if err != nil { return err } netID, err = kl.createNetworkContainer(pod) if err != nil { glog.Errorf("Failed to introspect network container: %v; Skipping pod %q", err, podFullName) return err } if count > 0 { // Re-list everything, otherwise we'll think we're ok. dockerContainers, err = dockertools.GetKubeletDockerContainers(kl.dockerClient, false) if err != nil { glog.Errorf("Error listing containers %#v", dockerContainers) return err } } } containersToKeep[netID] = empty{} podVolumes, err := kl.mountExternalVolumes(pod) if err != nil { glog.Errorf("Unable to mount volumes for pod %q: %v; skipping pod", podFullName, err) return err } podStatus := api.PodStatus{} info, err := kl.GetPodInfo(podFullName, uuid) if err != nil { glog.Errorf("Unable to get pod with name %q and uuid %q info, health checks may be invalid", podFullName, uuid) } netInfo, found := info[networkContainerName] if found { podStatus.PodIP = netInfo.PodIP } for _, container := range pod.Spec.Containers { expectedHash := dockertools.HashContainer(&container) if dockerContainer, found, hash := dockerContainers.FindPodContainer(podFullName, uuid, container.Name); found { containerID := dockertools.DockerID(dockerContainer.ID) glog.V(3).Infof("pod %q container %q exists as %v", podFullName, container.Name, containerID) // look for changes in the container. if hash == 0 || hash == expectedHash { // TODO: This should probably be separated out into a separate goroutine. healthy, err := kl.healthy(podFullName, uuid, podStatus, container, dockerContainer) if err != nil { glog.V(1).Infof("health check errored: %v", err) containersToKeep[containerID] = empty{} continue } if healthy == health.Healthy { containersToKeep[containerID] = empty{} continue } glog.V(1).Infof("pod %q container %q is unhealthy. Container will be killed and re-created.", podFullName, container.Name, healthy) } else { glog.V(1).Infof("pod %q container %q hash changed (%d vs %d). Container will be killed and re-created.", podFullName, container.Name, hash, expectedHash) } if err := kl.killContainer(dockerContainer); err != nil { glog.V(1).Infof("Failed to kill container %q: %v", dockerContainer.ID, err) continue } killedContainers[containerID] = empty{} // Also kill associated network container if netContainer, found, _ := dockerContainers.FindPodContainer(podFullName, uuid, networkContainerName); found { if err := kl.killContainer(netContainer); err != nil { glog.V(1).Infof("Failed to kill network container %q: %v", netContainer.ID, err) continue } } } // Check RestartPolicy for container recentContainers, err := dockertools.GetRecentDockerContainersWithNameAndUUID(kl.dockerClient, podFullName, uuid, container.Name) if err != nil { glog.Errorf("Error listing recent containers with name and uuid:%s--%s--%s", podFullName, uuid, container.Name) // TODO(dawnchen): error handling here? } if len(recentContainers) > 0 && pod.Spec.RestartPolicy.Always == nil { if pod.Spec.RestartPolicy.Never != nil { glog.V(3).Infof("Already ran container with name %s--%s--%s, do nothing", podFullName, uuid, container.Name) continue } if pod.Spec.RestartPolicy.OnFailure != nil { // Check the exit code of last run if recentContainers[0].State.ExitCode == 0 { glog.V(3).Infof("Already successfully ran container with name %s--%s--%s, do nothing", podFullName, uuid, container.Name) continue } } } glog.V(3).Infof("Container with name %s--%s--%s doesn't exist, creating %#v", podFullName, uuid, container.Name, container) ref, err := containerRef(pod, &container) if err != nil { glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err) } if !api.IsPullNever(container.ImagePullPolicy) { present, err := kl.dockerPuller.IsImagePresent(container.Image) latest := dockertools.RequireLatestImage(container.Image) if err != nil { if ref != nil { record.Eventf(ref, "failed", "failed", "Failed to inspect image %q", container.Image) } glog.Errorf("Failed to inspect image %q: %v; skipping pod %q container %q", container.Image, err, podFullName, container.Name) continue } if api.IsPullAlways(container.ImagePullPolicy) || (api.IsPullIfNotPresent(container.ImagePullPolicy) && (!present || latest)) { if err := kl.pullImage(container.Image, ref); err != nil { continue } } } // TODO(dawnchen): Check RestartPolicy.DelaySeconds before restart a container containerID, err := kl.runContainer(pod, &container, podVolumes, "container:"+string(netID)) if err != nil { // TODO(bburns) : Perhaps blacklist a container after N failures? glog.Errorf("Error running pod %q container %q: %v", podFullName, container.Name, err) continue } containersToKeep[containerID] = empty{} } // Kill any containers in this pod which were not identified above (guards against duplicates). for id, container := range dockerContainers { curPodFullName, curUUID, _, _ := dockertools.ParseDockerName(container.Names[0]) if curPodFullName == podFullName && curUUID == uuid { // Don't kill containers we want to keep or those we already killed. _, keep := containersToKeep[id] _, killed := killedContainers[id] if !keep && !killed { glog.V(1).Infof("Killing unwanted container in pod %q: %+v", curUUID, container) err = kl.killContainer(container) if err != nil { glog.Errorf("Error killing container: %v", err) } } } } return nil }
// GetPodStatus returns docker related status for all containers in the pod as // well as the infrastructure container. func (dm *DockerManager) GetPodStatus(pod *api.Pod) (*api.PodStatus, error) { podFullName := kubecontainer.GetPodFullName(pod) uid := pod.UID manifest := pod.Spec oldStatuses := make(map[string]api.ContainerStatus, len(pod.Spec.Containers)) lastObservedTime := make(map[string]util.Time, len(pod.Spec.Containers)) for _, status := range pod.Status.ContainerStatuses { oldStatuses[status.Name] = status if status.LastTerminationState.Termination != nil { lastObservedTime[status.Name] = status.LastTerminationState.Termination.FinishedAt } } var podStatus api.PodStatus statuses := make(map[string]*api.ContainerStatus, len(pod.Spec.Containers)) expectedContainers := make(map[string]api.Container) for _, container := range manifest.Containers { expectedContainers[container.Name] = container } expectedContainers[PodInfraContainerName] = api.Container{} containers, err := dm.client.ListContainers(docker.ListContainersOptions{All: true}) if err != nil { return nil, err } containerDone := util.NewStringSet() // Loop through list of running and exited docker containers to construct // the statuses. We assume docker returns a list of containers sorted in // reverse by time. for _, value := range containers { if len(value.Names) == 0 { continue } dockerName, _, err := ParseDockerName(value.Names[0]) if err != nil { continue } if dockerName.PodFullName != podFullName { continue } if uid != "" && dockerName.PodUID != uid { continue } dockerContainerName := dockerName.ContainerName c, found := expectedContainers[dockerContainerName] if !found { continue } terminationMessagePath := c.TerminationMessagePath if containerDone.Has(dockerContainerName) { continue } var terminationState *api.ContainerState = nil // Inspect the container. result := dm.inspectContainer(value.ID, dockerContainerName, terminationMessagePath) if result.err != nil { return nil, result.err } else if result.status.State.Termination != nil { terminationState = &result.status.State } if containerStatus, found := statuses[dockerContainerName]; found { if containerStatus.LastTerminationState.Termination == nil && terminationState != nil { // Populate the last termination state. containerStatus.LastTerminationState = *terminationState } count := true // Only count dead containers terminated after last time we observed, if lastObservedTime, ok := lastObservedTime[dockerContainerName]; ok { if terminationState != nil && terminationState.Termination.FinishedAt.After(lastObservedTime.Time) { count = false } else { // The container finished before the last observation. No // need to examine/count the older containers. Mark the // container name as done. containerDone.Insert(dockerContainerName) } } if count { containerStatus.RestartCount += 1 } continue } if dockerContainerName == PodInfraContainerName { // Found network container if result.status.State.Running != nil { podStatus.PodIP = result.ip } } else { // Add user container information. if oldStatus, found := oldStatuses[dockerContainerName]; found { // Use the last observed restart count if it's available. result.status.RestartCount = oldStatus.RestartCount } statuses[dockerContainerName] = &result.status } } // Handle the containers for which we cannot find any associated active or // dead docker containers. for _, container := range manifest.Containers { if _, found := statuses[container.Name]; found { continue } var containerStatus api.ContainerStatus containerStatus.Name = container.Name containerStatus.Image = container.Image if oldStatus, found := oldStatuses[container.Name]; found { // Some states may be lost due to GC; apply the last observed // values if possible. containerStatus.RestartCount = oldStatus.RestartCount containerStatus.LastTerminationState = oldStatus.LastTerminationState } //Check image is ready on the node or not. // TODO: If we integrate DockerPuller into DockerManager, we can // record the pull failure and eliminate the image checking below. image := container.Image // TODO(dchen1107): docker/docker/issues/8365 to figure out if the image exists _, err := dm.client.InspectImage(image) if err == nil { containerStatus.State.Waiting = &api.ContainerStateWaiting{ Reason: fmt.Sprintf("Image: %s is ready, container is creating", image), } } else if err == docker.ErrNoSuchImage { containerStatus.State.Waiting = &api.ContainerStateWaiting{ Reason: fmt.Sprintf("Image: %s is not ready on the node", image), } } statuses[container.Name] = &containerStatus } podStatus.ContainerStatuses = make([]api.ContainerStatus, 0) for containerName, status := range statuses { if status.State.Waiting != nil { // For containers in the waiting state, fill in a specific reason if it is recorded. if reason, ok := dm.reasonCache.Get(uid, containerName); ok { status.State.Waiting.Reason = reason } } podStatus.ContainerStatuses = append(podStatus.ContainerStatuses, *status) } return &podStatus, nil }