// We add this function, because apiserver only supports *RFC3339* now, which means that the timestamp returned by // apiserver has no nanosecond infromation. However, the timestamp returned by unversioned.Now() contains nanosecond, // so when we do comparison between status from apiserver and cached status, isStatusEqual() will always return false. // There is related issue #15262 and PR #15263 about this. // In fact, the best way to solve this is to do it on api side. However for now, we normalize the status locally in // kubelet temporarily. // TODO(random-liu): Remove timestamp related logic after apiserver supports nanosecond or makes it consistent. func normalizeStatus(pod *api.Pod, status *api.PodStatus) *api.PodStatus { normalizeTimeStamp := func(t *unversioned.Time) { *t = t.Rfc3339Copy() } normalizeContainerState := func(c *api.ContainerState) { if c.Running != nil { normalizeTimeStamp(&c.Running.StartedAt) } if c.Terminated != nil { normalizeTimeStamp(&c.Terminated.StartedAt) normalizeTimeStamp(&c.Terminated.FinishedAt) } } if status.StartTime != nil { normalizeTimeStamp(status.StartTime) } for i := range status.Conditions { condition := &status.Conditions[i] normalizeTimeStamp(&condition.LastProbeTime) normalizeTimeStamp(&condition.LastTransitionTime) } // update container statuses for i := range status.ContainerStatuses { cstatus := &status.ContainerStatuses[i] normalizeContainerState(&cstatus.State) normalizeContainerState(&cstatus.LastTerminationState) } // Sort the container statuses, so that the order won't affect the result of comparison sort.Sort(kubetypes.SortedContainerStatuses(status.ContainerStatuses)) // update init container statuses for i := range status.InitContainerStatuses { cstatus := &status.InitContainerStatuses[i] normalizeContainerState(&cstatus.State) normalizeContainerState(&cstatus.LastTerminationState) } // Sort the container statuses, so that the order won't affect the result of comparison kubetypes.SortInitContainerStatuses(pod, status.InitContainerStatuses) return status }
// convertToAPIContainerStatuses converts the given internal container // statuses into API container statuses. func (kl *Kubelet) convertToAPIContainerStatuses(pod *api.Pod, podStatus *kubecontainer.PodStatus, previousStatus []api.ContainerStatus, containers []api.Container, hasInitContainers, isInitContainer bool) []api.ContainerStatus { convertContainerStatus := func(cs *kubecontainer.ContainerStatus) *api.ContainerStatus { cid := cs.ID.String() status := &api.ContainerStatus{ Name: cs.Name, RestartCount: int32(cs.RestartCount), Image: cs.Image, ImageID: cs.ImageID, ContainerID: cid, } switch cs.State { case kubecontainer.ContainerStateRunning: status.State.Running = &api.ContainerStateRunning{StartedAt: unversioned.NewTime(cs.StartedAt)} case kubecontainer.ContainerStateExited: status.State.Terminated = &api.ContainerStateTerminated{ ExitCode: int32(cs.ExitCode), Reason: cs.Reason, Message: cs.Message, StartedAt: unversioned.NewTime(cs.StartedAt), FinishedAt: unversioned.NewTime(cs.FinishedAt), ContainerID: cid, } default: status.State.Waiting = &api.ContainerStateWaiting{} } return status } // Fetch old containers statuses from old pod status. oldStatuses := make(map[string]api.ContainerStatus, len(containers)) for _, status := range previousStatus { oldStatuses[status.Name] = status } // Set all container statuses to default waiting state statuses := make(map[string]*api.ContainerStatus, len(containers)) defaultWaitingState := api.ContainerState{Waiting: &api.ContainerStateWaiting{Reason: "ContainerCreating"}} if hasInitContainers { defaultWaitingState = api.ContainerState{Waiting: &api.ContainerStateWaiting{Reason: "PodInitializing"}} } for _, container := range containers { status := &api.ContainerStatus{ Name: container.Name, Image: container.Image, State: defaultWaitingState, } // Apply some values from the old statuses as the default values. if oldStatus, found := oldStatuses[container.Name]; found { status.RestartCount = oldStatus.RestartCount status.LastTerminationState = oldStatus.LastTerminationState } statuses[container.Name] = status } // Make the latest container status comes first. sort.Sort(sort.Reverse(kubecontainer.SortContainerStatusesByCreationTime(podStatus.ContainerStatuses))) // Set container statuses according to the statuses seen in pod status containerSeen := map[string]int{} for _, cStatus := range podStatus.ContainerStatuses { cName := cStatus.Name if _, ok := statuses[cName]; !ok { // This would also ignore the infra container. continue } if containerSeen[cName] >= 2 { continue } status := convertContainerStatus(cStatus) if containerSeen[cName] == 0 { statuses[cName] = status } else { statuses[cName].LastTerminationState = status.State } containerSeen[cName] = containerSeen[cName] + 1 } // Handle the containers failed to be started, which should be in Waiting state. for _, container := range containers { if isInitContainer { // If the init container is terminated with exit code 0, it won't be restarted. // TODO(random-liu): Handle this in a cleaner way. s := podStatus.FindContainerStatusByName(container.Name) if s != nil && s.State == kubecontainer.ContainerStateExited && s.ExitCode == 0 { continue } } // If a container should be restarted in next syncpod, it is *Waiting*. if !kubecontainer.ShouldContainerBeRestarted(&container, pod, podStatus) { continue } status := statuses[container.Name] reason, message, ok := kl.reasonCache.Get(pod.UID, container.Name) if !ok { // In fact, we could also apply Waiting state here, but it is less informative, // and the container will be restarted soon, so we prefer the original state here. // Note that with the current implementation of ShouldContainerBeRestarted the original state here // could be: // * Waiting: There is no associated historical container and start failure reason record. // * Terminated: The container is terminated. continue } if status.State.Terminated != nil { status.LastTerminationState = status.State } status.State = api.ContainerState{ Waiting: &api.ContainerStateWaiting{ Reason: reason.Error(), Message: message, }, } statuses[container.Name] = status } var containerStatuses []api.ContainerStatus for _, status := range statuses { containerStatuses = append(containerStatuses, *status) } // Sort the container statuses since clients of this interface expect the list // of containers in a pod has a deterministic order. if isInitContainer { kubetypes.SortInitContainerStatuses(pod, containerStatuses) } else { sort.Sort(kubetypes.SortedContainerStatuses(containerStatuses)) } return containerStatuses }