// GeneratePodInitializedCondition returns initialized condition if all init containers in a pod are ready, else it // returns an uninitialized condition. func GeneratePodInitializedCondition(spec *v1.PodSpec, containerStatuses []v1.ContainerStatus, podPhase v1.PodPhase) v1.PodCondition { // Find if all containers are ready or not. if containerStatuses == nil && len(spec.InitContainers) > 0 { return v1.PodCondition{ Type: v1.PodInitialized, Status: v1.ConditionFalse, Reason: "UnknownContainerStatuses", } } unknownContainers := []string{} unreadyContainers := []string{} for _, container := range spec.InitContainers { if containerStatus, ok := v1.GetContainerStatus(containerStatuses, container.Name); ok { if !containerStatus.Ready { unreadyContainers = append(unreadyContainers, container.Name) } } else { unknownContainers = append(unknownContainers, container.Name) } } // If all init containers are known and succeeded, just return PodCompleted. if podPhase == v1.PodSucceeded && len(unknownContainers) == 0 { return v1.PodCondition{ Type: v1.PodInitialized, Status: v1.ConditionTrue, Reason: "PodCompleted", } } unreadyMessages := []string{} if len(unknownContainers) > 0 { unreadyMessages = append(unreadyMessages, fmt.Sprintf("containers with unknown status: %s", unknownContainers)) } if len(unreadyContainers) > 0 { unreadyMessages = append(unreadyMessages, fmt.Sprintf("containers with incomplete status: %s", unreadyContainers)) } unreadyMessage := strings.Join(unreadyMessages, ", ") if unreadyMessage != "" { return v1.PodCondition{ Type: v1.PodInitialized, Status: v1.ConditionFalse, Reason: "ContainersNotInitialized", Message: unreadyMessage, } } return v1.PodCondition{ Type: v1.PodInitialized, Status: v1.ConditionTrue, } }
func getRestartDelay(podClient *framework.PodClient, podName string, containerName string) (time.Duration, error) { beginTime := time.Now() for time.Since(beginTime) < (2 * maxBackOffTolerance) { // may just miss the 1st MaxContainerBackOff delay time.Sleep(time.Second) pod, err := podClient.Get(podName, metav1.GetOptions{}) framework.ExpectNoError(err, fmt.Sprintf("getting pod %s", podName)) status, ok := v1.GetContainerStatus(pod.Status.ContainerStatuses, containerName) if !ok { framework.Logf("getRestartDelay: status missing") continue } if status.State.Waiting == nil && status.State.Running != nil && status.LastTerminationState.Terminated != nil && status.State.Running.StartedAt.Time.After(beginTime) { startedAt := status.State.Running.StartedAt.Time finishedAt := status.LastTerminationState.Terminated.FinishedAt.Time framework.Logf("getRestartDelay: restartCount = %d, finishedAt=%s restartedAt=%s (%s)", status.RestartCount, finishedAt, startedAt, startedAt.Sub(finishedAt)) return startedAt.Sub(finishedAt), nil } } return 0, fmt.Errorf("timeout getting pod restart delay") }
// doProbe probes the container once and records the result. // Returns whether the worker should continue. func (w *worker) doProbe() (keepGoing bool) { defer func() { recover() }() // Actually eat panics (HandleCrash takes care of logging) defer runtime.HandleCrash(func(_ interface{}) { keepGoing = true }) status, ok := w.probeManager.statusManager.GetPodStatus(w.pod.UID) if !ok { // Either the pod has not been created yet, or it was already deleted. glog.V(3).Infof("No status for pod: %v", format.Pod(w.pod)) return true } // Worker should terminate if pod is terminated. if status.Phase == v1.PodFailed || status.Phase == v1.PodSucceeded { glog.V(3).Infof("Pod %v %v, exiting probe worker", format.Pod(w.pod), status.Phase) return false } c, ok := v1.GetContainerStatus(status.ContainerStatuses, w.container.Name) if !ok || len(c.ContainerID) == 0 { // Either the container has not been created yet, or it was deleted. glog.V(3).Infof("Probe target container not found: %v - %v", format.Pod(w.pod), w.container.Name) return true // Wait for more information. } if w.containerID.String() != c.ContainerID { if !w.containerID.IsEmpty() { w.resultsManager.Remove(w.containerID) } w.containerID = kubecontainer.ParseContainerID(c.ContainerID) w.resultsManager.Set(w.containerID, w.initialValue, w.pod) // We've got a new container; resume probing. w.onHold = false } if w.onHold { // Worker is on hold until there is a new container. return true } if c.State.Running == nil { glog.V(3).Infof("Non-running container probed: %v - %v", format.Pod(w.pod), w.container.Name) if !w.containerID.IsEmpty() { w.resultsManager.Set(w.containerID, results.Failure, w.pod) } // Abort if the container will not be restarted. return c.State.Terminated == nil || w.pod.Spec.RestartPolicy != v1.RestartPolicyNever } if int32(time.Since(c.State.Running.StartedAt.Time).Seconds()) < w.spec.InitialDelaySeconds { return true } result, err := w.probeManager.prober.probe(w.probeType, w.pod, status, w.container, w.containerID) if err != nil { // Prober error, throw away the result. return true } if w.lastResult == result { w.resultRun++ } else { w.lastResult = result w.resultRun = 1 } if (result == results.Failure && w.resultRun < int(w.spec.FailureThreshold)) || (result == results.Success && w.resultRun < int(w.spec.SuccessThreshold)) { // Success or failure is below threshold - leave the probe state unchanged. return true } w.resultsManager.Set(w.containerID, result, w.pod) if w.probeType == liveness && result == results.Failure { // The container fails a liveness check, it will need to be restared. // Stop probing until we see a new container ID. This is to reduce the // chance of hitting #21751, where running `docker exec` when a // container is being stopped may lead to corrupted container state. w.onHold = true } return true }