// Test the pod's exit code to be zero.
func testPodSuccessOrFail(c clientset.Interface, ns string, pod *v1.Pod) {
By("Pod should terminate with exitcode 0 (success)")
err := framework.WaitForPodSuccessInNamespace(c, pod.Name, ns)
Expect(err).NotTo(HaveOccurred())
framework.Logf("Pod %v succeeded ", pod.Name)
}
// Insert index.html with given content into given volume. It does so by
// starting and auxiliary pod which writes the file there.
// The volume must be writable.
func injectHtml(client *client.Client, config VolumeTestConfig, volume api.VolumeSource, content string) {
By(fmt.Sprint("starting ", config.prefix, " injector"))
podClient := client.Pods(config.namespace)
injectPod := &api.Pod{
TypeMeta: unversioned.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: api.ObjectMeta{
Name: config.prefix + "-injector",
Labels: map[string]string{
"role": config.prefix + "-injector",
},
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: config.prefix + "-injector",
Image: "gcr.io/google_containers/busybox:1.24",
Command: []string{"/bin/sh"},
Args: []string{"-c", "echo '" + content + "' > /mnt/index.html && chmod o+rX /mnt /mnt/index.html"},
VolumeMounts: []api.VolumeMount{
{
Name: config.prefix + "-volume",
MountPath: "/mnt",
},
},
},
},
SecurityContext: &api.PodSecurityContext{
SELinuxOptions: &api.SELinuxOptions{
Level: "s0:c0,c1",
},
},
RestartPolicy: api.RestartPolicyNever,
Volumes: []api.Volume{
{
Name: config.prefix + "-volume",
VolumeSource: volume,
},
},
},
}
defer func() {
podClient.Delete(config.prefix+"-injector", nil)
}()
injectPod, err := podClient.Create(injectPod)
framework.ExpectNoError(err, "Failed to create injector pod: %v", err)
err = framework.WaitForPodSuccessInNamespace(client, injectPod.Name, injectPod.Namespace)
Expect(err).NotTo(HaveOccurred())
}
// Test the pod's exitcode to be zero.
func testPodSuccessOrFail(f *framework.Framework, c *client.Client, ns string, pod *api.Pod) error {
By("Pod should terminate with exitcode 0 (success)")
err := framework.WaitForPodSuccessInNamespace(c, pod.Name, pod.Spec.Containers[0].Name, ns)
if err != nil {
return fmt.Errorf("Pod %v returned non-zero exitcode: %+v", pod.Name, err)
}
framework.Logf("pod %v exited successfully", pod.Name)
return nil
}
. "github.com/onsi/gomega"
)
var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
f := framework.NewDefaultFramework("gcl-logging")
BeforeEach(func() {
// TODO (crassirostris): Expand to GKE once the test is stable
framework.SkipUnlessProviderIs("gce")
})
It("should check that logs from containers are ingested in GCL", func() {
By("Running synthetic logger")
createSynthLogger(f, expectedLinesCount)
defer f.PodClient().Delete(synthLoggerPodName, &api.DeleteOptions{})
err := framework.WaitForPodSuccessInNamespace(f.ClientSet, synthLoggerPodName, f.Namespace.Name)
framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to succeed", synthLoggerPodName))
By("Waiting for logs to ingest")
totalMissing := expectedLinesCount
for start := time.Now(); time.Since(start) < ingestionTimeout; time.Sleep(ingestionRetryDelay) {
var err error
totalMissing, err = getMissingLinesCountGcl(f, synthLoggerPodName, expectedLinesCount)
if err != nil {
framework.Logf("Failed to get missing lines count due to %v", err)
totalMissing = expectedLinesCount
} else if totalMissing > 0 {
framework.Logf("Still missing %d lines", totalMissing)
}
if totalMissing == 0 {
Name: "test-apparmor",
Annotations: map[string]string{
apparmor.ContainerAnnotationKeyPrefix + "test": profile,
},
},
Spec: api.PodSpec{
Containers: []api.Container{{
Name: "test",
Image: "gcr.io/google_containers/busybox:1.24",
Command: []string{"sh", "-c", testCmd},
}},
RestartPolicy: api.RestartPolicyNever,
},
}
f.PodClient().Create(pod)
framework.ExpectNoError(framework.WaitForPodSuccessInNamespace(
f.ClientSet, pod.Name, f.Namespace.Name))
framework.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
})
})
func SkipIfAppArmorNotSupported() {
framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu")
}
func LoadAppArmorProfiles(f *framework.Framework) {
_, err := createAppArmorProfileCM(f)
framework.ExpectNoError(err)
_, err = createAppArmorProfileLoader(f)
framework.ExpectNoError(err)
}
// allow the recycler a chance to catch up. it has to perform NFS scrub, which can be slow in e2e.
framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 5*time.Second, 300*time.Second)
pv, err = c.PersistentVolumes().Get(pv.Name)
Expect(err).NotTo(HaveOccurred())
if pv.Spec.ClaimRef != nil {
framework.Failf("Expected PersistentVolume to be unbound, but found non-nil ClaimRef: %+v", pv)
}
// The NFS Server pod we're using contains an index.html file
// Verify the file was really scrubbed from the volume
podTemplate := makeCheckPod(ns, serverIP)
checkpod, err := c.Pods(ns).Create(podTemplate)
framework.ExpectNoError(err, "Failed to create checker pod: %v", err)
err = framework.WaitForPodSuccessInNamespace(c, checkpod.Name, checkpod.Spec.Containers[0].Name, checkpod.Namespace)
Expect(err).NotTo(HaveOccurred())
})
})
func makePersistentVolume(serverIP string) *api.PersistentVolume {
return &api.PersistentVolume{
ObjectMeta: api.ObjectMeta{
GenerateName: "nfs-",
},
Spec: api.PersistentVolumeSpec{
PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimRecycle,
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("2Gi"),
},
PersistentVolumeSource: api.PersistentVolumeSource{
},
Spec: v1.PodSpec{
// this pod is expected to exit successfully
RestartPolicy: v1.RestartPolicyNever,
Containers: []v1.Container{
{
Image: "gcr.io/google_containers/busybox:1.24",
Name: logContName,
Command: []string{"sh", "-c", "echo " + logString},
},
},
},
}
podClient.Create(logPod)
err := framework.WaitForPodSuccessInNamespace(f.ClientSet, logPodName, ns)
framework.ExpectNoError(err, "Failed waiting for pod: %s to enter success state", logPodName)
// get containerID from created Pod
createdLogPod, err := podClient.Get(logPodName, metav1.GetOptions{})
logConID := kubecontainer.ParseContainerID(createdLogPod.Status.ContainerStatuses[0].ContainerID)
framework.ExpectNoError(err, "Failed to get pod: %s", logPodName)
expectedlogFile := logDir + "/" + logPodName + "_" + ns + "_" + logContName + "-" + logConID.ID + ".log"
checkPod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: checkPodName,
},
Spec: v1.PodSpec{
// this pod is expected to exit successfully
MountPath: "/tmp",
},
},
},
},
Volumes: []v1.Volume{
{
Name: volumeName,
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{Medium: v1.StorageMediumMemory},
},
},
},
},
})
err := framework.WaitForPodSuccessInNamespace(f.ClientSet, memoryBackedPod.Name, f.Namespace.Name)
Expect(err).NotTo(HaveOccurred())
})
By("Verifying the memory backed volume was removed from node", func() {
volumePath := fmt.Sprintf("/tmp/%s/volumes/kubernetes.io~empty-dir/%s", string(memoryBackedPod.UID), volumeName)
var err error
for i := 0; i < 10; i++ {
// need to create a new verification pod on each pass since updates
//to the HostPath volume aren't propogated to the pod
pod := f.PodClient().Create(&v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod" + string(uuid.NewUUID()),
Namespace: f.Namespace.Name,
},
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
// ClusterLevelLoggingWithElasticsearch is an end to end test for cluster level logging.
func ClusterLevelLoggingWithElasticsearch(f *framework.Framework) {
// graceTime is how long to keep retrying requests for status information.
const graceTime = 5 * time.Minute
// ingestionTimeout is how long to keep retrying to wait for all the
// logs to be ingested.
const ingestionTimeout = 10 * time.Minute
// Check for the existence of the Elasticsearch service.
By("Checking the Elasticsearch service exists.")
s := f.Client.Services(api.NamespaceSystem)
// Make a few attempts to connect. This makes the test robust against
// being run as the first e2e test just after the e2e cluster has been created.
var err error
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
if _, err = s.Get("elasticsearch-logging"); err == nil {
break
}
framework.Logf("Attempt to check for the existence of the Elasticsearch service failed after %v", time.Since(start))
}
Expect(err).NotTo(HaveOccurred())
// Wait for the Elasticsearch pods to enter the running state.
By("Checking to make sure the Elasticsearch pods are running")
label := labels.SelectorFromSet(labels.Set(map[string]string{k8sAppKey: esValue}))
options := api.ListOptions{LabelSelector: label}
pods, err := f.Client.Pods(api.NamespaceSystem).List(options)
Expect(err).NotTo(HaveOccurred())
for _, pod := range pods.Items {
err = framework.WaitForPodRunningInNamespace(f.Client, &pod)
Expect(err).NotTo(HaveOccurred())
}
By("Checking to make sure we are talking to an Elasticsearch service.")
// Perform a few checks to make sure this looks like an Elasticsearch cluster.
var statusCode float64
var esResponse map[string]interface{}
err = nil
var body []byte
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(10 * time.Second) {
proxyRequest, errProxy := framework.GetServicesProxyRequest(f.Client, f.Client.Get())
if errProxy != nil {
framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
continue
}
// Query against the root URL for Elasticsearch.
body, err = proxyRequest.Namespace(api.NamespaceSystem).
Name("elasticsearch-logging").
DoRaw()
if err != nil {
framework.Logf("After %v proxy call to elasticsearch-loigging failed: %v", time.Since(start), err)
continue
}
esResponse, err = bodyToJSON(body)
if err != nil {
framework.Logf("After %v failed to convert Elasticsearch JSON response %v to map[string]interface{}: %v", time.Since(start), string(body), err)
continue
}
statusIntf, ok := esResponse["status"]
if !ok {
framework.Logf("After %v Elasticsearch response has no status field: %v", time.Since(start), esResponse)
continue
}
statusCode, ok = statusIntf.(float64)
if !ok {
// Assume this is a string returning Failure. Retry.
framework.Logf("After %v expected status to be a float64 but got %v of type %T", time.Since(start), statusIntf, statusIntf)
continue
}
if int(statusCode) != 200 {
framework.Logf("After %v Elasticsearch cluster has a bad status: %v", time.Since(start), statusCode)
continue
}
break
}
Expect(err).NotTo(HaveOccurred())
if int(statusCode) != 200 {
framework.Failf("Elasticsearch cluster has a bad status: %v", statusCode)
}
// Check to see if have a cluster_name field.
clusterName, ok := esResponse["cluster_name"]
if !ok {
framework.Failf("No cluster_name field in Elasticsearch response: %v", esResponse)
}
if clusterName != "kubernetes-logging" {
framework.Failf("Connected to wrong cluster %q (expecting kubernetes_logging)", clusterName)
}
// Now assume we really are talking to an Elasticsearch instance.
// Check the cluster health.
By("Checking health of Elasticsearch service.")
healthy := false
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
proxyRequest, errProxy := framework.GetServicesProxyRequest(f.Client, f.Client.Get())
if errProxy != nil {
framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
continue
}
body, err = proxyRequest.Namespace(api.NamespaceSystem).
Name("elasticsearch-logging").
Suffix("_cluster/health").
Param("level", "indices").
DoRaw()
if err != nil {
continue
}
health, err := bodyToJSON(body)
if err != nil {
framework.Logf("Bad json response from elasticsearch: %v", err)
continue
}
statusIntf, ok := health["status"]
if !ok {
framework.Logf("No status field found in cluster health response: %v", health)
continue
}
status := statusIntf.(string)
if status != "green" && status != "yellow" {
framework.Logf("Cluster health has bad status: %v", health)
continue
}
if err == nil && ok {
healthy = true
break
}
}
if !healthy {
framework.Failf("After %v elasticsearch cluster is not healthy", graceTime)
}
// Obtain a list of nodes so we can place one synthetic logger on each node.
nodes := framework.GetReadySchedulableNodesOrDie(f.Client)
nodeCount := len(nodes.Items)
if nodeCount == 0 {
framework.Failf("Failed to find any nodes")
}
framework.Logf("Found %d nodes.", len(nodes.Items))
// Filter out unhealthy nodes.
// Previous tests may have cause failures of some nodes. Let's skip
// 'Not Ready' nodes, just in case (there is no need to fail the test).
framework.FilterNodes(nodes, func(node api.Node) bool {
return framework.IsNodeConditionSetAsExpected(&node, api.NodeReady, true)
})
if len(nodes.Items) < 2 {
framework.Failf("Less than two nodes were found Ready: %d", len(nodes.Items))
}
framework.Logf("Found %d healthy nodes.", len(nodes.Items))
// Wait for the Fluentd pods to enter the running state.
By("Checking to make sure the Fluentd pod are running on each healthy node")
label = labels.SelectorFromSet(labels.Set(map[string]string{k8sAppKey: fluentdValue}))
options = api.ListOptions{LabelSelector: label}
fluentdPods, err := f.Client.Pods(api.NamespaceSystem).List(options)
Expect(err).NotTo(HaveOccurred())
for _, pod := range fluentdPods.Items {
if nodeInNodeList(pod.Spec.NodeName, nodes) {
err = framework.WaitForPodRunningInNamespace(f.Client, &pod)
Expect(err).NotTo(HaveOccurred())
}
}
// Check if each healthy node has fluentd running on it
for _, node := range nodes.Items {
exists := false
for _, pod := range fluentdPods.Items {
if pod.Spec.NodeName == node.Name {
exists = true
break
}
}
if !exists {
framework.Failf("Node %v does not have fluentd pod running on it.", node.Name)
}
}
// Create a unique root name for the resources in this test to permit
// parallel executions of this test.
// Use a unique namespace for the resources created in this test.
ns := f.Namespace.Name
name := "synthlogger"
// Form a unique name to taint log lines to be collected.
// Replace '-' characters with '_' to prevent the analyzer from breaking apart names.
taintName := strings.Replace(ns+name, "-", "_", -1)
framework.Logf("Tainting log lines with %v", taintName)
// podNames records the names of the synthetic logging pods that are created in the
// loop below.
var podNames []string
// countTo is the number of log lines emitted (and checked) for each synthetic logging pod.
const countTo = 100
// Instantiate a synthetic logger pod on each node.
for i, node := range nodes.Items {
podName := fmt.Sprintf("%s-%d", name, i)
_, err := f.Client.Pods(ns).Create(&api.Pod{
ObjectMeta: api.ObjectMeta{
Name: podName,
Labels: map[string]string{"name": name},
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "synth-logger",
Image: "gcr.io/google_containers/ubuntu:14.04",
// notice: the subshell syntax is escaped with `$$`
Command: []string{"bash", "-c", fmt.Sprintf("i=0; while ((i < %d)); do echo \"%d %s $i %s\"; i=$$(($i+1)); done", countTo, i, taintName, podName)},
},
},
NodeName: node.Name,
RestartPolicy: api.RestartPolicyNever,
},
})
Expect(err).NotTo(HaveOccurred())
podNames = append(podNames, podName)
}
// Cleanup the pods when we are done.
defer func() {
for _, pod := range podNames {
if err = f.Client.Pods(ns).Delete(pod, nil); err != nil {
framework.Logf("Failed to delete pod %s: %v", pod, err)
}
}
}()
// Wait for the synthetic logging pods to finish.
By("Waiting for the pods to succeed.")
for _, pod := range podNames {
err = framework.WaitForPodSuccessInNamespace(f.Client, pod, "synth-logger", ns)
Expect(err).NotTo(HaveOccurred())
}
// Make several attempts to observe the logs ingested into Elasticsearch.
By("Checking all the log lines were ingested into Elasticsearch")
totalMissing := 0
expected := nodeCount * countTo
missingPerNode := []int{}
for start := time.Now(); time.Since(start) < ingestionTimeout; time.Sleep(25 * time.Second) {
// Debugging code to report the status of the elasticsearch logging endpoints.
selector := labels.Set{k8sAppKey: esValue}.AsSelector()
options := api.ListOptions{LabelSelector: selector}
esPods, err := f.Client.Pods(api.NamespaceSystem).List(options)
if err != nil {
framework.Logf("Attempt to list Elasticsearch nodes encountered a problem -- may retry: %v", err)
continue
} else {
for i, pod := range esPods.Items {
framework.Logf("pod %d: %s PodIP %s phase %s condition %+v", i, pod.Name, pod.Status.PodIP, pod.Status.Phase,
pod.Status.Conditions)
}
}
proxyRequest, errProxy := framework.GetServicesProxyRequest(f.Client, f.Client.Get())
if errProxy != nil {
framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
continue
}
// Ask Elasticsearch to return all the log lines that were tagged with the underscore
// version of the name. Ask for twice as many log lines as we expect to check for
// duplication bugs.
body, err = proxyRequest.Namespace(api.NamespaceSystem).
Name("elasticsearch-logging").
Suffix("_search").
Param("q", fmt.Sprintf("log:%s", taintName)).
Param("size", strconv.Itoa(2*expected)).
DoRaw()
if err != nil {
framework.Logf("After %v failed to make proxy call to elasticsearch-logging: %v", time.Since(start), err)
continue
}
response, err := bodyToJSON(body)
if err != nil {
framework.Logf("After %v failed to unmarshal response: %v", time.Since(start), err)
framework.Logf("Body: %s", string(body))
continue
}
hits, ok := response["hits"].(map[string]interface{})
if !ok {
framework.Logf("response[hits] not of the expected type: %T", response["hits"])
continue
}
totalF, ok := hits["total"].(float64)
if !ok {
framework.Logf("After %v hits[total] not of the expected type: %T", time.Since(start), hits["total"])
continue
}
total := int(totalF)
if total != expected {
framework.Logf("After %v expecting to find %d log lines but saw %d", time.Since(start), expected, total)
}
h, ok := hits["hits"].([]interface{})
if !ok {
framework.Logf("After %v hits not of the expected type: %T", time.Since(start), hits["hits"])
continue
}
// Initialize data-structure for observing counts.
observed := make([][]int, nodeCount)
for i := range observed {
observed[i] = make([]int, countTo)
}
// Iterate over the hits and populate the observed array.
for _, e := range h {
l, ok := e.(map[string]interface{})
if !ok {
framework.Logf("element of hit not of expected type: %T", e)
continue
}
source, ok := l["_source"].(map[string]interface{})
if !ok {
framework.Logf("_source not of the expected type: %T", l["_source"])
continue
}
msg, ok := source["log"].(string)
if !ok {
framework.Logf("log not of the expected type: %T", source["log"])
continue
}
words := strings.Split(msg, " ")
if len(words) != 4 {
framework.Logf("Malformed log line: %s", msg)
continue
}
n, err := strconv.ParseUint(words[0], 10, 0)
if err != nil {
framework.Logf("Expecting numer of node as first field of %s", msg)
continue
}
if n < 0 || int(n) >= nodeCount {
framework.Logf("Node count index out of range: %d", nodeCount)
continue
}
index, err := strconv.ParseUint(words[2], 10, 0)
if err != nil {
framework.Logf("Expecting number as third field of %s", msg)
continue
}
if index < 0 || index >= countTo {
framework.Logf("Index value out of range: %d", index)
continue
}
if words[1] != taintName {
framework.Logf("Elasticsearch query return unexpected log line: %s", msg)
continue
}
// Record the observation of a log line from node n at the given index.
observed[n][index]++
}
// Make sure we correctly observed the expected log lines from each node.
totalMissing = 0
missingPerNode = make([]int, nodeCount)
incorrectCount := false
for n := range observed {
for i, c := range observed[n] {
if c == 0 {
totalMissing++
missingPerNode[n]++
}
if c < 0 || c > 1 {
framework.Logf("Got incorrect count for node %d index %d: %d", n, i, c)
incorrectCount = true
}
}
}
if incorrectCount {
framework.Logf("After %v es still return duplicated log lines", time.Since(start))
continue
}
if totalMissing != 0 {
framework.Logf("After %v still missing %d log lines", time.Since(start), totalMissing)
continue
}
framework.Logf("After %s found all %d log lines", time.Since(start), expected)
return
}
for n := range missingPerNode {
if missingPerNode[n] > 0 {
framework.Logf("Node %d %s is missing %d logs", n, nodes.Items[n].Name, missingPerNode[n])
opts := &api.PodLogOptions{}
body, err = f.Client.Pods(ns).GetLogs(podNames[n], opts).DoRaw()
if err != nil {
framework.Logf("Cannot get logs from pod %v", podNames[n])
continue
}
framework.Logf("Pod %s has the following logs: %s", podNames[n], body)
for _, pod := range fluentdPods.Items {
if pod.Spec.NodeName == nodes.Items[n].Name {
body, err = f.Client.Pods(api.NamespaceSystem).GetLogs(pod.Name, opts).DoRaw()
if err != nil {
framework.Logf("Cannot get logs from pod %v", pod.Name)
break
}
framework.Logf("Fluentd Pod %s on node %s has the following logs: %s", pod.Name, nodes.Items[n].Name, body)
break
}
}
}
}
framework.Failf("Failed to find all %d log lines", expected)
}
// Check that the PersistentVolume's ClaimRef contains the UID of the PersistendVolumeClaim
if pvc.ObjectMeta.UID != pv.Spec.ClaimRef.UID {
framework.Failf("Binding failed: PersistentVolumeClaim UID does not match PersistentVolume's ClaimRef UID. ")
}
// writePod writes to the nfs volume
framework.Logf("Creating writePod")
pvc, _ = c.PersistentVolumeClaims(ns).Get(pvc.Name)
writePod := makeWritePod(ns, pvc.Name)
writePod, err = c.Pods(ns).Create(writePod)
if err != nil {
framework.Failf("Create writePod failed: %+v", err)
}
// Wait for the writePod to complete it's lifecycle
err = framework.WaitForPodSuccessInNamespace(c, writePod.Name, writePod.Spec.Containers[0].Name, writePod.Namespace)
if err != nil {
framework.Failf("WritePod exited with error: %+v", err)
} else {
framework.Logf("WritePod exited without error.")
}
// Delete the PersistentVolumeClaim
framework.Logf("Deleting PersistentVolumeClaim to trigger PV Recycling")
err = c.PersistentVolumeClaims(ns).Delete(pvc.Name)
if err != nil {
framework.Failf("Delete PersistentVolumeClaim failed: %v", err)
}
// Wait for the PersistentVolume phase to return to Available
framework.Logf("Waiting for recycling process to complete.")
Image: ImageRegistry[busyBoxImage],
Name: contName,
Command: []string{"sh", "-c", "if [ -d /tmp/memory/Burstable ] && [ -d /tmp/memory/BestEffort ]; then exit 0; else exit 1; fi"},
VolumeMounts: []api.VolumeMount{
{
Name: "sysfscgroup",
MountPath: "/tmp",
},
},
},
},
Volumes: []api.Volume{
{
Name: "sysfscgroup",
VolumeSource: api.VolumeSource{
HostPath: &api.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
},
},
},
},
}
podClient := f.PodClient()
podClient.Create(pod)
err := framework.WaitForPodSuccessInNamespace(f.Client, podName, contName, f.Namespace.Name)
Expect(err).NotTo(HaveOccurred())
}
})
})
})
})