// testHostIP tests that a pod gets a host IP
func testHostIP(c *client.Client, ns string, pod *api.Pod) {
podClient := c.Pods(ns)
By("creating pod")
defer podClient.Delete(pod.Name, api.NewDeleteOptions(0))
if _, err := podClient.Create(pod); err != nil {
framework.Failf("Failed to create pod: %v", err)
}
By("ensuring that pod is running and has a hostIP")
// Wait for the pods to enter the running state. Waiting loops until the pods
// are running so non-running pods cause a timeout for this test.
err := framework.WaitForPodRunningInNamespace(c, pod.Name, ns)
Expect(err).NotTo(HaveOccurred())
// Try to make sure we get a hostIP for each pod.
hostIPTimeout := 2 * time.Minute
t := time.Now()
for {
p, err := podClient.Get(pod.Name)
Expect(err).NotTo(HaveOccurred())
if p.Status.HostIP != "" {
framework.Logf("Pod %s has hostIP: %s", p.Name, p.Status.HostIP)
break
}
if time.Since(t) >= hostIPTimeout {
framework.Failf("Gave up waiting for hostIP of pod %s after %v seconds",
p.Name, time.Since(t).Seconds())
}
framework.Logf("Retrying to get the hostIP of pod %s", p.Name)
time.Sleep(5 * time.Second)
}
}
func runPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
pod := createPausePod(f, conf)
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, pod))
pod, err := f.ClientSet.Core().Pods(f.Namespace.Name).Get(conf.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
return pod
}
func runPausePod(f *framework.Framework, conf pausePodConfig) *api.Pod {
pod := createPausePod(f, conf)
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.Client, pod))
pod, err := f.Client.Pods(f.Namespace.Name).Get(conf.Name)
framework.ExpectNoError(err)
return pod
}
func forEachPod(c *client.Client, ns, selectorKey, selectorValue string, fn func(api.Pod)) {
pods := []*api.Pod{}
for t := time.Now(); time.Since(t) < framework.PodListTimeout; time.Sleep(framework.Poll) {
selector := labels.SelectorFromSet(labels.Set(map[string]string{selectorKey: selectorValue}))
options := api.ListOptions{LabelSelector: selector}
podList, err := c.Pods(ns).List(options)
Expect(err).NotTo(HaveOccurred())
for _, pod := range podList.Items {
if pod.Status.Phase == api.PodPending || pod.Status.Phase == api.PodRunning {
pods = append(pods, &pod)
}
}
if len(pods) > 0 {
break
}
}
if pods == nil || len(pods) == 0 {
framework.Failf("No pods found")
}
for _, pod := range pods {
err := framework.WaitForPodRunningInNamespace(c, pod.Name, ns)
Expect(err).NotTo(HaveOccurred())
fn(*pod)
}
}
// ClusterLevelLoggingWithKibana is an end to end test that checks to see if Kibana is alive.
func ClusterLevelLoggingWithKibana(f *framework.Framework) {
// graceTime is how long to keep retrying requests for status information.
const graceTime = 20 * time.Minute
// Check for the existence of the Kibana service.
By("Checking the Kibana service exists.")
s := f.ClientSet.Core().Services(api.NamespaceSystem)
// Make a few attempts to connect. This makes the test robust against
// being run as the first e2e test just after the e2e cluster has been created.
var err error
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
if _, err = s.Get("kibana-logging", metav1.GetOptions{}); err == nil {
break
}
framework.Logf("Attempt to check for the existence of the Kibana service failed after %v", time.Since(start))
}
Expect(err).NotTo(HaveOccurred())
// Wait for the Kibana pod(s) to enter the running state.
By("Checking to make sure the Kibana pods are running")
label := labels.SelectorFromSet(labels.Set(map[string]string{kibanaKey: kibanaValue}))
options := v1.ListOptions{LabelSelector: label.String()}
pods, err := f.ClientSet.Core().Pods(api.NamespaceSystem).List(options)
Expect(err).NotTo(HaveOccurred())
for _, pod := range pods.Items {
err = framework.WaitForPodRunningInNamespace(f.ClientSet, &pod)
Expect(err).NotTo(HaveOccurred())
}
By("Checking to make sure we get a response from the Kibana UI.")
err = nil
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
if errProxy != nil {
framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
err = errProxy
continue
}
ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout)
defer cancel()
// Query against the root URL for Kibana.
_, err = proxyRequest.Namespace(api.NamespaceSystem).
Context(ctx).
Name("kibana-logging").
DoRaw()
if err != nil {
if ctx.Err() != nil {
framework.Failf("After %v proxy call to kibana-logging failed: %v", time.Since(start), err)
break
}
framework.Logf("After %v proxy call to kibana-logging failed: %v", time.Since(start), err)
continue
}
break
}
Expect(err).NotTo(HaveOccurred())
}
// Define and create a pod with a mounted PV. Pod runs infinite loop until killed.
func createClientPod(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) *v1.Pod {
clientPod := makePod(ns, pvc.Name)
clientPod, err := c.Core().Pods(ns).Create(clientPod)
Expect(err).NotTo(HaveOccurred())
// Verify the pod is running before returning it
err = framework.WaitForPodRunningInNamespace(c, clientPod)
Expect(err).NotTo(HaveOccurred())
clientPod, err = c.Core().Pods(ns).Get(clientPod.Name, metav1.GetOptions{})
Expect(apierrs.IsNotFound(err)).To(BeFalse())
return clientPod
}
func verifyDNSPodIsRunning(f *framework.Framework) {
systemClient := f.Client.Pods(api.NamespaceSystem)
By("Waiting for DNS Service to be Running")
options := api.ListOptions{LabelSelector: dnsServiceLabelSelector}
dnsPods, err := systemClient.List(options)
if err != nil {
framework.Failf("Failed to list all dns service pods")
}
if len(dnsPods.Items) != 1 {
framework.Failf("Unexpected number of pods (%d) matches the label selector %v", len(dnsPods.Items), dnsServiceLabelSelector.String())
}
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.Client, dnsPods.Items[0].Name, api.NamespaceSystem))
}
func ensurePodsAreRemovedWhenNamespaceIsDeleted(f *framework.Framework) {
var err error
By("Creating a test namespace")
namespace, err := f.CreateNamespace("nsdeletetest", nil)
Expect(err).NotTo(HaveOccurred())
By("Waiting for a default service account to be provisioned in namespace")
err = framework.WaitForDefaultServiceAccountInNamespace(f.ClientSet, namespace.Name)
Expect(err).NotTo(HaveOccurred())
By("Creating a pod in the namespace")
pod := &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Name: "test-pod",
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "nginx",
Image: framework.GetPauseImageName(f.ClientSet),
},
},
},
}
pod, err = f.ClientSet.Core().Pods(namespace.Name).Create(pod)
Expect(err).NotTo(HaveOccurred())
By("Waiting for the pod to have running status")
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, pod))
By("Deleting the namespace")
err = f.ClientSet.Core().Namespaces().Delete(namespace.Name, nil)
Expect(err).NotTo(HaveOccurred())
By("Waiting for the namespace to be removed.")
maxWaitSeconds := int64(60) + *pod.Spec.TerminationGracePeriodSeconds
framework.ExpectNoError(wait.Poll(1*time.Second, time.Duration(maxWaitSeconds)*time.Second,
func() (bool, error) {
_, err = f.ClientSet.Core().Namespaces().Get(namespace.Name, metav1.GetOptions{})
if err != nil && errors.IsNotFound(err) {
return true, nil
}
return false, nil
}))
By("Verifying there is no pod in the namespace")
_, err = f.ClientSet.Core().Pods(namespace.Name).Get(pod.Name, metav1.GetOptions{})
Expect(err).To(HaveOccurred())
}
func verifyDNSPodIsRunning(f *framework.Framework) {
systemClient := f.ClientSet.Core().Pods(api.NamespaceSystem)
By("Waiting for DNS Service to be Running")
options := v1.ListOptions{LabelSelector: dnsServiceLabelSelector.String()}
dnsPods, err := systemClient.List(options)
if err != nil {
framework.Failf("Failed to list all dns service pods")
}
if len(dnsPods.Items) < 1 {
framework.Failf("No pods match the label selector %v", dnsServiceLabelSelector.String())
}
pod := dnsPods.Items[0]
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, &pod))
}
// initTestCase initializes spec resources (pv, pvc, and pod) and returns pointers to be consumed by the test
func initTestCase(f *framework.Framework, c clientset.Interface, pvConfig persistentVolumeConfig, ns, nodeName string) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
pv, pvc := createPVPVC(c, pvConfig, ns, false)
pod := makePod(ns, pvc.Name)
pod.Spec.NodeName = nodeName
framework.Logf("Creating nfs client Pod %s on node %s", pod.Name, nodeName)
pod, err := c.Core().Pods(ns).Create(pod)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForPodRunningInNamespace(c, pod)
Expect(err).NotTo(HaveOccurred())
pod, err = c.Core().Pods(ns).Get(pod.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
pvc, err = c.Core().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
pv, err = c.Core().PersistentVolumes().Get(pv.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
return pod, pv, pvc
}
func runPod(f *framework.Framework, name, image string) *api.Pod {
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: name,
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: name,
Image: image,
},
},
},
}
createdPod, err := f.Client.Pods(f.Namespace.Name).Create(pod)
framework.ExpectNoError(err)
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.Client, name, f.Namespace.Name))
return createdPod
}
func runPod(f *framework.Framework, name, image string) *v1.Pod {
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: name,
Image: image,
},
},
},
}
createdPod, err := f.ClientSet.Core().Pods(f.Namespace.Name).Create(pod)
framework.ExpectNoError(err)
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, createdPod))
return createdPod
}
}
bootstrapYaml := mkpath("redis-master.yaml")
sentinelServiceYaml := mkpath("redis-sentinel-service.yaml")
sentinelControllerYaml := mkpath("redis-sentinel-controller.yaml")
controllerYaml := mkpath("redis-controller.yaml")
bootstrapPodName := "redis-master"
redisRC := "redis"
sentinelRC := "redis-sentinel"
nsFlag := fmt.Sprintf("--namespace=%v", ns)
expectedOnServer := "The server is now ready to accept connections"
expectedOnSentinel := "+monitor master"
By("starting redis bootstrap")
framework.RunKubectlOrDie("create", "-f", bootstrapYaml, nsFlag)
err := framework.WaitForPodRunningInNamespace(c, bootstrapPodName, ns)
Expect(err).NotTo(HaveOccurred())
_, err = framework.LookForStringInLog(ns, bootstrapPodName, "master", expectedOnServer, serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
_, err = framework.LookForStringInLog(ns, bootstrapPodName, "sentinel", expectedOnSentinel, serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
By("setting up services and controllers")
framework.RunKubectlOrDie("create", "-f", sentinelServiceYaml, nsFlag)
framework.RunKubectlOrDie("create", "-f", sentinelControllerYaml, nsFlag)
framework.RunKubectlOrDie("create", "-f", controllerYaml, nsFlag)
label := labels.SelectorFromSet(labels.Set(map[string]string{"name": sentinelRC}))
err = framework.WaitForPodsWithLabelRunning(c, ns, label)
Expect(err).NotTo(HaveOccurred())
label = labels.SelectorFromSet(labels.Set(map[string]string{"name": redisRC}))
func testPodSELinuxLabeling(f *framework.Framework, hostIPC bool, hostPID bool) {
// Write and read a file with an empty_dir volume
// with a pod with the MCS label s0:c0,c1
pod := scTestPod(hostIPC, hostPID)
volumeName := "test-volume"
mountPath := "/mounted_volume"
pod.Spec.Containers[0].VolumeMounts = []api.VolumeMount{
{
Name: volumeName,
MountPath: mountPath,
},
}
pod.Spec.Volumes = []api.Volume{
{
Name: volumeName,
VolumeSource: api.VolumeSource{
EmptyDir: &api.EmptyDirVolumeSource{
Medium: api.StorageMediumDefault,
},
},
},
}
pod.Spec.SecurityContext.SELinuxOptions = &api.SELinuxOptions{
Level: "s0:c0,c1",
}
pod.Spec.Containers[0].Command = []string{"sleep", "6000"}
client := f.ClientSet.Core().Pods(f.Namespace.Name)
pod, err := client.Create(pod)
framework.ExpectNoError(err, "Error creating pod %v", pod)
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, pod))
testContent := "hello"
testFilePath := mountPath + "/TEST"
err = f.WriteFileViaContainer(pod.Name, pod.Spec.Containers[0].Name, testFilePath, testContent)
Expect(err).To(BeNil())
content, err := f.ReadFileViaContainer(pod.Name, pod.Spec.Containers[0].Name, testFilePath)
Expect(err).To(BeNil())
Expect(content).To(ContainSubstring(testContent))
foundPod, err := f.ClientSet.Core().Pods(f.Namespace.Name).Get(pod.Name)
Expect(err).NotTo(HaveOccurred())
// Confirm that the file can be accessed from a second
// pod using host_path with the same MCS label
volumeHostPath := fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/%s", framework.TestContext.KubeVolumeDir, foundPod.UID, volumeName)
By(fmt.Sprintf("confirming a container with the same label can read the file under --volume-dir=%s", framework.TestContext.KubeVolumeDir))
pod = scTestPod(hostIPC, hostPID)
pod.Spec.NodeName = foundPod.Spec.NodeName
volumeMounts := []api.VolumeMount{
{
Name: volumeName,
MountPath: mountPath,
},
}
volumes := []api.Volume{
{
Name: volumeName,
VolumeSource: api.VolumeSource{
HostPath: &api.HostPathVolumeSource{
Path: volumeHostPath,
},
},
},
}
pod.Spec.Containers[0].VolumeMounts = volumeMounts
pod.Spec.Volumes = volumes
pod.Spec.Containers[0].Command = []string{"cat", testFilePath}
pod.Spec.SecurityContext.SELinuxOptions = &api.SELinuxOptions{
Level: "s0:c0,c1",
}
f.TestContainerOutput("Pod with same MCS label reading test file", pod, 0, []string{testContent})
// Confirm that the same pod with a different MCS
// label cannot access the volume
pod = scTestPod(hostIPC, hostPID)
pod.Spec.Volumes = volumes
pod.Spec.Containers[0].VolumeMounts = volumeMounts
pod.Spec.Containers[0].Command = []string{"sleep", "6000"}
pod.Spec.SecurityContext.SELinuxOptions = &api.SELinuxOptions{
Level: "s0:c2,c3",
}
_, err = client.Create(pod)
framework.ExpectNoError(err, "Error creating pod %v", pod)
err = f.WaitForPodRunning(pod.Name)
framework.ExpectNoError(err, "Error waiting for pod to run %v", pod)
content, err = f.ReadFileViaContainer(pod.Name, "test-container", testFilePath)
Expect(content).NotTo(ContainSubstring(testContent))
}
Annotations: map[string]string{
"k8s.mesosphere.io/roles": "public",
},
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: podName,
Image: "beta.gcr.io/google_containers/pause-amd64:3.0",
},
},
},
})
framework.ExpectNoError(err)
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(c, podName, ns))
pod, err := c.Pods(ns).Get(podName)
framework.ExpectNoError(err)
nodeClient := f.Client.Nodes()
// schedule onto node with rack=2 being assigned to the "public" role
rack2 := labels.SelectorFromSet(map[string]string{
"k8s.mesosphere.io/attribute-rack": "2",
})
options := api.ListOptions{LabelSelector: rack2}
nodes, err := nodeClient.List(options)
framework.ExpectNoError(err)
Expect(nodes.Items[0].Name).To(Equal(pod.Spec.NodeName))
})
},
},
},
RestartPolicy: api.RestartPolicyNever,
},
}
defer func() {
By("Deleting the pod")
f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0))
}()
By("Creating the pod")
_, err = f.Client.Pods(f.Namespace.Name).Create(pod)
Expect(err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.Client, pod.Name, f.Namespace.Name))
pollLogs := func() (string, error) {
return framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, containerName)
}
Eventually(pollLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("value-1"))
By(fmt.Sprintf("Updating configmap %v", configMap.Name))
configMap.ResourceVersion = "" // to force update
configMap.Data["data-1"] = "value-2"
_, err = f.Client.ConfigMaps(f.Namespace.Name).Update(configMap)
Expect(err).NotTo(HaveOccurred())
By("waiting to observe update in volume")
Eventually(pollLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("value-2"))
// Starts a container specified by config.serverImage and exports all
// config.serverPorts from it. The returned pod should be used to get the server
// IP address and create appropriate VolumeSource.
func startVolumeServer(client *client.Client, config VolumeTestConfig) *api.Pod {
podClient := client.Pods(config.namespace)
portCount := len(config.serverPorts)
serverPodPorts := make([]api.ContainerPort, portCount)
for i := 0; i < portCount; i++ {
portName := fmt.Sprintf("%s-%d", config.prefix, i)
serverPodPorts[i] = api.ContainerPort{
Name: portName,
ContainerPort: int32(config.serverPorts[i]),
Protocol: api.ProtocolTCP,
}
}
volumeCount := len(config.volumes)
volumes := make([]api.Volume, volumeCount)
mounts := make([]api.VolumeMount, volumeCount)
i := 0
for src, dst := range config.volumes {
mountName := fmt.Sprintf("path%d", i)
volumes[i].Name = mountName
volumes[i].VolumeSource.HostPath = &api.HostPathVolumeSource{
Path: src,
}
mounts[i].Name = mountName
mounts[i].ReadOnly = false
mounts[i].MountPath = dst
i++
}
By(fmt.Sprint("creating ", config.prefix, " server pod"))
privileged := new(bool)
*privileged = true
serverPod := &api.Pod{
TypeMeta: unversioned.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: api.ObjectMeta{
Name: config.prefix + "-server",
Labels: map[string]string{
"role": config.prefix + "-server",
},
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: config.prefix + "-server",
Image: config.serverImage,
SecurityContext: &api.SecurityContext{
Privileged: privileged,
},
Args: config.serverArgs,
Ports: serverPodPorts,
VolumeMounts: mounts,
},
},
Volumes: volumes,
},
}
serverPod, err := podClient.Create(serverPod)
framework.ExpectNoError(err, "Failed to create %s pod: %v", serverPod.Name, err)
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(client, serverPod))
By("locating the server pod")
pod, err := podClient.Get(serverPod.Name)
framework.ExpectNoError(err, "Cannot locate the server pod %v: %v", serverPod.Name, err)
By("sleeping a bit to give the server time to start")
time.Sleep(20 * time.Second)
return pod
}
// Start a client pod using given VolumeSource (exported by startVolumeServer())
// and check that the pod sees the data from the server pod.
func testVolumeClient(client *client.Client, config VolumeTestConfig, volume api.VolumeSource, fsGroup *int64, expectedContent string) {
By(fmt.Sprint("starting ", config.prefix, " client"))
clientPod := &api.Pod{
TypeMeta: unversioned.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: api.ObjectMeta{
Name: config.prefix + "-client",
Labels: map[string]string{
"role": config.prefix + "-client",
},
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: config.prefix + "-client",
Image: "gcr.io/google_containers/busybox:1.24",
WorkingDir: "/opt",
// An imperative and easily debuggable container which reads vol contents for
// us to scan in the tests or by eye.
// We expect that /opt is empty in the minimal containers which we use in this test.
Command: []string{
"/bin/sh",
"-c",
"while true ; do cat /opt/index.html ; sleep 2 ; ls -altrh /opt/ ; sleep 2 ; done ",
},
VolumeMounts: []api.VolumeMount{
{
Name: config.prefix + "-volume",
MountPath: "/opt/",
},
},
},
},
SecurityContext: &api.PodSecurityContext{
SELinuxOptions: &api.SELinuxOptions{
Level: "s0:c0,c1",
},
},
Volumes: []api.Volume{
{
Name: config.prefix + "-volume",
VolumeSource: volume,
},
},
},
}
podsNamespacer := client.Pods(config.namespace)
if fsGroup != nil {
clientPod.Spec.SecurityContext.FSGroup = fsGroup
}
clientPod, err := podsNamespacer.Create(clientPod)
if err != nil {
framework.Failf("Failed to create %s pod: %v", clientPod.Name, err)
}
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(client, clientPod))
By("Checking that text file contents are perfect.")
_, err = framework.LookForStringInPodExec(config.namespace, clientPod.Name, []string{"cat", "/opt/index.html"}, expectedContent, time.Minute)
Expect(err).NotTo(HaveOccurred(), "failed: finding the contents of the mounted file.")
if fsGroup != nil {
By("Checking fsGroup is correct.")
_, err = framework.LookForStringInPodExec(config.namespace, clientPod.Name, []string{"ls", "-ld", "/opt"}, strconv.Itoa(int(*fsGroup)), time.Minute)
Expect(err).NotTo(HaveOccurred(), "failed: getting the right priviliges in the file %v", int(*fsGroup))
}
}
// Ensures that elasticsearch is running and ready to serve requests
func checkElasticsearchReadiness(f *framework.Framework) error {
// Check for the existence of the Elasticsearch service.
By("Checking the Elasticsearch service exists.")
s := f.ClientSet.Core().Services(api.NamespaceSystem)
// Make a few attempts to connect. This makes the test robust against
// being run as the first e2e test just after the e2e cluster has been created.
var err error
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
if _, err = s.Get("elasticsearch-logging"); err == nil {
break
}
framework.Logf("Attempt to check for the existence of the Elasticsearch service failed after %v", time.Since(start))
}
Expect(err).NotTo(HaveOccurred())
// Wait for the Elasticsearch pods to enter the running state.
By("Checking to make sure the Elasticsearch pods are running")
label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "elasticsearch-logging"}))
options := api.ListOptions{LabelSelector: label}
pods, err := f.ClientSet.Core().Pods(api.NamespaceSystem).List(options)
Expect(err).NotTo(HaveOccurred())
for _, pod := range pods.Items {
err = framework.WaitForPodRunningInNamespace(f.ClientSet, &pod)
Expect(err).NotTo(HaveOccurred())
}
By("Checking to make sure we are talking to an Elasticsearch service.")
// Perform a few checks to make sure this looks like an Elasticsearch cluster.
var statusCode int
err = nil
var body []byte
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(10 * time.Second) {
proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
if errProxy != nil {
framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
continue
}
// Query against the root URL for Elasticsearch.
response := proxyRequest.Namespace(api.NamespaceSystem).
Name("elasticsearch-logging").
Do()
err = response.Error()
response.StatusCode(&statusCode)
if err != nil {
framework.Logf("After %v proxy call to elasticsearch-loigging failed: %v", time.Since(start), err)
continue
}
if int(statusCode) != 200 {
framework.Logf("After %v Elasticsearch cluster has a bad status: %v", time.Since(start), statusCode)
continue
}
break
}
Expect(err).NotTo(HaveOccurred())
if int(statusCode) != 200 {
framework.Failf("Elasticsearch cluster has a bad status: %v", statusCode)
}
// Now assume we really are talking to an Elasticsearch instance.
// Check the cluster health.
By("Checking health of Elasticsearch service.")
healthy := false
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
if errProxy != nil {
framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
continue
}
body, err = proxyRequest.Namespace(api.NamespaceSystem).
Name("elasticsearch-logging").
Suffix("_cluster/health").
Param("level", "indices").
DoRaw()
if err != nil {
continue
}
health := make(map[string]interface{})
err := json.Unmarshal(body, &health)
if err != nil {
framework.Logf("Bad json response from elasticsearch: %v", err)
continue
}
statusIntf, ok := health["status"]
if !ok {
framework.Logf("No status field found in cluster health response: %v", health)
continue
}
status := statusIntf.(string)
if status != "green" && status != "yellow" {
framework.Logf("Cluster health has bad status: %v", health)
continue
}
if err == nil && ok {
healthy = true
break
}
}
if !healthy {
return fmt.Errorf("After %v elasticsearch cluster is not healthy", graceTime)
}
return nil
}
// ClusterLevelLoggingWithElasticsearch is an end to end test for cluster level logging.
func ClusterLevelLoggingWithElasticsearch(f *framework.Framework) {
// graceTime is how long to keep retrying requests for status information.
const graceTime = 5 * time.Minute
// ingestionTimeout is how long to keep retrying to wait for all the
// logs to be ingested.
const ingestionTimeout = 10 * time.Minute
// Check for the existence of the Elasticsearch service.
By("Checking the Elasticsearch service exists.")
s := f.Client.Services(api.NamespaceSystem)
// Make a few attempts to connect. This makes the test robust against
// being run as the first e2e test just after the e2e cluster has been created.
var err error
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
if _, err = s.Get("elasticsearch-logging"); err == nil {
break
}
framework.Logf("Attempt to check for the existence of the Elasticsearch service failed after %v", time.Since(start))
}
Expect(err).NotTo(HaveOccurred())
// Wait for the Elasticsearch pods to enter the running state.
By("Checking to make sure the Elasticsearch pods are running")
label := labels.SelectorFromSet(labels.Set(map[string]string{k8sAppKey: esValue}))
options := api.ListOptions{LabelSelector: label}
pods, err := f.Client.Pods(api.NamespaceSystem).List(options)
Expect(err).NotTo(HaveOccurred())
for _, pod := range pods.Items {
err = framework.WaitForPodRunningInNamespace(f.Client, &pod)
Expect(err).NotTo(HaveOccurred())
}
By("Checking to make sure we are talking to an Elasticsearch service.")
// Perform a few checks to make sure this looks like an Elasticsearch cluster.
var statusCode float64
var esResponse map[string]interface{}
err = nil
var body []byte
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(10 * time.Second) {
proxyRequest, errProxy := framework.GetServicesProxyRequest(f.Client, f.Client.Get())
if errProxy != nil {
framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
continue
}
// Query against the root URL for Elasticsearch.
body, err = proxyRequest.Namespace(api.NamespaceSystem).
Name("elasticsearch-logging").
DoRaw()
if err != nil {
framework.Logf("After %v proxy call to elasticsearch-loigging failed: %v", time.Since(start), err)
continue
}
esResponse, err = bodyToJSON(body)
if err != nil {
framework.Logf("After %v failed to convert Elasticsearch JSON response %v to map[string]interface{}: %v", time.Since(start), string(body), err)
continue
}
statusIntf, ok := esResponse["status"]
if !ok {
framework.Logf("After %v Elasticsearch response has no status field: %v", time.Since(start), esResponse)
continue
}
statusCode, ok = statusIntf.(float64)
if !ok {
// Assume this is a string returning Failure. Retry.
framework.Logf("After %v expected status to be a float64 but got %v of type %T", time.Since(start), statusIntf, statusIntf)
continue
}
if int(statusCode) != 200 {
framework.Logf("After %v Elasticsearch cluster has a bad status: %v", time.Since(start), statusCode)
continue
}
break
}
Expect(err).NotTo(HaveOccurred())
if int(statusCode) != 200 {
framework.Failf("Elasticsearch cluster has a bad status: %v", statusCode)
}
// Check to see if have a cluster_name field.
clusterName, ok := esResponse["cluster_name"]
if !ok {
framework.Failf("No cluster_name field in Elasticsearch response: %v", esResponse)
}
if clusterName != "kubernetes-logging" {
framework.Failf("Connected to wrong cluster %q (expecting kubernetes_logging)", clusterName)
}
// Now assume we really are talking to an Elasticsearch instance.
// Check the cluster health.
By("Checking health of Elasticsearch service.")
healthy := false
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
proxyRequest, errProxy := framework.GetServicesProxyRequest(f.Client, f.Client.Get())
if errProxy != nil {
framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
continue
}
body, err = proxyRequest.Namespace(api.NamespaceSystem).
Name("elasticsearch-logging").
Suffix("_cluster/health").
Param("level", "indices").
DoRaw()
if err != nil {
continue
}
health, err := bodyToJSON(body)
if err != nil {
framework.Logf("Bad json response from elasticsearch: %v", err)
continue
}
statusIntf, ok := health["status"]
if !ok {
framework.Logf("No status field found in cluster health response: %v", health)
continue
}
status := statusIntf.(string)
if status != "green" && status != "yellow" {
framework.Logf("Cluster health has bad status: %v", health)
continue
}
if err == nil && ok {
healthy = true
break
}
}
if !healthy {
framework.Failf("After %v elasticsearch cluster is not healthy", graceTime)
}
// Obtain a list of nodes so we can place one synthetic logger on each node.
nodes := framework.GetReadySchedulableNodesOrDie(f.Client)
nodeCount := len(nodes.Items)
if nodeCount == 0 {
framework.Failf("Failed to find any nodes")
}
framework.Logf("Found %d nodes.", len(nodes.Items))
// Filter out unhealthy nodes.
// Previous tests may have cause failures of some nodes. Let's skip
// 'Not Ready' nodes, just in case (there is no need to fail the test).
framework.FilterNodes(nodes, func(node api.Node) bool {
return framework.IsNodeConditionSetAsExpected(&node, api.NodeReady, true)
})
if len(nodes.Items) < 2 {
framework.Failf("Less than two nodes were found Ready: %d", len(nodes.Items))
}
framework.Logf("Found %d healthy nodes.", len(nodes.Items))
// Wait for the Fluentd pods to enter the running state.
By("Checking to make sure the Fluentd pod are running on each healthy node")
label = labels.SelectorFromSet(labels.Set(map[string]string{k8sAppKey: fluentdValue}))
options = api.ListOptions{LabelSelector: label}
fluentdPods, err := f.Client.Pods(api.NamespaceSystem).List(options)
Expect(err).NotTo(HaveOccurred())
for _, pod := range fluentdPods.Items {
if nodeInNodeList(pod.Spec.NodeName, nodes) {
err = framework.WaitForPodRunningInNamespace(f.Client, &pod)
Expect(err).NotTo(HaveOccurred())
}
}
// Check if each healthy node has fluentd running on it
for _, node := range nodes.Items {
exists := false
for _, pod := range fluentdPods.Items {
if pod.Spec.NodeName == node.Name {
exists = true
break
}
}
if !exists {
framework.Failf("Node %v does not have fluentd pod running on it.", node.Name)
}
}
// Create a unique root name for the resources in this test to permit
// parallel executions of this test.
// Use a unique namespace for the resources created in this test.
ns := f.Namespace.Name
name := "synthlogger"
// Form a unique name to taint log lines to be collected.
// Replace '-' characters with '_' to prevent the analyzer from breaking apart names.
taintName := strings.Replace(ns+name, "-", "_", -1)
framework.Logf("Tainting log lines with %v", taintName)
// podNames records the names of the synthetic logging pods that are created in the
// loop below.
var podNames []string
// countTo is the number of log lines emitted (and checked) for each synthetic logging pod.
const countTo = 100
// Instantiate a synthetic logger pod on each node.
for i, node := range nodes.Items {
podName := fmt.Sprintf("%s-%d", name, i)
_, err := f.Client.Pods(ns).Create(&api.Pod{
ObjectMeta: api.ObjectMeta{
Name: podName,
Labels: map[string]string{"name": name},
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "synth-logger",
Image: "gcr.io/google_containers/ubuntu:14.04",
// notice: the subshell syntax is escaped with `$$`
Command: []string{"bash", "-c", fmt.Sprintf("i=0; while ((i < %d)); do echo \"%d %s $i %s\"; i=$$(($i+1)); done", countTo, i, taintName, podName)},
},
},
NodeName: node.Name,
RestartPolicy: api.RestartPolicyNever,
},
})
Expect(err).NotTo(HaveOccurred())
podNames = append(podNames, podName)
}
// Cleanup the pods when we are done.
defer func() {
for _, pod := range podNames {
if err = f.Client.Pods(ns).Delete(pod, nil); err != nil {
framework.Logf("Failed to delete pod %s: %v", pod, err)
}
}
}()
// Wait for the synthetic logging pods to finish.
By("Waiting for the pods to succeed.")
for _, pod := range podNames {
err = framework.WaitForPodSuccessInNamespace(f.Client, pod, "synth-logger", ns)
Expect(err).NotTo(HaveOccurred())
}
// Make several attempts to observe the logs ingested into Elasticsearch.
By("Checking all the log lines were ingested into Elasticsearch")
totalMissing := 0
expected := nodeCount * countTo
missingPerNode := []int{}
for start := time.Now(); time.Since(start) < ingestionTimeout; time.Sleep(25 * time.Second) {
// Debugging code to report the status of the elasticsearch logging endpoints.
selector := labels.Set{k8sAppKey: esValue}.AsSelector()
options := api.ListOptions{LabelSelector: selector}
esPods, err := f.Client.Pods(api.NamespaceSystem).List(options)
if err != nil {
framework.Logf("Attempt to list Elasticsearch nodes encountered a problem -- may retry: %v", err)
continue
} else {
for i, pod := range esPods.Items {
framework.Logf("pod %d: %s PodIP %s phase %s condition %+v", i, pod.Name, pod.Status.PodIP, pod.Status.Phase,
pod.Status.Conditions)
}
}
proxyRequest, errProxy := framework.GetServicesProxyRequest(f.Client, f.Client.Get())
if errProxy != nil {
framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
continue
}
// Ask Elasticsearch to return all the log lines that were tagged with the underscore
// version of the name. Ask for twice as many log lines as we expect to check for
// duplication bugs.
body, err = proxyRequest.Namespace(api.NamespaceSystem).
Name("elasticsearch-logging").
Suffix("_search").
Param("q", fmt.Sprintf("log:%s", taintName)).
Param("size", strconv.Itoa(2*expected)).
DoRaw()
if err != nil {
framework.Logf("After %v failed to make proxy call to elasticsearch-logging: %v", time.Since(start), err)
continue
}
response, err := bodyToJSON(body)
if err != nil {
framework.Logf("After %v failed to unmarshal response: %v", time.Since(start), err)
framework.Logf("Body: %s", string(body))
continue
}
hits, ok := response["hits"].(map[string]interface{})
if !ok {
framework.Logf("response[hits] not of the expected type: %T", response["hits"])
continue
}
totalF, ok := hits["total"].(float64)
if !ok {
framework.Logf("After %v hits[total] not of the expected type: %T", time.Since(start), hits["total"])
continue
}
total := int(totalF)
if total != expected {
framework.Logf("After %v expecting to find %d log lines but saw %d", time.Since(start), expected, total)
}
h, ok := hits["hits"].([]interface{})
if !ok {
framework.Logf("After %v hits not of the expected type: %T", time.Since(start), hits["hits"])
continue
}
// Initialize data-structure for observing counts.
observed := make([][]int, nodeCount)
for i := range observed {
observed[i] = make([]int, countTo)
}
// Iterate over the hits and populate the observed array.
for _, e := range h {
l, ok := e.(map[string]interface{})
if !ok {
framework.Logf("element of hit not of expected type: %T", e)
continue
}
source, ok := l["_source"].(map[string]interface{})
if !ok {
framework.Logf("_source not of the expected type: %T", l["_source"])
continue
}
msg, ok := source["log"].(string)
if !ok {
framework.Logf("log not of the expected type: %T", source["log"])
continue
}
words := strings.Split(msg, " ")
if len(words) != 4 {
framework.Logf("Malformed log line: %s", msg)
continue
}
n, err := strconv.ParseUint(words[0], 10, 0)
if err != nil {
framework.Logf("Expecting numer of node as first field of %s", msg)
continue
}
if n < 0 || int(n) >= nodeCount {
framework.Logf("Node count index out of range: %d", nodeCount)
continue
}
index, err := strconv.ParseUint(words[2], 10, 0)
if err != nil {
framework.Logf("Expecting number as third field of %s", msg)
continue
}
if index < 0 || index >= countTo {
framework.Logf("Index value out of range: %d", index)
continue
}
if words[1] != taintName {
framework.Logf("Elasticsearch query return unexpected log line: %s", msg)
continue
}
// Record the observation of a log line from node n at the given index.
observed[n][index]++
}
// Make sure we correctly observed the expected log lines from each node.
totalMissing = 0
missingPerNode = make([]int, nodeCount)
incorrectCount := false
for n := range observed {
for i, c := range observed[n] {
if c == 0 {
totalMissing++
missingPerNode[n]++
}
if c < 0 || c > 1 {
framework.Logf("Got incorrect count for node %d index %d: %d", n, i, c)
incorrectCount = true
}
}
}
if incorrectCount {
framework.Logf("After %v es still return duplicated log lines", time.Since(start))
continue
}
if totalMissing != 0 {
framework.Logf("After %v still missing %d log lines", time.Since(start), totalMissing)
continue
}
framework.Logf("After %s found all %d log lines", time.Since(start), expected)
return
}
for n := range missingPerNode {
if missingPerNode[n] > 0 {
framework.Logf("Node %d %s is missing %d logs", n, nodes.Items[n].Name, missingPerNode[n])
opts := &api.PodLogOptions{}
body, err = f.Client.Pods(ns).GetLogs(podNames[n], opts).DoRaw()
if err != nil {
framework.Logf("Cannot get logs from pod %v", podNames[n])
continue
}
framework.Logf("Pod %s has the following logs: %s", podNames[n], body)
for _, pod := range fluentdPods.Items {
if pod.Spec.NodeName == nodes.Items[n].Name {
body, err = f.Client.Pods(api.NamespaceSystem).GetLogs(pod.Name, opts).DoRaw()
if err != nil {
framework.Logf("Cannot get logs from pod %v", pod.Name)
break
}
framework.Logf("Fluentd Pod %s on node %s has the following logs: %s", pod.Name, nodes.Items[n].Name, body)
break
}
}
}
}
framework.Failf("Failed to find all %d log lines", expected)
}
func testPreStop(c clientset.Interface, ns string) {
// This is the server that will receive the preStop notification
podDescr := &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Name: "server",
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "server",
Image: "gcr.io/google_containers/nettest:1.7",
Ports: []v1.ContainerPort{{ContainerPort: 8080}},
},
},
},
}
By(fmt.Sprintf("Creating server pod %s in namespace %s", podDescr.Name, ns))
podDescr, err := c.Core().Pods(ns).Create(podDescr)
framework.ExpectNoError(err, fmt.Sprintf("creating pod %s", podDescr.Name))
// At the end of the test, clean up by removing the pod.
defer func() {
By("Deleting the server pod")
c.Core().Pods(ns).Delete(podDescr.Name, nil)
}()
By("Waiting for pods to come up.")
err = framework.WaitForPodRunningInNamespace(c, podDescr)
framework.ExpectNoError(err, "waiting for server pod to start")
val := "{\"Source\": \"prestop\"}"
podOut, err := c.Core().Pods(ns).Get(podDescr.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "getting pod info")
preStopDescr := &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Name: "tester",
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "tester",
Image: "gcr.io/google_containers/busybox:1.24",
Command: []string{"sleep", "600"},
Lifecycle: &v1.Lifecycle{
PreStop: &v1.Handler{
Exec: &v1.ExecAction{
Command: []string{
"wget", "-O-", "--post-data=" + val, fmt.Sprintf("http://%s:8080/write", podOut.Status.PodIP),
},
},
},
},
},
},
},
}
By(fmt.Sprintf("Creating tester pod %s in namespace %s", preStopDescr.Name, ns))
preStopDescr, err = c.Core().Pods(ns).Create(preStopDescr)
framework.ExpectNoError(err, fmt.Sprintf("creating pod %s", preStopDescr.Name))
deletePreStop := true
// At the end of the test, clean up by removing the pod.
defer func() {
if deletePreStop {
By("Deleting the tester pod")
c.Core().Pods(ns).Delete(preStopDescr.Name, nil)
}
}()
err = framework.WaitForPodRunningInNamespace(c, preStopDescr)
framework.ExpectNoError(err, "waiting for tester pod to start")
// Delete the pod with the preStop handler.
By("Deleting pre-stop pod")
if err := c.Core().Pods(ns).Delete(preStopDescr.Name, nil); err == nil {
deletePreStop = false
}
framework.ExpectNoError(err, fmt.Sprintf("deleting pod: %s", preStopDescr.Name))
// Validate that the server received the web poke.
err = wait.Poll(time.Second*5, time.Second*60, func() (bool, error) {
subResourceProxyAvailable, err := framework.ServerVersionGTE(framework.SubResourcePodProxyVersion, c.Discovery())
if err != nil {
return false, err
}
var body []byte
if subResourceProxyAvailable {
body, err = c.Core().RESTClient().Get().
Namespace(ns).
Resource("pods").
SubResource("proxy").
Name(podDescr.Name).
Suffix("read").
DoRaw()
} else {
body, err = c.Core().RESTClient().Get().
Prefix("proxy").
Namespace(ns).
Resource("pods").
Name(podDescr.Name).
Suffix("read").
DoRaw()
}
if err != nil {
By(fmt.Sprintf("Error validating prestop: %v", err))
} else {
framework.Logf("Saw: %s", string(body))
state := State{}
err := json.Unmarshal(body, &state)
if err != nil {
framework.Logf("Error parsing: %v", err)
return false, nil
}
if state.Received["prestop"] != 0 {
return true, nil
}
}
return false, nil
})
framework.ExpectNoError(err, "validating pre-stop.")
}