// Blocks outgoing network traffic on 'node'. Then verifies that 'podNameToDisappear', // that belongs to replication controller 'rcName', really disappeared. // Finally, it checks that the replication controller recreates the // pods on another node and that now the number of replicas is equal 'replicas'. // At the end (even in case of errors), the network traffic is brought back to normal. // This function executes commands on a node so it will work only for some // environments. func performTemporaryNetworkFailure(c *client.Client, ns, rcName string, replicas int32, podNameToDisappear string, node *api.Node) { host := getNodeExternalIP(node) master := getMaster(c) By(fmt.Sprintf("block network traffic from node %s to the master", node.Name)) defer func() { // This code will execute even if setting the iptables rule failed. // It is on purpose because we may have an error even if the new rule // had been inserted. (yes, we could look at the error code and ssh error // separately, but I prefer to stay on the safe side). By(fmt.Sprintf("Unblock network traffic from node %s to the master", node.Name)) framework.UnblockNetwork(host, master) }() framework.Logf("Waiting %v to ensure node %s is ready before beginning test...", resizeNodeReadyTimeout, node.Name) if !framework.WaitForNodeToBe(c, node.Name, api.NodeReady, true, resizeNodeReadyTimeout) { framework.Failf("Node %s did not become ready within %v", node.Name, resizeNodeReadyTimeout) } framework.BlockNetwork(host, master) framework.Logf("Waiting %v for node %s to be not ready after simulated network failure", resizeNodeNotReadyTimeout, node.Name) if !framework.WaitForNodeToBe(c, node.Name, api.NodeReady, false, resizeNodeNotReadyTimeout) { framework.Failf("Node %s did not become not-ready within %v", node.Name, resizeNodeNotReadyTimeout) } framework.Logf("Waiting for pod %s to be removed", podNameToDisappear) err := framework.WaitForRCPodToDisappear(c, ns, rcName, podNameToDisappear) Expect(err).NotTo(HaveOccurred()) By("verifying whether the pod from the unreachable node is recreated") err = framework.VerifyPods(c, ns, rcName, true, replicas) Expect(err).NotTo(HaveOccurred()) // network traffic is unblocked in a deferred function }
func validate(f *framework.Framework, svcNameWant, rcNameWant string, ingress api.LoadBalancerIngress, podsWant int) error { framework.Logf("Beginning cluster validation") // Verify RC. rcs, err := f.Client.ReplicationControllers(f.Namespace.Name).List(api.ListOptions{}) if err != nil { return fmt.Errorf("error listing RCs: %v", err) } if len(rcs.Items) != 1 { return fmt.Errorf("wanted 1 RC with name %s, got %d", rcNameWant, len(rcs.Items)) } if got := rcs.Items[0].Name; got != rcNameWant { return fmt.Errorf("wanted RC name %q, got %q", rcNameWant, got) } // Verify pods. if err := framework.VerifyPods(f.Client, f.Namespace.Name, rcNameWant, false, podsWant); err != nil { return fmt.Errorf("failed to find %d %q pods: %v", podsWant, rcNameWant, err) } // Verify service. svc, err := f.Client.Services(f.Namespace.Name).Get(svcNameWant) if err != nil { return fmt.Errorf("error getting service %s: %v", svcNameWant, err) } if svcNameWant != svc.Name { return fmt.Errorf("wanted service name %q, got %q", svcNameWant, svc.Name) } // TODO(mikedanese): Make testLoadBalancerReachable return an error. testLoadBalancerReachable(ingress, 80) framework.Logf("Cluster validation succeeded") return nil }
func testDeploymentLabelAdopted(f *framework.Framework) { ns := f.Namespace.Name // TODO: remove unversionedClient when the refactoring is done. Currently some // functions like verifyPod still expects a unversioned#Client. unversionedClient := f.Client c := adapter.FromUnversionedClient(unversionedClient) // Create nginx pods. podName := "nginx" podLabels := map[string]string{"name": podName} rsName := "test-adopted-controller" replicas := 3 image := nginxImage _, err := c.Extensions().ReplicaSets(ns).Create(newRS(rsName, replicas, podLabels, podName, image)) Expect(err).NotTo(HaveOccurred()) // Verify that the required pods have come up. err = framework.VerifyPods(unversionedClient, ns, podName, false, 3) if err != nil { framework.Logf("error in waiting for pods to come up: %s", err) Expect(err).NotTo(HaveOccurred()) } // Create a nginx deployment to adopt the old rs. deploymentName := "test-adopted-deployment" framework.Logf("Creating deployment %s", deploymentName) _, err = c.Extensions().Deployments(ns).Create(newDeployment(deploymentName, replicas, podLabels, podName, image, extensions.RollingUpdateDeploymentStrategyType, nil)) Expect(err).NotTo(HaveOccurred()) defer stopDeployment(c, f.Client, ns, deploymentName) // Wait for it to be updated to revision 1 err = framework.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, "1", image) Expect(err).NotTo(HaveOccurred()) // The RS and pods should be relabeled before the status is updated by syncRollingUpdateDeployment err = framework.WaitForDeploymentStatus(c, ns, deploymentName, replicas, replicas-1, replicas+1, 0) Expect(err).NotTo(HaveOccurred()) // There should be no old RSs (overlapping RS) deployment, err := c.Extensions().Deployments(ns).Get(deploymentName) Expect(err).NotTo(HaveOccurred()) oldRSs, allOldRSs, err := deploymentutil.GetOldReplicaSets(deployment, c) Expect(err).NotTo(HaveOccurred()) Expect(len(oldRSs)).Should(Equal(0)) Expect(len(allOldRSs)).Should(Equal(0)) // New RS should contain pod-template-hash in its selector, label, and template label newRS, err := deploymentutil.GetNewReplicaSet(deployment, c) Expect(err).NotTo(HaveOccurred()) err = framework.CheckRSHashLabel(newRS) Expect(err).NotTo(HaveOccurred()) // All pods targeted by the deployment should contain pod-template-hash in their labels, and there should be only 3 pods selector, err := unversioned.LabelSelectorAsSelector(deployment.Spec.Selector) Expect(err).NotTo(HaveOccurred()) options := api.ListOptions{LabelSelector: selector} pods, err := c.Core().Pods(ns).List(options) Expect(err).NotTo(HaveOccurred()) err = framework.CheckPodHashLabel(pods) Expect(err).NotTo(HaveOccurred()) Expect(len(pods.Items)).Should(Equal(replicas)) }
func testRecreateDeployment(f *framework.Framework) { ns := f.Namespace.Name // TODO: remove unversionedClient when the refactoring is done. Currently some // functions like verifyPod still expects a unversioned#Client. unversionedClient := f.Client c := adapter.FromUnversionedClient(unversionedClient) // Create nginx pods. deploymentPodLabels := map[string]string{"name": "sample-pod-3"} rsPodLabels := map[string]string{ "name": "sample-pod-3", "pod": nginxImageName, } rsName := "test-recreate-controller" replicas := 3 _, err := c.Extensions().ReplicaSets(ns).Create(newRS(rsName, replicas, rsPodLabels, nginxImageName, nginxImage)) Expect(err).NotTo(HaveOccurred()) // Verify that the required pods have come up. err = framework.VerifyPods(unversionedClient, ns, "sample-pod-3", false, 3) if err != nil { framework.Logf("error in waiting for pods to come up: %s", err) Expect(err).NotTo(HaveOccurred()) } // Create a deployment to delete nginx pods and instead bring up redis pods. deploymentName := "test-recreate-deployment" framework.Logf("Creating deployment %s", deploymentName) _, err = c.Extensions().Deployments(ns).Create(newDeployment(deploymentName, replicas, deploymentPodLabels, redisImageName, redisImage, extensions.RecreateDeploymentStrategyType, nil)) Expect(err).NotTo(HaveOccurred()) defer stopDeployment(c, f.Client, ns, deploymentName) // Wait for it to be updated to revision 1 err = framework.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, "1", redisImage) Expect(err).NotTo(HaveOccurred()) err = framework.WaitForDeploymentStatus(c, ns, deploymentName, replicas, 0, replicas, 0) Expect(err).NotTo(HaveOccurred()) // Verify that the pods were scaled up and down as expected. We use events to verify that. deployment, err := c.Extensions().Deployments(ns).Get(deploymentName) Expect(err).NotTo(HaveOccurred()) framework.WaitForEvents(unversionedClient, ns, deployment, 2) events, err := c.Core().Events(ns).Search(deployment) if err != nil { framework.Logf("error in listing events: %s", err) Expect(err).NotTo(HaveOccurred()) } // There should be 2 events, one to scale up the new ReplicaSet and then to scale down the old ReplicaSet. Expect(len(events.Items)).Should(Equal(2)) newRS, err := deploymentutil.GetNewReplicaSet(deployment, c) Expect(err).NotTo(HaveOccurred()) Expect(newRS).NotTo(Equal(nil)) Expect(events.Items[0].Message).Should(Equal(fmt.Sprintf("Scaled down replica set %s to 0", rsName))) Expect(events.Items[1].Message).Should(Equal(fmt.Sprintf("Scaled up replica set %s to 3", newRS.Name))) }
func testRollingUpdateDeployment(f *framework.Framework) { ns := f.Namespace.Name // TODO: remove unversionedClient when the refactoring is done. Currently some // functions like verifyPod still expects a unversioned#Client. unversionedClient := f.Client c := adapter.FromUnversionedClient(unversionedClient) // Create nginx pods. deploymentPodLabels := map[string]string{"name": "sample-pod"} rsPodLabels := map[string]string{ "name": "sample-pod", "pod": nginxImageName, } rsName := "test-rolling-update-controller" replicas := 3 _, err := c.Extensions().ReplicaSets(ns).Create(newRS(rsName, replicas, rsPodLabels, nginxImageName, nginxImage)) Expect(err).NotTo(HaveOccurred()) // Verify that the required pods have come up. err = framework.VerifyPods(unversionedClient, ns, "sample-pod", false, 3) if err != nil { framework.Logf("error in waiting for pods to come up: %s", err) Expect(err).NotTo(HaveOccurred()) } // Create a deployment to delete nginx pods and instead bring up redis pods. deploymentName := "test-rolling-update-deployment" framework.Logf("Creating deployment %s", deploymentName) _, err = c.Extensions().Deployments(ns).Create(newDeployment(deploymentName, replicas, deploymentPodLabels, redisImageName, redisImage, extensions.RollingUpdateDeploymentStrategyType, nil)) Expect(err).NotTo(HaveOccurred()) defer stopDeployment(c, f.Client, ns, deploymentName) // Wait for it to be updated to revision 1 err = framework.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, "1", redisImage) Expect(err).NotTo(HaveOccurred()) err = framework.WaitForDeploymentStatus(c, ns, deploymentName, replicas, replicas-1, replicas+1, 0) Expect(err).NotTo(HaveOccurred()) // There should be 1 old RS (nginx-controller, which is adopted) deployment, err := c.Extensions().Deployments(ns).Get(deploymentName) Expect(err).NotTo(HaveOccurred()) _, allOldRSs, err := deploymentutil.GetOldReplicaSets(deployment, c) Expect(err).NotTo(HaveOccurred()) Expect(len(allOldRSs)).Should(Equal(1)) // The old RS should contain pod-template-hash in its selector, label, and template label Expect(len(allOldRSs[0].Labels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0)) Expect(len(allOldRSs[0].Spec.Selector.MatchLabels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0)) Expect(len(allOldRSs[0].Spec.Template.Labels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0)) }
func verifyRCs(c clientset.Interface, ns string, names []string) { for _, name := range names { framework.ExpectNoError(framework.VerifyPods(c, ns, name, true, 1)) } }
// Many e2e tests assume that the cluster is fully healthy before they start. Wait until // the cluster is restored to health. By("waiting for system pods to successfully restart") err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, systemPodsNo, framework.PodReadyBeforeTimeout, ignoreLabels) Expect(err).NotTo(HaveOccurred()) By("waiting for image prepulling pods to complete") framework.WaitForPodsSuccess(c, api.NamespaceSystem, framework.ImagePullerLabels, imagePrePullingTimeout) }) It("should be able to delete nodes", func() { // Create a replication controller for a service that serves its hostname. // The source for the Docker container kubernetes/serve_hostname is in contrib/for-demos/serve_hostname name := "my-hostname-delete-node" replicas := int32(framework.TestContext.CloudConfig.NumNodes) newRCByName(c, ns, name, replicas) err := framework.VerifyPods(c, ns, name, true, replicas) Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("decreasing cluster size to %d", replicas-1)) err = ResizeGroup(group, replicas-1) Expect(err).NotTo(HaveOccurred()) err = WaitForGroupSize(group, replicas-1) Expect(err).NotTo(HaveOccurred()) err = framework.WaitForClusterSize(c, int(replicas-1), 10*time.Minute) Expect(err).NotTo(HaveOccurred()) By("verifying whether the pods from the removed node are recreated") err = framework.VerifyPods(c, ns, name, true, replicas) Expect(err).NotTo(HaveOccurred()) })
// Many e2e tests assume that the cluster is fully healthy before they start. Wait until // the cluster is restored to health. By("waiting for system pods to successfully restart") err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, systemPodsNo, framework.PodReadyBeforeTimeout, ignoreLabels, true) Expect(err).NotTo(HaveOccurred()) By("waiting for image prepulling pods to complete") framework.WaitForPodsSuccess(c, api.NamespaceSystem, framework.ImagePullerLabels, imagePrePullingTimeout) }) It("should be able to delete nodes", func() { // Create a replication controller for a service that serves its hostname. // The source for the Docker container kubernetes/serve_hostname is in contrib/for-demos/serve_hostname name := "my-hostname-delete-node" replicas := int32(framework.TestContext.CloudConfig.NumNodes) newRCByName(c, ns, name, replicas, nil) err := framework.VerifyPods(c, ns, name, true, replicas) Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("decreasing cluster size to %d", replicas-1)) err = ResizeGroup(group, replicas-1) Expect(err).NotTo(HaveOccurred()) err = WaitForGroupSize(group, replicas-1) Expect(err).NotTo(HaveOccurred()) err = framework.WaitForClusterSize(c, int(replicas-1), 10*time.Minute) Expect(err).NotTo(HaveOccurred()) By("waiting 1 minute for the watch in the podGC to catch up, remove any pods scheduled on " + "the now non-existent node and the RC to recreate it") time.Sleep(time.Minute) By("verifying whether the pods from the removed node are recreated")
// testRolloverDeployment tests that deployment supports rollover. // i.e. we can change desired state and kick off rolling update, then change desired state again before it finishes. func testRolloverDeployment(f *framework.Framework) { ns := f.Namespace.Name // TODO: remove unversionedClient when the refactoring is done. Currently some // functions like verifyPod still expects a unversioned#Client. unversionedClient := f.Client c := adapter.FromUnversionedClient(unversionedClient) podName := "rollover-pod" deploymentPodLabels := map[string]string{"name": podName} rsPodLabels := map[string]string{ "name": podName, "pod": nginxImageName, } rsName := "test-rollover-controller" rsReplicas := 4 _, err := c.Extensions().ReplicaSets(ns).Create(newRS(rsName, rsReplicas, rsPodLabels, nginxImageName, nginxImage)) Expect(err).NotTo(HaveOccurred()) // Verify that the required pods have come up. err = framework.VerifyPods(unversionedClient, ns, podName, false, rsReplicas) if err != nil { framework.Logf("error in waiting for pods to come up: %s", err) Expect(err).NotTo(HaveOccurred()) } // Wait for the required pods to be ready for at least minReadySeconds (be available) deploymentMinReadySeconds := 5 err = framework.WaitForPodsReady(c, ns, podName, deploymentMinReadySeconds) Expect(err).NotTo(HaveOccurred()) // Create a deployment to delete nginx pods and instead bring up redis-slave pods. deploymentName, deploymentImageName := "test-rollover-deployment", "redis-slave" deploymentReplicas := 4 deploymentImage := "gcr.io/google_samples/gb-redisslave:v1" deploymentStrategyType := extensions.RollingUpdateDeploymentStrategyType framework.Logf("Creating deployment %s", deploymentName) newDeployment := newDeployment(deploymentName, deploymentReplicas, deploymentPodLabels, deploymentImageName, deploymentImage, deploymentStrategyType, nil) newDeployment.Spec.MinReadySeconds = deploymentMinReadySeconds newDeployment.Spec.Strategy.RollingUpdate = &extensions.RollingUpdateDeployment{ MaxUnavailable: intstr.FromInt(1), MaxSurge: intstr.FromInt(1), } _, err = c.Extensions().Deployments(ns).Create(newDeployment) Expect(err).NotTo(HaveOccurred()) defer stopDeployment(c, f.Client, ns, deploymentName) // Verify that the pods were scaled up and down as expected. We use events to verify that. deployment, err := c.Extensions().Deployments(ns).Get(deploymentName) Expect(err).NotTo(HaveOccurred()) // Make sure the deployment starts to scale up and down replica sets framework.WaitForPartialEvents(unversionedClient, ns, deployment, 2) // Check if it's updated to revision 1 correctly _, newRS := checkDeploymentRevision(c, ns, deploymentName, "1", deploymentImageName, deploymentImage) // Before the deployment finishes, update the deployment to rollover the above 2 ReplicaSets and bring up redis pods. // If the deployment already finished here, the test would fail. When this happens, increase its minReadySeconds or replicas to prevent it. Expect(newRS.Spec.Replicas).Should(BeNumerically("<", deploymentReplicas)) updatedDeploymentImageName, updatedDeploymentImage := redisImageName, redisImage deployment, err = framework.UpdateDeploymentWithRetries(c, ns, newDeployment.Name, func(update *extensions.Deployment) { update.Spec.Template.Spec.Containers[0].Name = updatedDeploymentImageName update.Spec.Template.Spec.Containers[0].Image = updatedDeploymentImage }) Expect(err).NotTo(HaveOccurred()) // Use observedGeneration to determine if the controller noticed the pod template update. err = framework.WaitForObservedDeployment(c, ns, deploymentName, deployment.Generation) Expect(err).NotTo(HaveOccurred()) // Wait for it to be updated to revision 2 err = framework.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, "2", updatedDeploymentImage) Expect(err).NotTo(HaveOccurred()) err = framework.WaitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, deploymentMinReadySeconds) Expect(err).NotTo(HaveOccurred()) }
// testDeploymentCleanUpPolicy tests that deployment supports cleanup policy func testDeploymentCleanUpPolicy(f *framework.Framework) { ns := f.Namespace.Name unversionedClient := f.Client c := adapter.FromUnversionedClient(unversionedClient) // Create nginx pods. deploymentPodLabels := map[string]string{"name": "cleanup-pod"} rsPodLabels := map[string]string{ "name": "cleanup-pod", "pod": nginxImageName, } rsName := "test-cleanup-controller" replicas := 1 revisionHistoryLimit := util.IntPtr(0) _, err := c.Extensions().ReplicaSets(ns).Create(newRS(rsName, replicas, rsPodLabels, nginxImageName, nginxImage)) Expect(err).NotTo(HaveOccurred()) // Verify that the required pods have come up. err = framework.VerifyPods(unversionedClient, ns, "cleanup-pod", false, 1) if err != nil { framework.Logf("error in waiting for pods to come up: %s", err) Expect(err).NotTo(HaveOccurred()) } // Create a deployment to delete nginx pods and instead bring up redis pods. deploymentName := "test-cleanup-deployment" framework.Logf("Creating deployment %s", deploymentName) pods, err := c.Pods(ns).List(api.ListOptions{LabelSelector: labels.Everything()}) if err != nil { Expect(err).NotTo(HaveOccurred(), "Failed to query for pods: %v", err) } options := api.ListOptions{ ResourceVersion: pods.ListMeta.ResourceVersion, } stopCh := make(chan struct{}) w, err := c.Pods(ns).Watch(options) go func() { // There should be only one pod being created, which is the pod with the redis image. // The old RS shouldn't create new pod when deployment controller adding pod template hash label to its selector. numPodCreation := 1 for { select { case event, _ := <-w.ResultChan(): if event.Type != watch.Added { continue } numPodCreation-- if numPodCreation < 0 { framework.Failf("Expect only one pod creation, the second creation event: %#v\n", event) } pod, ok := event.Object.(*api.Pod) if !ok { Fail("Expect event Object to be a pod") } if pod.Spec.Containers[0].Name != redisImageName { framework.Failf("Expect the created pod to have container name %s, got pod %#v\n", redisImageName, pod) } case <-stopCh: return } } }() _, err = c.Extensions().Deployments(ns).Create(newDeployment(deploymentName, replicas, deploymentPodLabels, redisImageName, redisImage, extensions.RollingUpdateDeploymentStrategyType, revisionHistoryLimit)) Expect(err).NotTo(HaveOccurred()) defer stopDeployment(c, f.Client, ns, deploymentName) err = framework.WaitForDeploymentOldRSsNum(c, ns, deploymentName, *revisionHistoryLimit) Expect(err).NotTo(HaveOccurred()) close(stopCh) }
func verifyRCs(c *client.Client, ns string, names []string) { for _, name := range names { framework.ExpectNoError(framework.VerifyPods(c, ns, name, true, 1)) } }