Expect(nodeCount).Should(Equal(sum)) if framework.ProviderIs("gke") { val, err := isAutoscalerEnabled(3) framework.ExpectNoError(err) if !val { err = enableAutoscaler("default-pool", 3, 5) framework.ExpectNoError(err) } } }) AfterEach(func() { By(fmt.Sprintf("Restoring initial size of the cluster")) setMigSizes(originalSizes) framework.ExpectNoError(framework.WaitForClusterSize(c, nodeCount, scaleDownTimeout)) }) It("shouldn't increase cluster size if pending pod is too large [Feature:ClusterSizeAutoscalingScaleUp]", func() { By("Creating unschedulable pod") ReserveMemory(f, "memory-reservation", 1, memCapacityMb, false) defer framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, "memory-reservation") By("Waiting for scale up hoping it won't happen") // Verfiy, that the appropreate event was generated. eventFound := false EventsLoop: for start := time.Now(); time.Since(start) < scaleUpTimeout; time.Sleep(20 * time.Second) { By("Waiting for NotTriggerScaleUp event") events, err := f.ClientSet.Core().Events(f.Namespace.Name).List(v1.ListOptions{}) framework.ExpectNoError(err)
// In GKE, our current tunneling setup has the potential to hold on to a broken tunnel (from a // rebooted/deleted node) for up to 5 minutes before all tunnels are dropped and recreated. // Most tests make use of some proxy feature to verify functionality. So, if a reboot test runs // right before a test that tries to get logs, for example, we may get unlucky and try to use a // closed tunnel to a node that was recently rebooted. There's no good way to framework.Poll for proxies // being closed, so we sleep. // // TODO(cjcullen) reduce this sleep (#19314) if framework.ProviderIs("gke") { By("waiting 5 minutes for all dead tunnels to be dropped") time.Sleep(5 * time.Minute) } if err := WaitForGroupSize(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil { framework.Failf("Couldn't restore the original node instance group size: %v", err) } if err := framework.WaitForClusterSize(c, framework.TestContext.CloudConfig.NumNodes, 10*time.Minute); err != nil { framework.Failf("Couldn't restore the original cluster size: %v", err) } // Many e2e tests assume that the cluster is fully healthy before they start. Wait until // the cluster is restored to health. By("waiting for system pods to successfully restart") err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, systemPodsNo, framework.PodReadyBeforeTimeout, ignoreLabels) Expect(err).NotTo(HaveOccurred()) By("waiting for image prepulling pods to complete") framework.WaitForPodsSuccess(c, api.NamespaceSystem, framework.ImagePullerLabels, imagePrePullingTimeout) }) It("should be able to delete nodes", func() { // Create a replication controller for a service that serves its hostname. // The source for the Docker container kubernetes/serve_hostname is in contrib/for-demos/serve_hostname name := "my-hostname-delete-node"
By("Wait for kube-dns scaled to expected number") getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_1) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) By("Replace the dns autoscaling parameters with another testing parameters") err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_3))) Expect(err).NotTo(HaveOccurred()) By("Wait for kube-dns scaled to expected number") getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_3) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) By("Restoring cluster size") setMigSizes(originalSizes) Expect(framework.WaitForClusterSize(c, sum, scaleDownTimeout)).NotTo(HaveOccurred()) By("Wait for kube-dns scaled to expected number") Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) }) It("kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios", func() { By("Replace the dns autoscaling parameters with testing parameters") err := updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_1))) Expect(err).NotTo(HaveOccurred()) defer func() { By("Restoring intial dns autoscaling parameters") Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(previousParams))).NotTo(HaveOccurred()) }() By("Wait for kube-dns scaled to expected number")
By("Manually increase cluster size") increasedSize := 0 increasedSizes := make(map[string]int) for key, val := range originalSizes { increasedSizes[key] = val + 1 increasedSize += increasedSizes[key] } setMigSizes(increasedSizes) ExpectNoError(WaitForClusterSizeFunc(c, func(size int) bool { return size == increasedSize }, DNSscaleUpTimeout)) By("Wait for kube-dns scaled to expected number") ExpectNoError(waitForDNSReplicasSatisfied(c, increasedSize, DNSdefaultTimeout)) By("Restoring cluster size") setMigSizes(originalSizes) framework.ExpectNoError(framework.WaitForClusterSize(c, nodeCount, DNSscaleDownTimeout)) By("Wait for kube-dns scaled to expected number") ExpectNoError(waitForDNSReplicasSatisfied(c, nodeCount, DNSdefaultTimeout)) }) It("kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios", func() { By("--- Scenario: should scale kube-dns based on changed parameters ---") By("Replace the dns autoscaling parameters with the second testing parameters") ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_2))) By("Wait for kube-dns scaled to expected number") ExpectNoError(waitForDNSReplicasSatisfied(c, int(math.Ceil(float64(nodeCount)/2.0)), DNSdefaultTimeout)) By("--- Scenario: should re-create scaling parameters with default value when parameters got deleted ---") By("Delete the ConfigMap for autoscaler") err := deleteDNSScalingConfigMap(c)
cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU] mem := nodes.Items[0].Status.Capacity[api.ResourceMemory] coresPerNode = int((&cpu).MilliValue() / 1000) memCapacityMb = int((&mem).Value() / 1024 / 1024) }) AfterEach(func() { cleanUpAutoscaler() }) It("Should scale cluster size based on cpu utilization", func() { setUpAutoscaler("cpu/node_utilization", 0.4, nodeCount, nodeCount+1) // Consume 50% CPU rcs := createConsumingRCs(f, "cpu-utilization", nodeCount*coresPerNode, 500, 0) err := framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout) for _, rc := range rcs { rc.CleanUp() } framework.ExpectNoError(err) framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) }) It("Should scale cluster size based on cpu reservation", func() { setUpAutoscaler("cpu/node_reservation", 0.5, nodeCount, nodeCount+1) ReserveCpu(f, "cpu-reservation", 600*nodeCount*coresPerNode) framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)) framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "cpu-reservation"))
nodes := framework.GetReadySchedulableNodesOrDie(f.Client) nodeCount = len(nodes.Items) Expect(nodeCount).NotTo(BeZero()) cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU] mem := nodes.Items[0].Status.Capacity[api.ResourceMemory] coresPerNode = int((&cpu).MilliValue() / 1000) memCapacityMb = int((&mem).Value() / 1024 / 1024) }) It("Should correctly handle pending pods", func() { By("Too large pending pod does not increase cluster size") ReserveMemory(f, "memory-reservation", 1, memCapacityMb, false) // Verify, that cluster size is not changed. // TODO: find a better way of verification that the cluster size will remain unchanged. time.Sleep(scaleTimeout) framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleTimeout)) framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "memory-reservation")) framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleTimeout)) By("Small pending pods increase cluster size") ReserveMemory(f, "memory-reservation", 100, nodeCount*memCapacityMb, false) // Verify, that cluster size is increased framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleTimeout)) framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "memory-reservation")) // TODO(jsz): Disable the line bellow when scale down is implemented. framework.ExpectNoError(ResizeGroup(int32(nodeCount))) framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleTimeout)) By("Handling node port pods") CreateHostPortPods(f, "host-port", nodeCount+2, false)