// createOutOfDiskPod creates a pod in the given namespace with the requested amount of CPU. func createOutOfDiskPod(c *client.Client, ns, name string, milliCPU int64) { podClient := c.Pods(ns) pod := &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: name, }, Spec: api.PodSpec{ Containers: []api.Container{ { Name: "pause", Image: framework.GetPauseImageName(c), Resources: api.ResourceRequirements{ Requests: api.ResourceList{ // Request enough CPU to fit only two pods on a given node. api.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), }, }, }, }, }, } _, err := podClient.Create(pod) framework.ExpectNoError(err) }
func runServiceLatencies(f *framework.Framework, inParallel, total int) (output []time.Duration, err error) { cfg := testutils.RCConfig{ Client: f.Client, Image: framework.GetPauseImageName(f.Client), Name: "svc-latency-rc", Namespace: f.Namespace.Name, Replicas: 1, PollInterval: time.Second, } if err := framework.RunRC(cfg); err != nil { return nil, err } // Run a single watcher, to reduce the number of API calls we have to // make; this is to minimize the timing error. It's how kube-proxy // consumes the endpoints data, so it seems like the right thing to // test. endpointQueries := newQuerier() startEndpointWatcher(f, endpointQueries) defer close(endpointQueries.stop) // run one test and throw it away-- this is to make sure that the pod's // ready status has propagated. singleServiceLatency(f, cfg.Name, endpointQueries) // These channels are never closed, and each attempt sends on exactly // one of these channels, so the sum of the things sent over them will // be exactly total. errs := make(chan error, total) durations := make(chan time.Duration, total) blocker := make(chan struct{}, inParallel) for i := 0; i < total; i++ { go func() { defer GinkgoRecover() blocker <- struct{}{} defer func() { <-blocker }() if d, err := singleServiceLatency(f, cfg.Name, endpointQueries); err != nil { errs <- err } else { durations <- d } }() } errCount := 0 for i := 0; i < total; i++ { select { case e := <-errs: framework.Logf("Got error: %v", e) errCount += 1 case d := <-durations: output = append(output, d) } } if errCount != 0 { return output, fmt.Errorf("got %v errors", errCount) } return output, nil }
func initPausePod(f *framework.Framework, conf pausePodConfig) *api.Pod { if conf.Affinity != "" { if conf.Annotations == nil { conf.Annotations = map[string]string{ api.AffinityAnnotationKey: conf.Affinity, } } else { conf.Annotations[api.AffinityAnnotationKey] = conf.Affinity } } pod := &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: conf.Name, Labels: conf.Labels, Annotations: conf.Annotations, }, Spec: api.PodSpec{ NodeSelector: conf.NodeSelector, Containers: []api.Container{ { Name: podName, Image: framework.GetPauseImageName(f.Client), }, }, }, } if conf.Resources != nil { pod.Spec.Containers[0].Resources = *conf.Resources } return pod }
func runSchedulerNoPhantomPodsTest(client *client.Client) { pod := &api.Pod{ Spec: api.PodSpec{ Containers: []api.Container{ { Name: "c1", Image: e2e.GetPauseImageName(client), Ports: []api.ContainerPort{ {ContainerPort: 1234, HostPort: 9999}, }, ImagePullPolicy: api.PullIfNotPresent, }, }, }, } // Assuming we only have two kubelets, the third pod here won't schedule // if the scheduler doesn't correctly handle the delete for the second // pod. pod.ObjectMeta.Name = "phantom.foo" foo, err := client.Pods(api.NamespaceDefault).Create(pod) if err != nil { glog.Fatalf("Failed to create pod: %v, %v", pod, err) } if err := wait.Poll(time.Second, longTestTimeout, podRunning(client, foo.Namespace, foo.Name)); err != nil { glog.Fatalf("FAILED: pod never started running %v", err) } pod.ObjectMeta.Name = "phantom.bar" bar, err := client.Pods(api.NamespaceDefault).Create(pod) if err != nil { glog.Fatalf("Failed to create pod: %v, %v", pod, err) } if err := wait.Poll(time.Second, longTestTimeout, podRunning(client, bar.Namespace, bar.Name)); err != nil { glog.Fatalf("FAILED: pod never started running %v", err) } // Delete a pod to free up room. glog.Infof("Deleting pod %v", bar.Name) err = client.Pods(api.NamespaceDefault).Delete(bar.Name, api.NewDeleteOptions(0)) if err != nil { glog.Fatalf("FAILED: couldn't delete pod %q: %v", bar.Name, err) } pod.ObjectMeta.Name = "phantom.baz" baz, err := client.Pods(api.NamespaceDefault).Create(pod) if err != nil { glog.Fatalf("Failed to create pod: %v, %v", pod, err) } if err := wait.Poll(time.Second, longTestTimeout, podRunning(client, baz.Namespace, baz.Name)); err != nil { if pod, perr := client.Pods(api.NamespaceDefault).Get("phantom.bar"); perr == nil { glog.Fatalf("FAILED: 'phantom.bar' was never deleted: %#v, err: %v", pod, err) } else { glog.Fatalf("FAILED: (Scheduler probably didn't process deletion of 'phantom.bar') Pod never started running: err: %v, perr: %v", err, perr) } } glog.Info("Scheduler doesn't make phantom pods: test passed.") }
func createPod(client clientset.Interface, name string, annotation map[string]string) *api.Pod { return &api.Pod{ ObjectMeta: api.ObjectMeta{Name: name, Annotations: annotation}, Spec: api.PodSpec{ Containers: []api.Container{{Name: "container", Image: e2e.GetPauseImageName(client)}}, }, } }
func runResourceTrackingTest(f *framework.Framework, podsPerNode int, nodeNames sets.String, rm *framework.ResourceMonitor, expectedCPU map[string]map[float64]float64, expectedMemory framework.ResourceUsagePerContainer) { numNodes := nodeNames.Len() totalPods := podsPerNode * numNodes By(fmt.Sprintf("Creating a RC of %d pods and wait until all pods of this RC are running", totalPods)) rcName := fmt.Sprintf("resource%d-%s", totalPods, string(uuid.NewUUID())) // TODO: Use a more realistic workload Expect(framework.RunRC(testutils.RCConfig{ Client: f.ClientSet, InternalClient: f.InternalClientset, Name: rcName, Namespace: f.Namespace.Name, Image: framework.GetPauseImageName(f.ClientSet), Replicas: totalPods, })).NotTo(HaveOccurred()) // Log once and flush the stats. rm.LogLatest() rm.Reset() By("Start monitoring resource usage") // Periodically dump the cpu summary until the deadline is met. // Note that without calling framework.ResourceMonitor.Reset(), the stats // would occupy increasingly more memory. This should be fine // for the current test duration, but we should reclaim the // entries if we plan to monitor longer (e.g., 8 hours). deadline := time.Now().Add(monitoringTime) for time.Now().Before(deadline) { timeLeft := deadline.Sub(time.Now()) framework.Logf("Still running...%v left", timeLeft) if timeLeft < reportingPeriod { time.Sleep(timeLeft) } else { time.Sleep(reportingPeriod) } logPodsOnNodes(f.ClientSet, nodeNames.List()) } By("Reporting overall resource usage") logPodsOnNodes(f.ClientSet, nodeNames.List()) usageSummary, err := rm.GetLatest() Expect(err).NotTo(HaveOccurred()) // TODO(random-liu): Remove the original log when we migrate to new perfdash framework.Logf("%s", rm.FormatResourceUsage(usageSummary)) // Log perf result framework.PrintPerfData(framework.ResourceUsageToPerfData(rm.GetMasterNodeLatest(usageSummary))) verifyMemoryLimits(f.ClientSet, expectedMemory, usageSummary) cpuSummary := rm.GetCPUSummary() framework.Logf("%s", rm.FormatCPUSummary(cpuSummary)) // Log perf result framework.PrintPerfData(framework.CPUUsageToPerfData(rm.GetMasterNodeCPUSummary(cpuSummary))) verifyCPULimits(expectedCPU, cpuSummary) By("Deleting the RC") framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, rcName) }
func createPod(client clientset.Interface, name string, scheduler string) *v1.Pod { return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: v1.PodSpec{ Containers: []v1.Container{{Name: "container", Image: e2e.GetPauseImageName(client)}}, SchedulerName: scheduler, }, } }
// Check that the pods comprising a service get spread evenly across available zones func SpreadServiceOrFail(f *framework.Framework, replicaCount int, image string) { // First create the service serviceName := "test-service" serviceSpec := &api.Service{ ObjectMeta: api.ObjectMeta{ Name: serviceName, Namespace: f.Namespace.Name, }, Spec: api.ServiceSpec{ Selector: map[string]string{ "service": serviceName, }, Ports: []api.ServicePort{{ Port: 80, TargetPort: intstr.FromInt(80), }}, }, } _, err := f.Client.Services(f.Namespace.Name).Create(serviceSpec) Expect(err).NotTo(HaveOccurred()) // Now create some pods behind the service podSpec := &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: serviceName, Labels: map[string]string{"service": serviceName}, }, Spec: api.PodSpec{ Containers: []api.Container{ { Name: "test", Image: framework.GetPauseImageName(f.Client), }, }, }, } // Caution: StartPods requires at least one pod to replicate. // Based on the callers, replicas is always positive number: zoneCount >= 0 implies (2*zoneCount)+1 > 0. // Thus, no need to test for it. Once the precondition changes to zero number of replicas, // test for replicaCount > 0. Otherwise, StartPods panics. framework.StartPods(f.Client, replicaCount, f.Namespace.Name, serviceName, *podSpec, false) // Wait for all of them to be scheduled selector := labels.SelectorFromSet(labels.Set(map[string]string{"service": serviceName})) pods, err := framework.WaitForPodsWithLabelScheduled(f.Client, f.Namespace.Name, selector) Expect(err).NotTo(HaveOccurred()) // Now make sure they're spread across zones zoneNames, err := getZoneNames(f.Client) Expect(err).NotTo(HaveOccurred()) Expect(checkZoneSpreading(f.Client, pods, zoneNames)).To(Equal(true)) }
func DoTestPodScheduling(ns *v1.Namespace, t *testing.T, cs clientset.Interface) { // NOTE: This test cannot run in parallel, because it is creating and deleting // non-namespaced objects (Nodes). defer cs.Core().Nodes().DeleteCollection(nil, v1.ListOptions{}) goodCondition := v1.NodeCondition{ Type: v1.NodeReady, Status: v1.ConditionTrue, Reason: fmt.Sprintf("schedulable condition"), LastHeartbeatTime: unversioned.Time{time.Now()}, } node := &v1.Node{ Spec: v1.NodeSpec{Unschedulable: false}, Status: v1.NodeStatus{ Capacity: v1.ResourceList{ v1.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), }, Conditions: []v1.NodeCondition{goodCondition}, }, } for ii := 0; ii < 5; ii++ { node.Name = fmt.Sprintf("machine%d", ii+1) if _, err := cs.Core().Nodes().Create(node); err != nil { t.Fatalf("Failed to create nodes: %v", err) } } pod := &v1.Pod{ ObjectMeta: v1.ObjectMeta{Name: "extender-test-pod"}, Spec: v1.PodSpec{ Containers: []v1.Container{{Name: "container", Image: e2e.GetPauseImageName(cs)}}, }, } myPod, err := cs.Core().Pods(ns.Name).Create(pod) if err != nil { t.Fatalf("Failed to create pod: %v", err) } err = wait.Poll(time.Second, wait.ForeverTestTimeout, podScheduled(cs, myPod.Namespace, myPod.Name)) if err != nil { t.Fatalf("Failed to schedule pod: %v", err) } if myPod, err := cs.Core().Pods(ns.Name).Get(myPod.Name); err != nil { t.Fatalf("Failed to get pod: %v", err) } else if myPod.Spec.NodeName != "machine3" { t.Fatalf("Failed to schedule using extender, expected machine3, got %v", myPod.Spec.NodeName) } t.Logf("Scheduled pod using extenders") }
func ensurePodsAreRemovedWhenNamespaceIsDeleted(f *framework.Framework) { var err error By("Creating a test namespace") namespace, err := f.CreateNamespace("nsdeletetest", nil) Expect(err).NotTo(HaveOccurred()) By("Waiting for a default service account to be provisioned in namespace") err = framework.WaitForDefaultServiceAccountInNamespace(f.ClientSet, namespace.Name) Expect(err).NotTo(HaveOccurred()) By("Creating a pod in the namespace") pod := &v1.Pod{ ObjectMeta: v1.ObjectMeta{ Name: "test-pod", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name: "nginx", Image: framework.GetPauseImageName(f.ClientSet), }, }, }, } pod, err = f.ClientSet.Core().Pods(namespace.Name).Create(pod) Expect(err).NotTo(HaveOccurred()) By("Waiting for the pod to have running status") framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, pod)) By("Deleting the namespace") err = f.ClientSet.Core().Namespaces().Delete(namespace.Name, nil) Expect(err).NotTo(HaveOccurred()) By("Waiting for the namespace to be removed.") maxWaitSeconds := int64(60) + *pod.Spec.TerminationGracePeriodSeconds framework.ExpectNoError(wait.Poll(1*time.Second, time.Duration(maxWaitSeconds)*time.Second, func() (bool, error) { _, err = f.ClientSet.Core().Namespaces().Get(namespace.Name, metav1.GetOptions{}) if err != nil && errors.IsNotFound(err) { return true, nil } return false, nil })) By("Verifying there is no pod in the namespace") _, err = f.ClientSet.Core().Pods(namespace.Name).Get(pod.Name, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) }
func ReserveCpu(f *framework.Framework, id string, replicas, millicores int) { By(fmt.Sprintf("Running RC which reserves %v millicores", millicores)) request := int64(millicores / replicas) config := &testutils.RCConfig{ Client: f.Client, Name: id, Namespace: f.Namespace.Name, Timeout: defaultTimeout, Image: framework.GetPauseImageName(f.Client), Replicas: replicas, CpuRequest: request, } framework.ExpectNoError(framework.RunRC(*config)) }
// Check that the pods comprising a service get spread evenly across available zones func SpreadServiceOrFail(f *framework.Framework, replicaCount int, image string) { // First create the service serviceName := "test-service" serviceSpec := &api.Service{ ObjectMeta: api.ObjectMeta{ Name: serviceName, Namespace: f.Namespace.Name, }, Spec: api.ServiceSpec{ Selector: map[string]string{ "service": serviceName, }, Ports: []api.ServicePort{{ Port: 80, TargetPort: intstr.FromInt(80), }}, }, } _, err := f.Client.Services(f.Namespace.Name).Create(serviceSpec) Expect(err).NotTo(HaveOccurred()) // Now create some pods behind the service podSpec := &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: serviceName, Labels: map[string]string{"service": serviceName}, }, Spec: api.PodSpec{ Containers: []api.Container{ { Name: "test", Image: framework.GetPauseImageName(f.Client), }, }, }, } framework.StartPods(f.Client, replicaCount, f.Namespace.Name, serviceName, *podSpec, false) // Wait for all of them to be scheduled selector := labels.SelectorFromSet(labels.Set(map[string]string{"service": serviceName})) pods, err := framework.WaitForPodsWithLabelScheduled(f.Client, f.Namespace.Name, selector) Expect(err).NotTo(HaveOccurred()) // Now make sure they're spread across zones zoneNames, err := getZoneNames(f.Client) Expect(err).NotTo(HaveOccurred()) Expect(checkZoneSpreading(f.Client, pods, zoneNames)).To(Equal(true)) }
func DoTestPodScheduling(t *testing.T, restClient *client.Client) { goodCondition := api.NodeCondition{ Type: api.NodeReady, Status: api.ConditionTrue, Reason: fmt.Sprintf("schedulable condition"), LastHeartbeatTime: unversioned.Time{time.Now()}, } node := &api.Node{ Spec: api.NodeSpec{Unschedulable: false}, Status: api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), }, Conditions: []api.NodeCondition{goodCondition}, }, } for ii := 0; ii < 5; ii++ { node.Name = fmt.Sprintf("machine%d", ii+1) if _, err := restClient.Nodes().Create(node); err != nil { t.Fatalf("Failed to create nodes: %v", err) } } pod := &api.Pod{ ObjectMeta: api.ObjectMeta{Name: "extender-test-pod"}, Spec: api.PodSpec{ Containers: []api.Container{{Name: "container", Image: e2e.GetPauseImageName(restClient)}}, }, } myPod, err := restClient.Pods(api.NamespaceDefault).Create(pod) if err != nil { t.Fatalf("Failed to create pod: %v", err) } err = wait.Poll(time.Second, wait.ForeverTestTimeout, podScheduled(restClient, myPod.Namespace, myPod.Name)) if err != nil { t.Fatalf("Failed to schedule pod: %v", err) } if myPod, err := restClient.Pods(api.NamespaceDefault).Get(myPod.Name); err != nil { t.Fatalf("Failed to get pod: %v", err) } else if myPod.Spec.NodeName != "machine3" { t.Fatalf("Failed to schedule using extender, expected machine3, got %v", myPod.Spec.NodeName) } t.Logf("Scheduled pod using extenders") }
func CreateHostPortPods(f *framework.Framework, id string, replicas int, expectRunning bool) { By(fmt.Sprintf("Running RC which reserves host port")) config := &testutils.RCConfig{ Client: f.Client, Name: id, Namespace: f.Namespace.Name, Timeout: defaultTimeout, Image: framework.GetPauseImageName(f.Client), Replicas: replicas, HostPorts: map[string]int{"port1": 4321}, } err := framework.RunRC(*config) if expectRunning { framework.ExpectNoError(err) } }
func ReserveMemory(f *framework.Framework, id string, replicas, megabytes int, expectRunning bool) { By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes)) request := int64(1024 * 1024 * megabytes / replicas) config := &testutils.RCConfig{ Client: f.Client, Name: id, Namespace: f.Namespace.Name, Timeout: defaultTimeout, Image: framework.GetPauseImageName(f.Client), Replicas: replicas, MemRequest: request, } err := framework.RunRC(*config) if expectRunning { framework.ExpectNoError(err) } }
func CreateNodeSelectorPods(f *framework.Framework, id string, replicas int, nodeSelector map[string]string, expectRunning bool) { By(fmt.Sprintf("Running RC which reserves host port and defines node selector")) config := &testutils.RCConfig{ Client: f.Client, Name: "node-selector", Namespace: f.Namespace.Name, Timeout: defaultTimeout, Image: framework.GetPauseImageName(f.Client), Replicas: replicas, HostPorts: map[string]int{"port1": 4321}, NodeSelector: map[string]string{"cluster-autoscaling-test.special-node": "true"}, } err := framework.RunRC(*config) if expectRunning { framework.ExpectNoError(err) } }
// newTestPod returns a pod that has the specified requests and limits func newTestPod(f *framework.Framework, name string, requests api.ResourceList, limits api.ResourceList) *api.Pod { return &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: name, }, Spec: api.PodSpec{ Containers: []api.Container{ { Name: "pause", Image: framework.GetPauseImageName(f.ClientSet), Resources: api.ResourceRequirements{ Requests: requests, Limits: limits, }, }, }, }, } }
// newTestPodForQuota returns a pod that has the specified requests and limits func newTestPodForQuota(f *framework.Framework, name string, requests v1.ResourceList, limits v1.ResourceList) *v1.Pod { return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name: "pause", Image: framework.GetPauseImageName(f.ClientSet), Resources: v1.ResourceRequirements{ Requests: requests, Limits: limits, }, }, }, }, } }
func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod { pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: conf.Name, Labels: conf.Labels, Annotations: conf.Annotations, }, Spec: v1.PodSpec{ NodeSelector: conf.NodeSelector, Affinity: conf.Affinity, Containers: []v1.Container{ { Name: podName, Image: framework.GetPauseImageName(f.ClientSet), }, }, }, } if conf.Resources != nil { pod.Spec.Containers[0].Resources = *conf.Resources } return pod }
// This test will verify scheduler can work well regardless of whether kubelet is allocatable aware or not. func TestAllocatable(t *testing.T) { _, s := framework.RunAMaster(nil) defer s.Close() ns := framework.CreateTestingNamespace("allocatable", s, t) defer framework.DeleteTestingNamespace(ns, s, t) // 1. create and start default-scheduler clientSet := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: ®istered.GroupOrDie(api.GroupName).GroupVersion}}) // NOTE: This test cannot run in parallel, because it is creating and deleting // non-namespaced objects (Nodes). defer clientSet.Core().Nodes().DeleteCollection(nil, api.ListOptions{}) schedulerConfigFactory := factory.NewConfigFactory(clientSet, api.DefaultSchedulerName, api.DefaultHardPodAffinitySymmetricWeight, api.DefaultFailureDomains) schedulerConfig, err := schedulerConfigFactory.Create() if err != nil { t.Fatalf("Couldn't create scheduler config: %v", err) } eventBroadcaster := record.NewBroadcaster() schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName}) eventBroadcaster.StartRecordingToSink(&unversionedcore.EventSinkImpl{Interface: clientSet.Core().Events(ns.Name)}) scheduler.New(schedulerConfig).Run() // default-scheduler will be stopped later defer close(schedulerConfig.StopEverything) // 2. create a node without allocatable awareness node := &api.Node{ ObjectMeta: api.ObjectMeta{Name: "node-allocatable-scheduler-test-node"}, Spec: api.NodeSpec{Unschedulable: false}, Status: api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), api.ResourceCPU: *resource.NewMilliQuantity(30, resource.DecimalSI), api.ResourceMemory: *resource.NewQuantity(30, resource.BinarySI), }, }, } allocNode, err := clientSet.Core().Nodes().Create(node) if err != nil { t.Fatalf("Failed to create node: %v", err) } // 3. create resource pod which requires less than Capacity podResource := &api.Pod{ ObjectMeta: api.ObjectMeta{Name: "pod-test-allocatable"}, Spec: api.PodSpec{ Containers: []api.Container{ { Name: "container", Image: e2e.GetPauseImageName(clientSet), Resources: api.ResourceRequirements{ Requests: api.ResourceList{ api.ResourceCPU: *resource.NewMilliQuantity(20, resource.DecimalSI), api.ResourceMemory: *resource.NewQuantity(20, resource.BinarySI), }, }, }, }, }, } testAllocPod, err := clientSet.Core().Pods(ns.Name).Create(podResource) if err != nil { t.Fatalf("Test allocatable unawareness failed to create pod: %v", err) } // 4. Test: this test pod should be scheduled since api-server will use Capacity as Allocatable err = wait.Poll(time.Second, time.Second*5, podScheduled(clientSet, testAllocPod.Namespace, testAllocPod.Name)) if err != nil { t.Errorf("Test allocatable unawareness: %s Pod not scheduled: %v", testAllocPod.Name, err) } else { t.Logf("Test allocatable unawareness: %s Pod scheduled", testAllocPod.Name) } // 5. Change the node status to allocatable aware, note that Allocatable is less than Pod's requirement allocNode.Status = api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), api.ResourceCPU: *resource.NewMilliQuantity(30, resource.DecimalSI), api.ResourceMemory: *resource.NewQuantity(30, resource.BinarySI), }, Allocatable: api.ResourceList{ api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), api.ResourceCPU: *resource.NewMilliQuantity(10, resource.DecimalSI), api.ResourceMemory: *resource.NewQuantity(10, resource.BinarySI), }, } if _, err := clientSet.Core().Nodes().UpdateStatus(allocNode); err != nil { t.Fatalf("Failed to update node with Status.Allocatable: %v", err) } if err := clientSet.Core().Pods(ns.Name).Delete(podResource.Name, &api.DeleteOptions{}); err != nil { t.Fatalf("Failed to remove first resource pod: %v", err) } // 6. Make another pod with different name, same resource request podResource.ObjectMeta.Name = "pod-test-allocatable2" testAllocPod2, err := clientSet.Core().Pods(ns.Name).Create(podResource) if err != nil { t.Fatalf("Test allocatable awareness failed to create pod: %v", err) } // 7. Test: this test pod should not be scheduled since it request more than Allocatable err = wait.Poll(time.Second, time.Second*5, podScheduled(clientSet, testAllocPod2.Namespace, testAllocPod2.Name)) if err == nil { t.Errorf("Test allocatable awareness: %s Pod got scheduled unexpectly, %v", testAllocPod2.Name, err) } else { t.Logf("Test allocatable awareness: %s Pod not scheduled as expected", testAllocPod2.Name) } }
// This test will verify scheduler can work well regardless of whether kubelet is allocatable aware or not. func TestAllocatable(t *testing.T) { framework.DeleteAllEtcdKeys() var m *master.Master s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { m.Handler.ServeHTTP(w, req) })) defer s.Close() masterConfig := framework.NewIntegrationTestMasterConfig() m, err := master.New(masterConfig) if err != nil { t.Fatalf("Error in bringing up the master: %v", err) } // 1. create and start default-scheduler restClient := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}) schedulerConfigFactory := factory.NewConfigFactory(restClient, api.DefaultSchedulerName, api.DefaultHardPodAffinitySymmetricWeight, api.DefaultFailureDomains) schedulerConfig, err := schedulerConfigFactory.Create() if err != nil { t.Fatalf("Couldn't create scheduler config: %v", err) } eventBroadcaster := record.NewBroadcaster() schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName}) eventBroadcaster.StartRecordingToSink(restClient.Events("")) scheduler.New(schedulerConfig).Run() // default-scheduler will be stopped later defer close(schedulerConfig.StopEverything) // 2. create a node without allocatable awareness node := &api.Node{ ObjectMeta: api.ObjectMeta{Name: "node-allocatable-scheduler-test-node"}, Spec: api.NodeSpec{Unschedulable: false}, Status: api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), api.ResourceCPU: *resource.NewMilliQuantity(30, resource.DecimalSI), api.ResourceMemory: *resource.NewQuantity(30, resource.BinarySI), }, }, } allocNode, err := restClient.Nodes().Create(node) if err != nil { t.Fatalf("Failed to create node: %v", err) } // 3. create resource pod which requires less than Capacity podResource := &api.Pod{ ObjectMeta: api.ObjectMeta{Name: "pod-test-allocatable"}, Spec: api.PodSpec{ Containers: []api.Container{ { Name: "container", Image: e2e.GetPauseImageName(restClient), Resources: api.ResourceRequirements{ Requests: api.ResourceList{ api.ResourceCPU: *resource.NewMilliQuantity(20, resource.DecimalSI), api.ResourceMemory: *resource.NewQuantity(20, resource.BinarySI), }, }, }, }, }, } testAllocPod, err := restClient.Pods(api.NamespaceDefault).Create(podResource) if err != nil { t.Fatalf("Test allocatable unawareness failed to create pod: %v", err) } // 4. Test: this test pod should be scheduled since api-server will use Capacity as Allocatable err = wait.Poll(time.Second, time.Second*5, podScheduled(restClient, testAllocPod.Namespace, testAllocPod.Name)) if err != nil { t.Errorf("Test allocatable unawareness: %s Pod not scheduled: %v", testAllocPod.Name, err) } else { t.Logf("Test allocatable unawareness: %s Pod scheduled", testAllocPod.Name) } // 5. Change the node status to allocatable aware, note that Allocatable is less than Pod's requirement allocNode.Status = api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), api.ResourceCPU: *resource.NewMilliQuantity(30, resource.DecimalSI), api.ResourceMemory: *resource.NewQuantity(30, resource.BinarySI), }, Allocatable: api.ResourceList{ api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), api.ResourceCPU: *resource.NewMilliQuantity(10, resource.DecimalSI), api.ResourceMemory: *resource.NewQuantity(10, resource.BinarySI), }, } if _, err := restClient.Nodes().UpdateStatus(allocNode); err != nil { t.Fatalf("Failed to update node with Status.Allocatable: %v", err) } if err := restClient.Pods(api.NamespaceDefault).Delete(podResource.Name, &api.DeleteOptions{}); err != nil { t.Fatalf("Failed to remove first resource pod: %v", err) } // 6. Make another pod with different name, same resource request podResource.ObjectMeta.Name = "pod-test-allocatable2" testAllocPod2, err := restClient.Pods(api.NamespaceDefault).Create(podResource) if err != nil { t.Fatalf("Test allocatable awareness failed to create pod: %v", err) } // 7. Test: this test pod should not be scheduled since it request more than Allocatable err = wait.Poll(time.Second, time.Second*5, podScheduled(restClient, testAllocPod2.Namespace, testAllocPod2.Name)) if err == nil { t.Errorf("Test allocatable awareness: %s Pod got scheduled unexpectly, %v", testAllocPod2.Name, err) } else { t.Logf("Test allocatable awareness: %s Pod not scheduled as expected", testAllocPod2.Name) } }
func DoTestUnschedulableNodes(t *testing.T, cs clientset.Interface, ns *api.Namespace, nodeStore cache.Store) { // NOTE: This test cannot run in parallel, because it is creating and deleting // non-namespaced objects (Nodes). defer cs.Core().Nodes().DeleteCollection(nil, api.ListOptions{}) goodCondition := api.NodeCondition{ Type: api.NodeReady, Status: api.ConditionTrue, Reason: fmt.Sprintf("schedulable condition"), LastHeartbeatTime: unversioned.Time{time.Now()}, } badCondition := api.NodeCondition{ Type: api.NodeReady, Status: api.ConditionUnknown, Reason: fmt.Sprintf("unschedulable condition"), LastHeartbeatTime: unversioned.Time{time.Now()}, } // Create a new schedulable node, since we're first going to apply // the unschedulable condition and verify that pods aren't scheduled. node := &api.Node{ ObjectMeta: api.ObjectMeta{Name: "node-scheduling-test-node"}, Spec: api.NodeSpec{Unschedulable: false}, Status: api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), }, Conditions: []api.NodeCondition{goodCondition}, }, } nodeKey, err := cache.MetaNamespaceKeyFunc(node) if err != nil { t.Fatalf("Couldn't retrieve key for node %v", node.Name) } // The test does the following for each nodeStateManager in this list: // 1. Create a new node // 2. Apply the makeUnSchedulable function // 3. Create a new pod // 4. Check that the pod doesn't get assigned to the node // 5. Apply the schedulable function // 6. Check that the pod *does* get assigned to the node // 7. Delete the pod and node. nodeModifications := []nodeStateManager{ // Test node.Spec.Unschedulable=true/false { makeUnSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c clientset.Interface) { n.Spec.Unschedulable = true if _, err := c.Core().Nodes().Update(n); err != nil { t.Fatalf("Failed to update node with unschedulable=true: %v", err) } err = waitForReflection(t, s, nodeKey, func(node interface{}) bool { // An unschedulable node should still be present in the store // Nodes that are unschedulable or that are not ready or // have their disk full (Node.Spec.Conditions) are exluded // based on NodeConditionPredicate, a separate check return node != nil && node.(*api.Node).Spec.Unschedulable == true }) if err != nil { t.Fatalf("Failed to observe reflected update for setting unschedulable=true: %v", err) } }, makeSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c clientset.Interface) { n.Spec.Unschedulable = false if _, err := c.Core().Nodes().Update(n); err != nil { t.Fatalf("Failed to update node with unschedulable=false: %v", err) } err = waitForReflection(t, s, nodeKey, func(node interface{}) bool { return node != nil && node.(*api.Node).Spec.Unschedulable == false }) if err != nil { t.Fatalf("Failed to observe reflected update for setting unschedulable=false: %v", err) } }, }, // Test node.Status.Conditions=ConditionTrue/Unknown { makeUnSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c clientset.Interface) { n.Status = api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), }, Conditions: []api.NodeCondition{badCondition}, } if _, err = c.Core().Nodes().UpdateStatus(n); err != nil { t.Fatalf("Failed to update node with bad status condition: %v", err) } err = waitForReflection(t, s, nodeKey, func(node interface{}) bool { return node != nil && node.(*api.Node).Status.Conditions[0].Status == api.ConditionUnknown }) if err != nil { t.Fatalf("Failed to observe reflected update for status condition update: %v", err) } }, makeSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c clientset.Interface) { n.Status = api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), }, Conditions: []api.NodeCondition{goodCondition}, } if _, err = c.Core().Nodes().UpdateStatus(n); err != nil { t.Fatalf("Failed to update node with healthy status condition: %v", err) } err = waitForReflection(t, s, nodeKey, func(node interface{}) bool { return node != nil && node.(*api.Node).Status.Conditions[0].Status == api.ConditionTrue }) if err != nil { t.Fatalf("Failed to observe reflected update for status condition update: %v", err) } }, }, } for i, mod := range nodeModifications { unSchedNode, err := cs.Core().Nodes().Create(node) if err != nil { t.Fatalf("Failed to create node: %v", err) } // Apply the unschedulable modification to the node, and wait for the reflection mod.makeUnSchedulable(t, unSchedNode, nodeStore, cs) // Create the new pod, note that this needs to happen post unschedulable // modification or we have a race in the test. pod := &api.Pod{ ObjectMeta: api.ObjectMeta{Name: "node-scheduling-test-pod"}, Spec: api.PodSpec{ Containers: []api.Container{{Name: "container", Image: e2e.GetPauseImageName(cs)}}, }, } myPod, err := cs.Core().Pods(ns.Name).Create(pod) if err != nil { t.Fatalf("Failed to create pod: %v", err) } // There are no schedulable nodes - the pod shouldn't be scheduled. err = wait.Poll(time.Second, wait.ForeverTestTimeout, podScheduled(cs, myPod.Namespace, myPod.Name)) if err == nil { t.Errorf("Pod scheduled successfully on unschedulable nodes") } if err != wait.ErrWaitTimeout { t.Errorf("Test %d: failed while trying to confirm the pod does not get scheduled on the node: %v", i, err) } else { t.Logf("Test %d: Pod did not get scheduled on an unschedulable node", i) } // Apply the schedulable modification to the node, and wait for the reflection schedNode, err := cs.Core().Nodes().Get(unSchedNode.Name) if err != nil { t.Fatalf("Failed to get node: %v", err) } mod.makeSchedulable(t, schedNode, nodeStore, cs) // Wait until the pod is scheduled. err = wait.Poll(time.Second, wait.ForeverTestTimeout, podScheduled(cs, myPod.Namespace, myPod.Name)) if err != nil { t.Errorf("Test %d: failed to schedule a pod: %v", i, err) } else { t.Logf("Test %d: Pod got scheduled on a schedulable node", i) } err = cs.Core().Pods(ns.Name).Delete(myPod.Name, api.NewDeleteOptions(0)) if err != nil { t.Errorf("Failed to delete pod: %v", err) } err = cs.Core().Nodes().Delete(schedNode.Name, nil) if err != nil { t.Errorf("Failed to delete node: %v", err) } } }
func TestMultiWatch(t *testing.T) { // Disable this test as long as it demonstrates a problem. // TODO: Reenable this test when we get #6059 resolved. return const watcherCount = 50 rt.GOMAXPROCS(watcherCount) framework.DeleteAllEtcdKeys() defer framework.DeleteAllEtcdKeys() _, s := framework.RunAMaster(t) defer s.Close() ns := api.NamespaceDefault client := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}) dummyEvent := func(i int) *api.Event { name := fmt.Sprintf("unrelated-%v", i) return &api.Event{ ObjectMeta: api.ObjectMeta{ Name: fmt.Sprintf("%v.%x", name, time.Now().UnixNano()), Namespace: ns, }, InvolvedObject: api.ObjectReference{ Name: name, Namespace: ns, }, Reason: fmt.Sprintf("unrelated change %v", i), } } type timePair struct { t time.Time name string } receivedTimes := make(chan timePair, watcherCount*2) watchesStarted := sync.WaitGroup{} // make a bunch of pods and watch them for i := 0; i < watcherCount; i++ { watchesStarted.Add(1) name := fmt.Sprintf("multi-watch-%v", i) got, err := client.Pods(ns).Create(&api.Pod{ ObjectMeta: api.ObjectMeta{ Name: name, Labels: labels.Set{"watchlabel": name}, }, Spec: api.PodSpec{ Containers: []api.Container{{ Name: "pause", Image: e2e.GetPauseImageName(client), }}, }, }) if err != nil { t.Fatalf("Couldn't make %v: %v", name, err) } go func(name, rv string) { options := api.ListOptions{ LabelSelector: labels.Set{"watchlabel": name}.AsSelector(), ResourceVersion: rv, } w, err := client.Pods(ns).Watch(options) if err != nil { panic(fmt.Sprintf("watch error for %v: %v", name, err)) } defer w.Stop() watchesStarted.Done() e, ok := <-w.ResultChan() // should get the update (that we'll do below) if !ok { panic(fmt.Sprintf("%v ended early?", name)) } if e.Type != watch.Modified { panic(fmt.Sprintf("Got unexpected watch notification:\n%v: %+v %+v", name, e, e.Object)) } receivedTimes <- timePair{time.Now(), name} }(name, got.ObjectMeta.ResourceVersion) } log.Printf("%v: %v pods made and watchers started", time.Now(), watcherCount) // wait for watches to start before we start spamming the system with // objects below, otherwise we'll hit the watch window restriction. watchesStarted.Wait() const ( useEventsAsUnrelatedType = false usePodsAsUnrelatedType = true ) // make a bunch of unrelated changes in parallel if useEventsAsUnrelatedType { const unrelatedCount = 3000 var wg sync.WaitGroup defer wg.Wait() changeToMake := make(chan int, unrelatedCount*2) changeMade := make(chan int, unrelatedCount*2) go func() { for i := 0; i < unrelatedCount; i++ { changeToMake <- i } close(changeToMake) }() for i := 0; i < 50; i++ { wg.Add(1) go func() { defer wg.Done() for { i, ok := <-changeToMake if !ok { return } if _, err := client.Events(ns).Create(dummyEvent(i)); err != nil { panic(fmt.Sprintf("couldn't make an event: %v", err)) } changeMade <- i } }() } for i := 0; i < 2000; i++ { <-changeMade if (i+1)%50 == 0 { log.Printf("%v: %v unrelated changes made", time.Now(), i+1) } } } if usePodsAsUnrelatedType { const unrelatedCount = 3000 var wg sync.WaitGroup defer wg.Wait() changeToMake := make(chan int, unrelatedCount*2) changeMade := make(chan int, unrelatedCount*2) go func() { for i := 0; i < unrelatedCount; i++ { changeToMake <- i } close(changeToMake) }() for i := 0; i < 50; i++ { wg.Add(1) go func() { defer wg.Done() for { i, ok := <-changeToMake if !ok { return } name := fmt.Sprintf("unrelated-%v", i) _, err := client.Pods(ns).Create(&api.Pod{ ObjectMeta: api.ObjectMeta{ Name: name, }, Spec: api.PodSpec{ Containers: []api.Container{{ Name: "nothing", Image: e2e.GetPauseImageName(client), }}, }, }) if err != nil { panic(fmt.Sprintf("couldn't make unrelated pod: %v", err)) } changeMade <- i } }() } for i := 0; i < 2000; i++ { <-changeMade if (i+1)%50 == 0 { log.Printf("%v: %v unrelated changes made", time.Now(), i+1) } } } // Now we still have changes being made in parallel, but at least 1000 have been made. // Make some updates to send down the watches. sentTimes := make(chan timePair, watcherCount*2) for i := 0; i < watcherCount; i++ { go func(i int) { name := fmt.Sprintf("multi-watch-%v", i) pod, err := client.Pods(ns).Get(name) if err != nil { panic(fmt.Sprintf("Couldn't get %v: %v", name, err)) } pod.Spec.Containers[0].Image = e2e.GetPauseImageName(client) sentTimes <- timePair{time.Now(), name} if _, err := client.Pods(ns).Update(pod); err != nil { panic(fmt.Sprintf("Couldn't make %v: %v", name, err)) } }(i) } sent := map[string]time.Time{} for i := 0; i < watcherCount; i++ { tp := <-sentTimes sent[tp.name] = tp.t } log.Printf("all changes made") dur := map[string]time.Duration{} for i := 0; i < watcherCount; i++ { tp := <-receivedTimes delta := tp.t.Sub(sent[tp.name]) dur[tp.name] = delta log.Printf("%v: %v", tp.name, delta) } log.Printf("all watches ended") t.Errorf("durations: %v", dur) }
itArg := testArg It(name, func() { podsPerNode := itArg.podsPerNode totalPods = podsPerNode * nodeCount fileHndl, err := os.Create(fmt.Sprintf(framework.TestContext.OutputDir+"/%s/pod_states.csv", uuid)) framework.ExpectNoError(err) defer fileHndl.Close() timeout := 10 * time.Minute // TODO: loop to podsPerNode instead of 1 when we're ready. numberOrRCs := 1 RCConfigs := make([]framework.RCConfig, numberOrRCs) for i := 0; i < numberOrRCs; i++ { RCName := "density" + strconv.Itoa(totalPods) + "-" + strconv.Itoa(i) + "-" + uuid RCConfigs[i] = framework.RCConfig{Client: c, Image: framework.GetPauseImageName(f.Client), Name: RCName, Namespace: ns, Labels: map[string]string{"type": "densityPod"}, PollInterval: itArg.interval, PodStatusFile: fileHndl, Replicas: (totalPods + numberOrRCs - 1) / numberOrRCs, CpuRequest: nodeCpuCapacity / 100, MemRequest: nodeMemCapacity / 100, MaxContainerFailures: &MaxContainerFailures, Silent: true, } } dConfig := DensityTestConfig{Client: c, Configs: RCConfigs,
podName := "with-label" _, err := c.Core().Pods(ns).Create(&v1.Pod{ TypeMeta: metav1.TypeMeta{ Kind: "Pod", }, ObjectMeta: v1.ObjectMeta{ Name: podName, Annotations: map[string]string{ "k8s.mesosphere.io/roles": "public", }, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name: podName, Image: framework.GetPauseImageName(f.ClientSet), }, }, }, }) framework.ExpectNoError(err) framework.ExpectNoError(framework.WaitForPodNameRunningInNamespace(c, podName, ns)) pod, err := c.Core().Pods(ns).Get(podName, metav1.GetOptions{}) framework.ExpectNoError(err) nodeClient := f.ClientSet.Core().Nodes() // schedule onto node with rack=2 being assigned to the "public" role rack2 := labels.SelectorFromSet(map[string]string{ "k8s.mesosphere.io/attribute-rack": "2",