} if len(nodeErrs) > 0 { errList = append(errList, fmt.Sprintf("node %v:\n %s", nodeName, strings.Join(nodeErrs, ", "))) } } if len(errList) > 0 { framework.Failf("CPU usage exceeding limits:\n %s", strings.Join(errList, "\n")) } } // Slow by design (1 hour) var _ = framework.KubeDescribe("Kubelet [Serial] [Slow]", func() { var nodeNames sets.String f := framework.NewDefaultFramework("kubelet-perf") var om *framework.RuntimeOperationMonitor var rm *framework.ResourceMonitor BeforeEach(func() { // Wait until image prepull pod has completed so that they wouldn't // affect the runtime cpu usage. Fail the test if prepulling cannot // finish in time. if err := framework.WaitForPodsSuccess(f.ClientSet, api.NamespaceSystem, framework.ImagePullerLabels, imagePrePullingLongTimeout); err != nil { framework.Failf("Image puller didn't complete in %v, not running resource usage test since the metrics might be adultrated", imagePrePullingLongTimeout) } nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) nodeNames = sets.NewString() for _, node := range nodes.Items { nodeNames.Insert(node.Name) } om = framework.NewRuntimeOperationMonitor(f.ClientSet) rm = framework.NewResourceMonitor(f.ClientSet, framework.TargetContainers(), containerStatsPollingPeriod)
func runResourceTrackingTest(f *framework.Framework, podsPerNode int, nodeNames sets.String, rm *framework.ResourceMonitor, expectedCPU map[string]map[float64]float64, expectedMemory framework.ResourceUsagePerContainer) { numNodes := nodeNames.Len() totalPods := podsPerNode * numNodes By(fmt.Sprintf("Creating a RC of %d pods and wait until all pods of this RC are running", totalPods)) rcName := fmt.Sprintf("resource%d-%s", totalPods, string(uuid.NewUUID())) // TODO: Use a more realistic workload Expect(framework.RunRC(testutils.RCConfig{ Client: f.ClientSet, Name: rcName, Namespace: f.Namespace.Name, Image: framework.GetPauseImageName(f.ClientSet), Replicas: totalPods, })).NotTo(HaveOccurred()) // Log once and flush the stats. rm.LogLatest() rm.Reset() By("Start monitoring resource usage") // Periodically dump the cpu summary until the deadline is met. // Note that without calling framework.ResourceMonitor.Reset(), the stats // would occupy increasingly more memory. This should be fine // for the current test duration, but we should reclaim the // entries if we plan to monitor longer (e.g., 8 hours). deadline := time.Now().Add(monitoringTime) for time.Now().Before(deadline) { timeLeft := deadline.Sub(time.Now()) framework.Logf("Still running...%v left", timeLeft) if timeLeft < reportingPeriod { time.Sleep(timeLeft) } else { time.Sleep(reportingPeriod) } logPodsOnNodes(f.ClientSet, nodeNames.List()) } By("Reporting overall resource usage") logPodsOnNodes(f.ClientSet, nodeNames.List()) usageSummary, err := rm.GetLatest() Expect(err).NotTo(HaveOccurred()) // TODO(random-liu): Remove the original log when we migrate to new perfdash framework.Logf("%s", rm.FormatResourceUsage(usageSummary)) // Log perf result framework.PrintPerfData(framework.ResourceUsageToPerfData(rm.GetMasterNodeLatest(usageSummary))) verifyMemoryLimits(f.ClientSet, expectedMemory, usageSummary) cpuSummary := rm.GetCPUSummary() framework.Logf("%s", rm.FormatCPUSummary(cpuSummary)) // Log perf result framework.PrintPerfData(framework.CPUUsageToPerfData(rm.GetMasterNodeCPUSummary(cpuSummary))) verifyCPULimits(expectedCPU, cpuSummary) By("Deleting the RC") framework.DeleteRCAndPods(f.ClientSet, f.Namespace.Name, rcName) }
framework.Logf("Error updating node %s: %v", nodeName, err) } else { break } } Expect(err).NotTo(HaveOccurred()) } } var _ = framework.KubeDescribe("kubelet", func() { var c clientset.Interface var numNodes int var nodeNames sets.String var nodeLabels map[string]string f := framework.NewDefaultFramework("kubelet") var resourceMonitor *framework.ResourceMonitor BeforeEach(func() { c = f.ClientSet // Use node labels to restrict the pods to be assigned only to the // nodes we observe initially. nodeLabels = make(map[string]string) nodeLabels["kubelet_cleanup"] = "true" nodes := framework.GetReadySchedulableNodesOrDie(c) numNodes = len(nodes.Items) nodeNames = sets.NewString() // If there are a lot of nodes, we don't want to use all of them // (if there are 1000 nodes in the cluster, starting 10 pods/node // will take ~10 minutes today). And there is also deletion phase. // Instead, we choose at most 10 nodes.