// benchmarkScheduling benchmarks scheduling rate with specific number of nodes
// and specific number of pods already scheduled. Since an operation takes relatively
// long time, b.N should be small: 10 - 100.
func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) {
schedulerConfigFactory, finalFunc := mustSetupScheduler()
defer finalFunc()
c := schedulerConfigFactory.GetClient()
nodePreparer := framework.NewIntegrationTestNodePreparer(
c,
[]testutils.CountToStrategy{{Count: numNodes, Strategy: &testutils.TrivialNodePrepareStrategy{}}},
"scheduler-perf-",
)
if err := nodePreparer.PrepareNodes(); err != nil {
glog.Fatalf("%v", err)
}
defer nodePreparer.CleanupNodes()
config := testutils.NewTestPodCreatorConfig()
config.AddStrategy("sched-test", numScheduledPods, testutils.NewSimpleWithControllerCreatePodStrategy("rc1"))
podCreator := testutils.NewTestPodCreator(c, config)
podCreator.CreatePods()
for {
scheduled := schedulerConfigFactory.GetScheduledPodListerIndexer().List()
if len(scheduled) >= numScheduledPods {
break
}
time.Sleep(1 * time.Second)
}
// start benchmark
b.ResetTimer()
config = testutils.NewTestPodCreatorConfig()
config.AddStrategy("sched-test", b.N, testutils.NewSimpleWithControllerCreatePodStrategy("rc2"))
podCreator = testutils.NewTestPodCreator(c, config)
podCreator.CreatePods()
for {
// This can potentially affect performance of scheduler, since List() is done under mutex.
// TODO: Setup watch on apiserver and wait until all pods scheduled.
scheduled := schedulerConfigFactory.GetScheduledPodListerIndexer().List()
if len(scheduled) >= numScheduledPods+b.N {
break
}
// Note: This might introduce slight deviation in accuracy of benchmark results.
// Since the total amount of time is relatively large, it might not be a concern.
time.Sleep(100 * time.Millisecond)
}
}
// schedulePods schedules specific number of pods on specific number of nodes.
// This is used to learn the scheduling throughput on various
// sizes of cluster and changes as more and more pods are scheduled.
// It won't stop until all pods are scheduled.
// It retruns the minimum of throughput over whole run.
func schedulePods(numNodes, numPods int) int32 {
schedulerConfigFactory, destroyFunc := mustSetupScheduler()
defer destroyFunc()
c := schedulerConfigFactory.Client
nodePreparer := framework.NewIntegrationTestNodePreparer(
c,
[]testutils.CountToStrategy{{Count: numNodes, Strategy: &testutils.TrivialNodePrepareStrategy{}}},
"scheduler-perf-",
)
if err := nodePreparer.PrepareNodes(); err != nil {
glog.Fatalf("%v", err)
}
defer nodePreparer.CleanupNodes()
config := testutils.NewTestPodCreatorConfig()
config.AddStrategy("sched-test", numPods, testutils.NewSimpleWithControllerCreatePodStrategy("rc1"))
podCreator := testutils.NewTestPodCreator(c, config)
podCreator.CreatePods()
prev := 0
minQps := int32(math.MaxInt32)
start := time.Now()
for {
// This can potentially affect performance of scheduler, since List() is done under mutex.
// Listing 10000 pods is an expensive operation, so running it frequently may impact scheduler.
// TODO: Setup watch on apiserver and wait until all pods scheduled.
scheduled := schedulerConfigFactory.ScheduledPodLister.Indexer.List()
if len(scheduled) >= numPods {
fmt.Printf("Scheduled %v Pods in %v seconds (%v per second on average).\n",
numPods, int(time.Since(start)/time.Second), numPods/int(time.Since(start)/time.Second))
return minQps
}
// There's no point in printing it for the last iteration, as the value is random
qps := len(scheduled) - prev
if int32(qps) < minQps {
minQps = int32(qps)
}
fmt.Printf("%ds\trate: %d\ttotal: %d\n", time.Since(start)/time.Second, qps, len(scheduled))
prev = len(scheduled)
time.Sleep(1 * time.Second)
}
}
func defaultSchedulerBenchmarkConfig(numNodes, numPods int) *testConfig {
baseConfig := baseConfig()
nodePreparer := framework.NewIntegrationTestNodePreparer(
baseConfig.schedulerConfigFactory.Client,
[]testutils.CountToStrategy{{Count: numNodes, Strategy: &testutils.TrivialNodePrepareStrategy{}}},
"scheduler-perf-",
)
config := testutils.NewTestPodCreatorConfig()
config.AddStrategy("sched-test", numPods, testutils.NewSimpleWithControllerCreatePodStrategy("rc1"))
podCreator := testutils.NewTestPodCreator(baseConfig.schedulerConfigFactory.Client, config)
baseConfig.nodePreparer = nodePreparer
baseConfig.podCreator = podCreator
baseConfig.numPods = numPods
baseConfig.numNodes = numNodes
return baseConfig
}
// TestSchedule100Node3KNodeAffinityPods schedules 3k pods using Node affinity on 100 nodes.
func TestSchedule100Node3KNodeAffinityPods(t *testing.T) {
if testing.Short() {
t.Skip("Skipping because we want to run short tests")
}
config := baseConfig()
config.numNodes = 100
config.numPods = 3000
// number of Node-Pod sets with Pods NodeAffinity matching given Nodes.
numGroups := 10
nodeAffinityKey := "kubernetes.io/sched-perf-node-affinity"
nodeStrategies := make([]testutils.CountToStrategy, 0, 10)
for i := 0; i < numGroups; i++ {
nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{
Count: config.numNodes / numGroups,
Strategy: testutils.NewLabelNodePrepareStrategy(nodeAffinityKey, fmt.Sprintf("%v", i)),
})
}
config.nodePreparer = framework.NewIntegrationTestNodePreparer(
config.schedulerConfigFactory.Client,
nodeStrategies,
"scheduler-perf-",
)
affinityTemplate := dedent.Dedent(`
{
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "` + nodeAffinityKey + `",
"operator": "In",
"values": ["%v"]
}]
}]
}
}
}`)
podCreatorConfig := testutils.NewTestPodCreatorConfig()
for i := 0; i < numGroups; i++ {
podCreatorConfig.AddStrategy("sched-perf-node-affinity", config.numPods/numGroups,
testutils.NewCustomCreatePodStrategy(&v1.Pod{
ObjectMeta: v1.ObjectMeta{
GenerateName: "sched-perf-node-affinity-pod-",
Annotations: map[string]string{v1.AffinityAnnotationKey: fmt.Sprintf(affinityTemplate, i)},
},
Spec: testutils.MakePodSpec(),
}),
)
}
config.podCreator = testutils.NewTestPodCreator(config.schedulerConfigFactory.Client, podCreatorConfig)
if min := schedulePods(config); min < threshold30K {
t.Errorf("Too small pod scheduling throughput for 30k pods. Expected %v got %v", threshold30K, min)
} else {
fmt.Printf("Minimal observed throughput for 30k pod test: %v\n", min)
}
}