// benchmarkScheduling benchmarks scheduling rate with specific number of nodes
// and specific number of pods already scheduled. Since an operation takes relatively
// long time, b.N should be small: 10 - 100.
func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) {
	schedulerConfigFactory, finalFunc := mustSetupScheduler()
	defer finalFunc()
	c := schedulerConfigFactory.GetClient()

	nodePreparer := framework.NewIntegrationTestNodePreparer(
		c,
		[]testutils.CountToStrategy{{Count: numNodes, Strategy: &testutils.TrivialNodePrepareStrategy{}}},
		"scheduler-perf-",
	)
	if err := nodePreparer.PrepareNodes(); err != nil {
		glog.Fatalf("%v", err)
	}
	defer nodePreparer.CleanupNodes()

	config := testutils.NewTestPodCreatorConfig()
	config.AddStrategy("sched-test", numScheduledPods, testutils.NewSimpleWithControllerCreatePodStrategy("rc1"))
	podCreator := testutils.NewTestPodCreator(c, config)
	podCreator.CreatePods()

	for {
		scheduled := schedulerConfigFactory.GetScheduledPodListerIndexer().List()
		if len(scheduled) >= numScheduledPods {
			break
		}
		time.Sleep(1 * time.Second)
	}
	// start benchmark
	b.ResetTimer()
	config = testutils.NewTestPodCreatorConfig()
	config.AddStrategy("sched-test", b.N, testutils.NewSimpleWithControllerCreatePodStrategy("rc2"))
	podCreator = testutils.NewTestPodCreator(c, config)
	podCreator.CreatePods()
	for {
		// This can potentially affect performance of scheduler, since List() is done under mutex.
		// TODO: Setup watch on apiserver and wait until all pods scheduled.
		scheduled := schedulerConfigFactory.GetScheduledPodListerIndexer().List()
		if len(scheduled) >= numScheduledPods+b.N {
			break
		}
		// Note: This might introduce slight deviation in accuracy of benchmark results.
		// Since the total amount of time is relatively large, it might not be a concern.
		time.Sleep(100 * time.Millisecond)
	}
}
// schedulePods schedules specific number of pods on specific number of nodes.
// This is used to learn the scheduling throughput on various
// sizes of cluster and changes as more and more pods are scheduled.
// It won't stop until all pods are scheduled.
// It retruns the minimum of throughput over whole run.
func schedulePods(numNodes, numPods int) int32 {
	schedulerConfigFactory, destroyFunc := mustSetupScheduler()
	defer destroyFunc()
	c := schedulerConfigFactory.Client

	nodePreparer := framework.NewIntegrationTestNodePreparer(
		c,
		[]testutils.CountToStrategy{{Count: numNodes, Strategy: &testutils.TrivialNodePrepareStrategy{}}},
		"scheduler-perf-",
	)
	if err := nodePreparer.PrepareNodes(); err != nil {
		glog.Fatalf("%v", err)
	}
	defer nodePreparer.CleanupNodes()

	config := testutils.NewTestPodCreatorConfig()
	config.AddStrategy("sched-test", numPods, testutils.NewSimpleWithControllerCreatePodStrategy("rc1"))
	podCreator := testutils.NewTestPodCreator(c, config)
	podCreator.CreatePods()

	prev := 0
	minQps := int32(math.MaxInt32)
	start := time.Now()
	for {
		// This can potentially affect performance of scheduler, since List() is done under mutex.
		// Listing 10000 pods is an expensive operation, so running it frequently may impact scheduler.
		// TODO: Setup watch on apiserver and wait until all pods scheduled.
		scheduled := schedulerConfigFactory.ScheduledPodLister.Indexer.List()
		if len(scheduled) >= numPods {
			fmt.Printf("Scheduled %v Pods in %v seconds (%v per second on average).\n",
				numPods, int(time.Since(start)/time.Second), numPods/int(time.Since(start)/time.Second))
			return minQps
		}
		// There's no point in printing it for the last iteration, as the value is random
		qps := len(scheduled) - prev
		if int32(qps) < minQps {
			minQps = int32(qps)
		}

		fmt.Printf("%ds\trate: %d\ttotal: %d\n", time.Since(start)/time.Second, qps, len(scheduled))
		prev = len(scheduled)
		time.Sleep(1 * time.Second)
	}
}
func defaultSchedulerBenchmarkConfig(numNodes, numPods int) *testConfig {
	baseConfig := baseConfig()

	nodePreparer := framework.NewIntegrationTestNodePreparer(
		baseConfig.schedulerConfigFactory.Client,
		[]testutils.CountToStrategy{{Count: numNodes, Strategy: &testutils.TrivialNodePrepareStrategy{}}},
		"scheduler-perf-",
	)

	config := testutils.NewTestPodCreatorConfig()
	config.AddStrategy("sched-test", numPods, testutils.NewSimpleWithControllerCreatePodStrategy("rc1"))
	podCreator := testutils.NewTestPodCreator(baseConfig.schedulerConfigFactory.Client, config)

	baseConfig.nodePreparer = nodePreparer
	baseConfig.podCreator = podCreator
	baseConfig.numPods = numPods
	baseConfig.numNodes = numNodes

	return baseConfig
}
// TestSchedule100Node3KNodeAffinityPods schedules 3k pods using Node affinity on 100 nodes.
func TestSchedule100Node3KNodeAffinityPods(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping because we want to run short tests")
	}

	config := baseConfig()
	config.numNodes = 100
	config.numPods = 3000

	// number of Node-Pod sets with Pods NodeAffinity matching given Nodes.
	numGroups := 10
	nodeAffinityKey := "kubernetes.io/sched-perf-node-affinity"

	nodeStrategies := make([]testutils.CountToStrategy, 0, 10)
	for i := 0; i < numGroups; i++ {
		nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{
			Count:    config.numNodes / numGroups,
			Strategy: testutils.NewLabelNodePrepareStrategy(nodeAffinityKey, fmt.Sprintf("%v", i)),
		})
	}
	config.nodePreparer = framework.NewIntegrationTestNodePreparer(
		config.schedulerConfigFactory.Client,
		nodeStrategies,
		"scheduler-perf-",
	)

	affinityTemplate := dedent.Dedent(`
		{
			"nodeAffinity": {
				"requiredDuringSchedulingIgnoredDuringExecution": {
					"nodeSelectorTerms": [{
						"matchExpressions": [{
							"key": "` + nodeAffinityKey + `",
							"operator": "In",
							"values": ["%v"]
						}]
					}]
				}
			}
		}`)

	podCreatorConfig := testutils.NewTestPodCreatorConfig()
	for i := 0; i < numGroups; i++ {
		podCreatorConfig.AddStrategy("sched-perf-node-affinity", config.numPods/numGroups,
			testutils.NewCustomCreatePodStrategy(&v1.Pod{
				ObjectMeta: v1.ObjectMeta{
					GenerateName: "sched-perf-node-affinity-pod-",
					Annotations:  map[string]string{v1.AffinityAnnotationKey: fmt.Sprintf(affinityTemplate, i)},
				},
				Spec: testutils.MakePodSpec(),
			}),
		)
	}
	config.podCreator = testutils.NewTestPodCreator(config.schedulerConfigFactory.Client, podCreatorConfig)

	if min := schedulePods(config); min < threshold30K {
		t.Errorf("Too small pod scheduling throughput for 30k pods. Expected %v got %v", threshold30K, min)
	} else {
		fmt.Printf("Minimal observed throughput for 30k pod test: %v\n", min)
	}
}