// FilterOutSchedulable checks whether pods from <unschedulableCandidates> marked as unschedulable
// by Scheduler actually can't be scheduled on any node and filter out the ones that can.
func FilterOutSchedulable(unschedulableCandidates []*kube_api.Pod, nodes []*kube_api.Node, allPods []*kube_api.Pod, predicateChecker *simulator.PredicateChecker) []*kube_api.Pod {
unschedulablePods := []*kube_api.Pod{}
nodeNameToNodeInfo := createNodeNameToInfoMap(allPods, nodes)
for _, pod := range unschedulableCandidates {
if nodeName, err := predicateChecker.FitsAny(pod, nodeNameToNodeInfo); err == nil {
glog.Warningf("Pod %s marked as unschedulable can be scheduled on %s. Ignoring in scale up.", pod.Name, nodeName)
} else {
unschedulablePods = append(unschedulablePods, pod)
}
}
return unschedulablePods
}
// The caller of this function must remove the taint if this function returns error.
func prepareNodeForPod(client *kube_client.Client, recorder kube_record.EventRecorder, predicateChecker *ca_simulator.PredicateChecker, originalNode *kube_api.Node, criticalPod *kube_api.Pod) error {
// Operate on a copy of the node to ensure pods running on the node will pass CheckPredicates below.
node, err := copyNode(originalNode)
if err != nil {
return fmt.Errorf("Error while copying node: %v", err)
}
err = addTaint(client, originalNode, podId(criticalPod))
if err != nil {
return fmt.Errorf("Error while adding taint: %v", err)
}
requiredPods, otherPods, err := groupPods(client, node)
if err != nil {
return err
}
nodeInfo := schedulercache.NewNodeInfo(requiredPods...)
nodeInfo.SetNode(node)
// check whether critical pod still fit
if err := predicateChecker.CheckPredicates(criticalPod, nodeInfo); err != nil {
return fmt.Errorf("Pod %s doesn't fit to node %v: %v", podId(criticalPod), node.Name, err)
}
requiredPods = append(requiredPods, criticalPod)
nodeInfo = schedulercache.NewNodeInfo(requiredPods...)
nodeInfo.SetNode(node)
for _, p := range otherPods {
if err := predicateChecker.CheckPredicates(p, nodeInfo); err != nil {
glog.Infof("Pod %s will be deleted in order to schedule critical pod %s.", podId(p), podId(criticalPod))
recorder.Eventf(p, kube_api.EventTypeNormal, "DeletedByRescheduler",
"Deleted by rescheduler in order to schedule critical pod %s.", podId(criticalPod))
// TODO(piosz): add better support of graceful deletion
delErr := client.Pods(p.Namespace).Delete(p.Name, kube_api.NewDeleteOptions(10))
if delErr != nil {
return fmt.Errorf("Failed to delete pod %s: %v", podId(p), delErr)
}
} else {
newPods := append(nodeInfo.Pods(), p)
nodeInfo = schedulercache.NewNodeInfo(newPods...)
nodeInfo.SetNode(node)
}
}
// TODO(piosz): how to reset scheduler backoff?
return nil
}
// Currently the logic choose a random node which satisfies requirements (a critical pod fits there).
// TODO(piosz): add a prioritization to this logic
func findNodeForPod(client *kube_client.Client, predicateChecker *ca_simulator.PredicateChecker, nodes []*kube_api.Node, pod *kube_api.Pod) *kube_api.Node {
for _, node := range nodes {
// ignore nodes with taints
if err := checkTaints(node); err != nil {
glog.Warningf("Skipping node %v due to %v", node.Name, err)
}
requiredPods, _, err := groupPods(client, node)
if err != nil {
glog.Warningf("Skipping node %v due to error: %v", node.Name, err)
continue
}
nodeInfo := schedulercache.NewNodeInfo(requiredPods...)
nodeInfo.SetNode(node)
if err := predicateChecker.CheckPredicates(pod, nodeInfo); err == nil {
return node
}
}
return nil
}
// The caller of this function must remove the taint if this function returns error.
func prepareNodeForPod(client *kube_client.Client, predicateChecker *ca_simulator.PredicateChecker, node *kube_api.Node, criticalPod *kube_api.Pod) error {
addTaint(client, node, podId(criticalPod))
requiredPods, otherPods, err := groupPods(client, node)
if err != nil {
return err
}
nodeInfo := schedulercache.NewNodeInfo(requiredPods...)
nodeInfo.SetNode(node)
// check whether critical pod still fit
if err := predicateChecker.CheckPredicates(criticalPod, nodeInfo); err != nil {
return fmt.Errorf("Pod %s doesn't fit to node %v: %v", podId(criticalPod), node.Name, err)
}
requiredPods = append(requiredPods, criticalPod)
nodeInfo = schedulercache.NewNodeInfo(requiredPods...)
nodeInfo.SetNode(node)
for _, p := range otherPods {
if err := predicateChecker.CheckPredicates(p, nodeInfo); err != nil {
glog.Infof("Pod %s will be deleted in order to schedule critical pods %s.", podId(p), podId(criticalPod))
// TODO(piosz): add better support of graceful deletion
delErr := client.Pods(p.Namespace).Delete(p.Name, kube_api.NewDeleteOptions(10))
if delErr != nil {
return fmt.Errorf("Failed to delete pod %s: %v", podId(p), delErr)
}
} else {
newPods := append(nodeInfo.Pods(), p)
nodeInfo = schedulercache.NewNodeInfo(newPods...)
nodeInfo.SetNode(node)
}
}
// TODO(piosz): how to reset scheduler backoff?
return nil
}
// ScaleUp tries to scale the cluster up. Return true if it found a way to increase the size,
// false if it didn't and error if an error occured. Assumes that all nodes in the cluster are
// ready and in sync with instance groups.
func ScaleUp(unschedulablePods []*kube_api.Pod, nodes []*kube_api.Node, cloudProvider cloudprovider.CloudProvider, kubeClient *kube_client.Client,
predicateChecker *simulator.PredicateChecker, recorder kube_record.EventRecorder, maxNodesTotal int,
estimatorName string) (bool, error) {
// From now on we only care about unschedulable pods that were marked after the newest
// node became available for the scheduler.
if len(unschedulablePods) == 0 {
glog.V(1).Info("No unschedulable pods")
return false, nil
}
for _, pod := range unschedulablePods {
glog.V(1).Infof("Pod %s/%s is unschedulable", pod.Namespace, pod.Name)
}
expansionOptions := make([]ExpansionOption, 0)
nodeInfos, err := GetNodeInfosForGroups(nodes, cloudProvider, kubeClient)
if err != nil {
return false, fmt.Errorf("failed to build node infos for node groups: %v", err)
}
podsRemainUnshedulable := make(map[*kube_api.Pod]struct{})
for _, nodeGroup := range cloudProvider.NodeGroups() {
currentSize, err := nodeGroup.TargetSize()
if err != nil {
glog.Errorf("Failed to get node group size: %v", err)
continue
}
if currentSize >= nodeGroup.MaxSize() {
// skip this node group.
glog.V(4).Infof("Skipping node group %s - max size reached", nodeGroup.Id())
continue
}
option := ExpansionOption{
nodeGroup: nodeGroup,
pods: make([]*kube_api.Pod, 0),
}
nodeInfo, found := nodeInfos[nodeGroup.Id()]
if !found {
glog.Errorf("No node info for: %s", nodeGroup.Id())
continue
}
for _, pod := range unschedulablePods {
err = predicateChecker.CheckPredicates(pod, nodeInfo)
if err == nil {
option.pods = append(option.pods, pod)
} else {
glog.V(2).Infof("Scale-up predicate failed: %v", err)
podsRemainUnshedulable[pod] = struct{}{}
}
}
if len(option.pods) > 0 {
if estimatorName == BinpackingEstimatorName {
binpackingEstimator := estimator.NewBinpackingNodeEstimator(predicateChecker)
option.nodeCount = binpackingEstimator.Estimate(option.pods, nodeInfo)
} else if estimatorName == BasicEstimatorName {
basicEstimator := estimator.NewBasicNodeEstimator()
for _, pod := range option.pods {
basicEstimator.Add(pod)
}
option.nodeCount, option.debug = basicEstimator.Estimate(nodeInfo.Node())
} else {
glog.Fatalf("Unrecognized estimator: %s", estimatorName)
}
expansionOptions = append(expansionOptions, option)
}
}
// Pick some expansion option.
bestOption := BestExpansionOption(expansionOptions)
if bestOption != nil && bestOption.nodeCount > 0 {
glog.V(1).Infof("Best option to resize: %s", bestOption.nodeGroup.Id())
if len(bestOption.debug) > 0 {
glog.V(1).Info(bestOption.debug)
}
glog.V(1).Infof("Estimated %d nodes needed in %s", bestOption.nodeCount, bestOption.nodeGroup.Id())
currentSize, err := bestOption.nodeGroup.TargetSize()
if err != nil {
return false, fmt.Errorf("failed to get node group size: %v", err)
}
newSize := currentSize + bestOption.nodeCount
if newSize >= bestOption.nodeGroup.MaxSize() {
glog.V(1).Infof("Capping size to MAX (%d)", bestOption.nodeGroup.MaxSize())
newSize = bestOption.nodeGroup.MaxSize()
}
if maxNodesTotal > 0 && len(nodes)+(newSize-currentSize) > maxNodesTotal {
glog.V(1).Infof("Capping size to max cluster total size (%d)", maxNodesTotal)
newSize = maxNodesTotal - len(nodes) + currentSize
if newSize < currentSize {
return false, fmt.Errorf("max node total count already reached")
}
}
glog.V(0).Infof("Scale-up: setting group %s size to %d", bestOption.nodeGroup.Id(), newSize)
if err := bestOption.nodeGroup.IncreaseSize(newSize - currentSize); err != nil {
return false, fmt.Errorf("failed to increase node group size: %v", err)
}
for _, pod := range bestOption.pods {
recorder.Eventf(pod, kube_api.EventTypeNormal, "TriggeredScaleUp",
"pod triggered scale-up, group: %s, sizes (current/new): %d/%d", bestOption.nodeGroup.Id(), currentSize, newSize)
}
return true, nil
}
for pod := range podsRemainUnshedulable {
recorder.Event(pod, kube_api.EventTypeNormal, "NotTriggerScaleUp",
"pod didn't trigger scale-up (it wouldn't fit if a new node is added)")
}
return false, nil
}
// ScaleUp tries to scale the cluster up. Return true if it found a way to increase the size,
// false if it didn't and error if an error occured.
func ScaleUp(unschedulablePods []*kube_api.Pod, nodes []*kube_api.Node, migConfigs []*config.MigConfig,
gceManager *gce.GceManager, kubeClient *kube_client.Client,
predicateChecker *simulator.PredicateChecker, recorder kube_record.EventRecorder) (bool, error) {
// From now on we only care about unschedulable pods that were marked after the newest
// node became available for the scheduler.
if len(unschedulablePods) == 0 {
glog.V(1).Info("No unschedulable pods")
return false, nil
}
for _, pod := range unschedulablePods {
glog.V(1).Infof("Pod %s/%s is unschedulable", pod.Namespace, pod.Name)
}
expansionOptions := make([]ExpansionOption, 0)
nodeInfos, err := GetNodeInfosForMigs(nodes, gceManager, kubeClient)
if err != nil {
return false, fmt.Errorf("failed to build node infors for migs: %v", err)
}
podsRemainUnshedulable := make(map[*kube_api.Pod]struct{})
for _, migConfig := range migConfigs {
currentSize, err := gceManager.GetMigSize(migConfig)
if err != nil {
glog.Errorf("Failed to get MIG size: %v", err)
continue
}
if currentSize >= int64(migConfig.MaxSize) {
// skip this mig.
glog.V(4).Infof("Skipping MIG %s - max size reached", migConfig.Url())
continue
}
option := ExpansionOption{
migConfig: migConfig,
estimator: estimator.NewBasicNodeEstimator(),
}
migHelpsSomePods := false
nodeInfo, found := nodeInfos[migConfig.Url()]
if !found {
glog.Errorf("No node info for: %s", migConfig.Url())
continue
}
for _, pod := range unschedulablePods {
err = predicateChecker.CheckPredicates(pod, nodeInfo)
if err == nil {
migHelpsSomePods = true
option.estimator.Add(pod)
} else {
glog.V(2).Infof("Scale-up predicate failed: %v", err)
podsRemainUnshedulable[pod] = struct{}{}
}
}
if migHelpsSomePods {
expansionOptions = append(expansionOptions, option)
}
}
// Pick some expansion option.
bestOption := BestExpansionOption(expansionOptions)
if bestOption != nil && bestOption.estimator.GetCount() > 0 {
glog.V(1).Infof("Best option to resize: %s", bestOption.migConfig.Url())
nodeInfo, found := nodeInfos[bestOption.migConfig.Url()]
if !found {
return false, fmt.Errorf("no sample node for: %s", bestOption.migConfig.Url())
}
node := nodeInfo.Node()
estimate, report := bestOption.estimator.Estimate(node)
glog.V(1).Info(bestOption.estimator.GetDebug())
glog.V(1).Info(report)
glog.V(1).Infof("Estimated %d nodes needed in %s", estimate, bestOption.migConfig.Url())
currentSize, err := gceManager.GetMigSize(bestOption.migConfig)
if err != nil {
return false, fmt.Errorf("failed to get MIG size: %v", err)
}
newSize := currentSize + int64(estimate)
if newSize >= int64(bestOption.migConfig.MaxSize) {
glog.V(1).Infof("Capping size to MAX (%d)", bestOption.migConfig.MaxSize)
newSize = int64(bestOption.migConfig.MaxSize)
}
glog.V(1).Infof("Setting %s size to %d", bestOption.migConfig.Url(), newSize)
if err := gceManager.SetMigSize(bestOption.migConfig, newSize); err != nil {
return false, fmt.Errorf("failed to set MIG size: %v", err)
}
for pod := range bestOption.estimator.FittingPods {
recorder.Eventf(pod, kube_api.EventTypeNormal, "TriggeredScaleUp",
"pod triggered scale-up, mig: %s, sizes (current/new): %d/%d", bestOption.migConfig.Name, currentSize, newSize)
}
return true, nil
}
for pod := range podsRemainUnshedulable {
recorder.Event(pod, kube_api.EventTypeNormal, "NotTriggerScaleUp",
"pod didn't trigger scale-up (it wouldn't fit if a new node is added)")
}
return false, nil
}