// FindUnneededNodes calculates which nodes are not needed, i.e. all pods can be scheduled somewhere else,
// and updates unneededNodes map accordingly. It also returns information where pods can be rescheduld and
// node utilization level.
func FindUnneededNodes(
context AutoscalingContext,
nodes []*apiv1.Node,
unneededNodes map[string]time.Time,
pods []*apiv1.Pod,
oldHints map[string]string,
tracker *simulator.UsageTracker,
timestamp time.Time) (unnededTimeMap map[string]time.Time, podReschedulingHints map[string]string, utilizationMap map[string]float64) {
currentlyUnneededNodes := make([]*apiv1.Node, 0)
nodeNameToNodeInfo := schedulercache.CreateNodeNameToInfoMap(pods, nodes)
utilizationMap = make(map[string]float64)
// Phase1 - look at the nodes utilization.
for _, node := range nodes {
nodeInfo, found := nodeNameToNodeInfo[node.Name]
if !found {
glog.Errorf("Node info for %s not found", node.Name)
continue
}
utilization, err := simulator.CalculateUtilization(node, nodeInfo)
if err != nil {
glog.Warningf("Failed to calculate utilization for %s: %v", node.Name, err)
}
glog.V(4).Infof("Node %s - utilization %f", node.Name, utilization)
utilizationMap[node.Name] = utilization
if utilization >= context.ScaleDownUtilizationThreshold {
glog.V(4).Infof("Node %s is not suitable for removal - utilization too big (%f)", node.Name, utilization)
continue
}
currentlyUnneededNodes = append(currentlyUnneededNodes, node)
}
// Phase2 - check which nodes can be probably removed using fast drain.
nodesToRemove, newHints, err := simulator.FindNodesToRemove(currentlyUnneededNodes, nodes, pods,
nil, context.PredicateChecker,
len(currentlyUnneededNodes), true, oldHints, tracker, timestamp)
if err != nil {
glog.Errorf("Error while simulating node drains: %v", err)
return map[string]time.Time{}, oldHints, map[string]float64{}
}
// Update the timestamp map.
now := time.Now()
result := make(map[string]time.Time)
for _, node := range nodesToRemove {
name := node.Node.Name
if val, found := unneededNodes[name]; !found {
result[name] = now
} else {
result[name] = val
}
}
return result, newHints, utilizationMap
}
// CalculateUnderutilizedNodes calculates which nodes are underutilized.
func CalculateUnderutilizedNodes(nodes []*kube_api.Node,
underutilizedNodes map[string]time.Time,
utilizationThreshold float64,
pods []*kube_api.Pod,
client *kube_client.Client,
predicateChecker *simulator.PredicateChecker) map[string]time.Time {
currentlyUnderutilizedNodes := make([]*kube_api.Node, 0)
nodeNameToNodeInfo := schedulercache.CreateNodeNameToInfoMap(pods)
// Phase1 - look at the nodes reservation.
for _, node := range nodes {
nodeInfo, found := nodeNameToNodeInfo[node.Name]
if !found {
glog.Errorf("Node info for %s not found", node.Name)
continue
}
reservation, err := simulator.CalculateReservation(node, nodeInfo)
if err != nil {
glog.Warningf("Failed to calculate reservation for %s: %v", node.Name, err)
}
glog.V(4).Infof("Node %s - reservation %f", node.Name, reservation)
if reservation >= utilizationThreshold {
glog.V(4).Infof("Node %s is not suitable for removal - reservation to big (%f)", node.Name, reservation)
continue
}
currentlyUnderutilizedNodes = append(currentlyUnderutilizedNodes, node)
}
// Phase2 - check which nodes can be probably removed using fast drain.
nodesToRemove, err := simulator.FindNodesToRemove(currentlyUnderutilizedNodes, nodes, pods,
client, predicateChecker,
len(currentlyUnderutilizedNodes), true)
if err != nil {
glog.Errorf("Error while evaluating node utilization: %v", err)
return map[string]time.Time{}
}
// Update the timestamp map.
now := time.Now()
result := make(map[string]time.Time)
for _, node := range nodesToRemove {
name := node.Name
if val, found := underutilizedNodes[name]; !found {
result[name] = now
} else {
result[name] = val
}
}
return result
}
// ScaleDown tries to scale down the cluster. It returns ScaleDownResult indicating if any node was
// removed and error if such occured.
func ScaleDown(
nodes []*kube_api.Node,
unneededNodes map[string]time.Time,
unneededTime time.Duration,
pods []*kube_api.Pod,
cloudProvider cloudprovider.CloudProvider,
client *kube_client.Client,
predicateChecker *simulator.PredicateChecker,
oldHints map[string]string,
usageTracker *simulator.UsageTracker,
recorder kube_record.EventRecorder) (ScaleDownResult, error) {
now := time.Now()
candidates := make([]*kube_api.Node, 0)
for _, node := range nodes {
if val, found := unneededNodes[node.Name]; found {
glog.V(2).Infof("%s was unneeded for %s", node.Name, now.Sub(val).String())
// Check how long the node was underutilized.
if !val.Add(unneededTime).Before(now) {
continue
}
nodeGroup, err := cloudProvider.NodeGroupForNode(node)
if err != nil {
glog.Errorf("Error while checking node group for %s: %v", node.Name, err)
continue
}
if nodeGroup == nil || reflect.ValueOf(nodeGroup).IsNil() {
glog.V(4).Infof("Skipping %s - no node group config", node.Name)
continue
}
size, err := nodeGroup.TargetSize()
if err != nil {
glog.Errorf("Error while checking node group size %s: %v", nodeGroup.Id(), err)
continue
}
if size <= nodeGroup.MinSize() {
glog.V(1).Infof("Skipping %s - node group min size reached", node.Name)
continue
}
candidates = append(candidates, node)
}
}
if len(candidates) == 0 {
glog.Infof("No candidates for scale down")
return ScaleDownNoUnneeded, nil
}
// We look for only 1 node so new hints may be incomplete.
nodesToRemove, _, err := simulator.FindNodesToRemove(candidates, nodes, pods, client, predicateChecker, 1, false,
oldHints, usageTracker, time.Now())
if err != nil {
return ScaleDownError, fmt.Errorf("Find node to remove failed: %v", err)
}
if len(nodesToRemove) == 0 {
glog.V(1).Infof("No node to remove")
return ScaleDownNoNodeDeleted, nil
}
nodeToRemove := nodesToRemove[0]
glog.Infof("Removing %s", nodeToRemove.Name)
nodeGroup, err := cloudProvider.NodeGroupForNode(nodeToRemove)
if err != nil {
return ScaleDownError, fmt.Errorf("failed to node group for %s: %v", nodeToRemove.Name, err)
}
if nodeGroup == nil || reflect.ValueOf(nodeGroup).IsNil() {
return ScaleDownError, fmt.Errorf("picked node that doesn't belong to a node group: %s", nodeToRemove.Name)
}
err = nodeGroup.DeleteNodes([]*kube_api.Node{nodeToRemove})
simulator.RemoveNodeFromTracker(usageTracker, nodeToRemove.Name, unneededNodes)
if err != nil {
return ScaleDownError, fmt.Errorf("Failed to delete %s: %v", nodeToRemove.Name, err)
}
recorder.Eventf(nodeToRemove, kube_api.EventTypeNormal, "ScaleDown",
"node removed by cluster autoscaler")
return ScaleDownNodeDeleted, nil
}
// ScaleDown tries to scale down the cluster. It returns ScaleDownResult indicating if any node was
// removed and error if such occured.
func ScaleDown(
nodes []*kube_api.Node,
unneededNodes map[string]time.Time,
unneededTime time.Duration,
pods []*kube_api.Pod,
cloudProvider cloudprovider.CloudProvider,
client *kube_client.Client,
predicateChecker *simulator.PredicateChecker) (ScaleDownResult, error) {
now := time.Now()
candidates := make([]*kube_api.Node, 0)
for _, node := range nodes {
if val, found := unneededNodes[node.Name]; found {
glog.V(2).Infof("%s was unneeded for %s", node.Name, now.Sub(val).String())
// Check how long the node was underutilized.
if !val.Add(unneededTime).Before(now) {
continue
}
nodeGroup, err := cloudProvider.NodeGroupForNode(node)
if err != nil {
glog.Errorf("Error while checking node group for %s: %v", node.Name, err)
continue
}
if nodeGroup == nil {
glog.V(4).Infof("Skipping %s - no node group config", node.Name)
continue
}
size, err := nodeGroup.TargetSize()
if err != nil {
glog.Errorf("Error while checking node group size %s: %v", nodeGroup.Id(), err)
continue
}
if size <= nodeGroup.MinSize() {
glog.V(1).Infof("Skipping %s - node group min size reached", node.Name)
continue
}
candidates = append(candidates, node)
}
}
if len(candidates) == 0 {
glog.Infof("No candidates for scale down")
return ScaleDownNoUnneeded, nil
}
nodesToRemove, err := simulator.FindNodesToRemove(candidates, nodes, pods, client, predicateChecker, 1, false)
if err != nil {
return ScaleDownError, fmt.Errorf("Find node to remove failed: %v", err)
}
if len(nodesToRemove) == 0 {
glog.V(1).Infof("No node to remove")
return ScaleDownNoNodeDeleted, nil
}
nodeToRemove := nodesToRemove[0]
glog.Infof("Removing %s", nodeToRemove.Name)
nodeGroup, err := cloudProvider.NodeGroupForNode(nodeToRemove)
if err != nil {
return ScaleDownError, fmt.Errorf("failed to node group for %s: %v", nodeToRemove.Name, err)
}
if nodeGroup == nil {
return ScaleDownError, fmt.Errorf("picked node that doesn't belong to a node group: %s", nodeToRemove.Name)
}
err = nodeGroup.DeleteNodes([]*kube_api.Node{nodeToRemove})
if err != nil {
return ScaleDownError, fmt.Errorf("Failed to delete %s: %v", nodeToRemove.Name, err)
}
return ScaleDownNodeDeleted, nil
}
// ScaleDown tries to scale down the cluster. It returns ScaleDownResult indicating if any node was
// removed and error if such occured.
func ScaleDown(
context AutoscalingContext,
nodes []*apiv1.Node,
lastUtilizationMap map[string]float64,
unneededNodes map[string]time.Time,
pods []*apiv1.Pod,
oldHints map[string]string,
usageTracker *simulator.UsageTracker,
) (ScaleDownResult, error) {
now := time.Now()
candidates := make([]*apiv1.Node, 0)
for _, node := range nodes {
if val, found := unneededNodes[node.Name]; found {
glog.V(2).Infof("%s was unneeded for %s", node.Name, now.Sub(val).String())
// Check how long the node was underutilized.
if !val.Add(context.ScaleDownUnneededTime).Before(now) {
continue
}
nodeGroup, err := context.CloudProvider.NodeGroupForNode(node)
if err != nil {
glog.Errorf("Error while checking node group for %s: %v", node.Name, err)
continue
}
if nodeGroup == nil || reflect.ValueOf(nodeGroup).IsNil() {
glog.V(4).Infof("Skipping %s - no node group config", node.Name)
continue
}
size, err := nodeGroup.TargetSize()
if err != nil {
glog.Errorf("Error while checking node group size %s: %v", nodeGroup.Id(), err)
continue
}
if size <= nodeGroup.MinSize() {
glog.V(1).Infof("Skipping %s - node group min size reached", node.Name)
continue
}
candidates = append(candidates, node)
}
}
if len(candidates) == 0 {
glog.Infof("No candidates for scale down")
return ScaleDownNoUnneeded, nil
}
// Trying to delete empty nodes in bulk. If there are no empty nodes then CA will
// try to delete not-so-empty nodes, possibly killing some pods and allowing them
// to recreate on other nodes.
emptyNodes := getEmptyNodes(candidates, pods, context.MaxEmptyBulkDelete, context.CloudProvider)
if len(emptyNodes) > 0 {
confirmation := make(chan error, len(emptyNodes))
for _, node := range emptyNodes {
glog.V(0).Infof("Scale-down: removing empty node %s", node.Name)
simulator.RemoveNodeFromTracker(usageTracker, node.Name, unneededNodes)
go func(nodeToDelete *apiv1.Node) {
confirmation <- deleteNodeFromCloudProvider(nodeToDelete, context.CloudProvider, context.Recorder)
}(node)
}
var finalError error
for range emptyNodes {
if err := <-confirmation; err != nil {
glog.Errorf("Problem with empty node deletion: %v", err)
finalError = err
}
}
if finalError == nil {
return ScaleDownNodeDeleted, nil
}
return ScaleDownError, fmt.Errorf("failed to delete at least one empty node: %v", finalError)
}
// We look for only 1 node so new hints may be incomplete.
nodesToRemove, _, err := simulator.FindNodesToRemove(candidates, nodes, pods, context.ClientSet,
context.PredicateChecker, 1, false,
oldHints, usageTracker, time.Now())
if err != nil {
return ScaleDownError, fmt.Errorf("Find node to remove failed: %v", err)
}
if len(nodesToRemove) == 0 {
glog.V(1).Infof("No node to remove")
return ScaleDownNoNodeDeleted, nil
}
toRemove := nodesToRemove[0]
utilization := lastUtilizationMap[toRemove.Node.Name]
podNames := make([]string, 0, len(toRemove.PodsToReschedule))
for _, pod := range toRemove.PodsToReschedule {
podNames = append(podNames, pod.Namespace+"/"+pod.Name)
}
glog.V(0).Infof("Scale-down: removing node %s, utilization: %v, pods to reschedule: ", toRemove.Node.Name, utilization,
strings.Join(podNames, ","))
// Nothing super-bad should happen if the node is removed from tracker prematurely.
simulator.RemoveNodeFromTracker(usageTracker, toRemove.Node.Name, unneededNodes)
err = deleteNode(context, toRemove.Node, toRemove.PodsToReschedule)
if err != nil {
return ScaleDownError, fmt.Errorf("Failed to delete %s: %v", toRemove.Node.Name, err)
}
return ScaleDownNodeDeleted, nil
}
// ScaleDown tries to scale down the cluster. It returns ScaleDownResult indicating if any node was
// removed and error if such occured.
func ScaleDown(
nodes []*kube_api.Node,
unneededNodes map[string]time.Time,
unneededTime time.Duration,
pods []*kube_api.Pod,
gceManager *gce.GceManager,
client *kube_client.Client,
predicateChecker *simulator.PredicateChecker) (ScaleDownResult, error) {
now := time.Now()
candidates := make([]*kube_api.Node, 0)
for _, node := range nodes {
if val, found := unneededNodes[node.Name]; found {
glog.V(2).Infof("%s was unneeded for %s", node.Name, now.Sub(val).String())
// Check how long the node was underutilized.
if !val.Add(unneededTime).Before(now) {
continue
}
// Check mig size.
instance, err := config.InstanceConfigFromProviderId(node.Spec.ProviderID)
if err != nil {
glog.Errorf("Error while parsing providerid of %s: %v", node.Name, err)
continue
}
migConfig, err := gceManager.GetMigForInstance(instance)
if err != nil {
glog.Errorf("Error while checking mig config for instance %v: %v", instance, err)
continue
}
size, err := gceManager.GetMigSize(migConfig)
if err != nil {
glog.Errorf("Error while checking mig size for instance %v: %v", instance, err)
continue
}
if size <= int64(migConfig.MinSize) {
glog.V(1).Infof("Skipping %s - mig min size reached", node.Name)
continue
}
candidates = append(candidates, node)
}
}
if len(candidates) == 0 {
glog.Infof("No candidates for scale down")
return ScaleDownNoUnneeded, nil
}
nodesToRemove, err := simulator.FindNodesToRemove(candidates, nodes, pods, client, predicateChecker, 1, false)
if err != nil {
return ScaleDownError, fmt.Errorf("Find node to remove failed: %v", err)
}
if len(nodesToRemove) == 0 {
glog.V(1).Infof("No node to remove")
return ScaleDownNoNodeDeleted, nil
}
nodeToRemove := nodesToRemove[0]
glog.Infof("Removing %s", nodeToRemove.Name)
instanceConfig, err := config.InstanceConfigFromProviderId(nodeToRemove.Spec.ProviderID)
if err != nil {
return ScaleDownError, fmt.Errorf("Failed to get instance config for %s: %v", nodeToRemove.Name, err)
}
err = gceManager.DeleteInstances([]*config.InstanceConfig{instanceConfig})
if err != nil {
return ScaleDownError, fmt.Errorf("Failed to delete %v: %v", instanceConfig, err)
}
return ScaleDownNodeDeleted, nil
}