func (a *HorizontalController) updateStatus(hpa extensions.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int, cpuCurrentUtilization *int, cmStatus string, rescale bool) error { hpa.Status = extensions.HorizontalPodAutoscalerStatus{ CurrentReplicas: currentReplicas, DesiredReplicas: desiredReplicas, CurrentCPUUtilizationPercentage: cpuCurrentUtilization, LastScaleTime: hpa.Status.LastScaleTime, } if cmStatus != "" { hpa.Annotations[HpaCustomMetricsStatusAnnotationName] = cmStatus } if rescale { now := unversioned.NewTime(time.Now()) hpa.Status.LastScaleTime = &now } _, err := a.hpaNamespacer.HorizontalPodAutoscalers(hpa.Namespace).UpdateStatus(&hpa) if err != nil { a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedUpdateStatus", err.Error()) return fmt.Errorf("failed to update status for %s: %v", hpa.Name, err) } return nil }
func (a *HorizontalController) reconcileAutoscaler(hpa extensions.HorizontalPodAutoscaler) error { reference := fmt.Sprintf("%s/%s/%s", hpa.Spec.ScaleRef.Kind, hpa.Namespace, hpa.Spec.ScaleRef.Name) scale, err := a.scaleNamespacer.Scales(hpa.Namespace).Get(hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Name) if err != nil { a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedGetScale", err.Error()) return fmt.Errorf("failed to query scale subresource for %s: %v", reference, err) } currentReplicas := scale.Status.Replicas cpuDesiredReplicas := 0 var cpuCurrentUtilization *int = nil cpuTimestamp := time.Time{} cmDesiredReplicas := 0 cmStatus := "" cmTimestamp := time.Time{} if hpa.Spec.CPUUtilization != nil { cpuDesiredReplicas, cpuCurrentUtilization, cpuTimestamp, err = a.computeReplicasForCPUUtilization(hpa, scale) if err != nil { a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedComputeReplicas", err.Error()) return fmt.Errorf("failed to compute desired number of replicas based on CPU utilization for %s: %v", reference, err) } } if cmAnnotation, cmAnnotationFound := hpa.Annotations[HpaCustomMetricsTargetAnnotationName]; cmAnnotationFound { cmDesiredReplicas, cmStatus, cmTimestamp, err = a.computeReplicasForCustomMetrics(hpa, scale, cmAnnotation) if err != nil { a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedComputeCMReplicas", err.Error()) return fmt.Errorf("failed to compute desired number of replicas based on Custom Metrics for %s: %v", reference, err) } } desiredReplicas := 0 timestamp := time.Time{} if cpuDesiredReplicas > desiredReplicas { desiredReplicas = cpuDesiredReplicas timestamp = cpuTimestamp } if cmDesiredReplicas > desiredReplicas { desiredReplicas = cmDesiredReplicas timestamp = cmTimestamp } if hpa.Spec.MinReplicas != nil && desiredReplicas < *hpa.Spec.MinReplicas { desiredReplicas = *hpa.Spec.MinReplicas } // TODO: remove when pod idling is done. if desiredReplicas == 0 { desiredReplicas = 1 } if desiredReplicas > hpa.Spec.MaxReplicas { desiredReplicas = hpa.Spec.MaxReplicas } rescale := false if desiredReplicas != currentReplicas { // Going down only if the usageRatio dropped significantly below the target // and there was no rescaling in the last downscaleForbiddenWindow. if desiredReplicas < currentReplicas && (hpa.Status.LastScaleTime == nil || hpa.Status.LastScaleTime.Add(downscaleForbiddenWindow).Before(timestamp)) { rescale = true } // Going up only if the usage ratio increased significantly above the target // and there was no rescaling in the last upscaleForbiddenWindow. if desiredReplicas > currentReplicas && (hpa.Status.LastScaleTime == nil || hpa.Status.LastScaleTime.Add(upscaleForbiddenWindow).Before(timestamp)) { rescale = true } } if rescale { scale.Spec.Replicas = desiredReplicas _, err = a.scaleNamespacer.Scales(hpa.Namespace).Update(hpa.Spec.ScaleRef.Kind, scale) if err != nil { a.eventRecorder.Eventf(&hpa, api.EventTypeWarning, "FailedRescale", "New size: %d; error: %v", desiredReplicas, err.Error()) return fmt.Errorf("failed to rescale %s: %v", reference, err) } a.eventRecorder.Eventf(&hpa, api.EventTypeNormal, "SuccessfulRescale", "New size: %d", desiredReplicas) glog.Infof("Successfull rescale of %s, old size: %d, new size: %d", hpa.Name, currentReplicas, desiredReplicas) } else { desiredReplicas = currentReplicas } hpa.Status = extensions.HorizontalPodAutoscalerStatus{ CurrentReplicas: currentReplicas, DesiredReplicas: desiredReplicas, CurrentCPUUtilizationPercentage: cpuCurrentUtilization, LastScaleTime: hpa.Status.LastScaleTime, } if cmStatus != "" { hpa.Annotations[HpaCustomMetricsStatusAnnotationName] = cmStatus } if rescale { now := unversioned.NewTime(time.Now()) hpa.Status.LastScaleTime = &now } _, err = a.hpaNamespacer.HorizontalPodAutoscalers(hpa.Namespace).UpdateStatus(&hpa) if err != nil { a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedUpdateStatus", err.Error()) return fmt.Errorf("failed to update status for %s: %v", hpa.Name, err) } return nil }
func (a *HorizontalController) reconcileAutoscaler(hpa extensions.HorizontalPodAutoscaler) error { reference := fmt.Sprintf("%s/%s/%s", hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Namespace, hpa.Spec.ScaleRef.Name) scale, err := a.client.Extensions().Scales(hpa.Spec.ScaleRef.Namespace).Get(hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Name) if err != nil { a.eventRecorder.Event(&hpa, "FailedGetScale", err.Error()) return fmt.Errorf("failed to query scale subresource for %s: %v", reference, err) } currentReplicas := scale.Status.Replicas desiredReplicas, currentUtilization, err := a.computeReplicasForCPUUtilization(hpa, scale) if err != nil { a.eventRecorder.Event(&hpa, "FailedComputeReplicas", err.Error()) return fmt.Errorf("failed to compute desired number of replicas based on CPU utilization for %s: %v", reference, err) } if hpa.Spec.MinReplicas != nil && desiredReplicas < *hpa.Spec.MinReplicas { desiredReplicas = *hpa.Spec.MinReplicas } // TODO: remove when pod idling is done. if desiredReplicas == 0 { desiredReplicas = 1 } if desiredReplicas > hpa.Spec.MaxReplicas { desiredReplicas = hpa.Spec.MaxReplicas } now := time.Now() rescale := false if desiredReplicas != currentReplicas { // Going down only if the usageRatio dropped significantly below the target // and there was no rescaling in the last downscaleForbiddenWindow. if desiredReplicas < currentReplicas && (hpa.Status.LastScaleTime == nil || hpa.Status.LastScaleTime.Add(downscaleForbiddenWindow).Before(now)) { rescale = true } // Going up only if the usage ratio increased significantly above the target // and there was no rescaling in the last upscaleForbiddenWindow. if desiredReplicas > currentReplicas && (hpa.Status.LastScaleTime == nil || hpa.Status.LastScaleTime.Add(upscaleForbiddenWindow).Before(now)) { rescale = true } } if rescale { scale.Spec.Replicas = desiredReplicas _, err = a.client.Extensions().Scales(hpa.Namespace).Update(hpa.Spec.ScaleRef.Kind, scale) if err != nil { a.eventRecorder.Eventf(&hpa, "FailedRescale", "New size: %d; error: %v", desiredReplicas, err.Error()) return fmt.Errorf("failed to rescale %s: %v", reference, err) } a.eventRecorder.Eventf(&hpa, "SuccessfulRescale", "New size: %d", desiredReplicas) glog.Infof("Successfull rescale of %s, old size: %d, new size: %d", hpa.Name, currentReplicas, desiredReplicas) } else { desiredReplicas = currentReplicas } hpa.Status = extensions.HorizontalPodAutoscalerStatus{ CurrentReplicas: currentReplicas, DesiredReplicas: desiredReplicas, CurrentCPUUtilizationPercentage: currentUtilization, } if rescale { now := unversioned.NewTime(now) hpa.Status.LastScaleTime = &now } _, err = a.client.Extensions().HorizontalPodAutoscalers(hpa.Namespace).UpdateStatus(&hpa) if err != nil { a.eventRecorder.Event(&hpa, "FailedUpdateStatus", err.Error()) return fmt.Errorf("failed to update status for %s: %v", hpa.Name, err) } return nil }
func (a *HorizontalController) reconcileAutoscaler(hpa extensions.HorizontalPodAutoscaler) error { reference := fmt.Sprintf("%s/%s/%s", hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Namespace, hpa.Spec.ScaleRef.Name) scale, err := a.client.Experimental().Scales(hpa.Spec.ScaleRef.Namespace).Get(hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Name) if err != nil { a.eventRecorder.Event(&hpa, "FailedGetScale", err.Error()) return fmt.Errorf("failed to query scale subresource for %s: %v", reference, err) } currentReplicas := scale.Status.Replicas currentConsumption, err := a.metricsClient. ResourceConsumption(hpa.Spec.ScaleRef.Namespace). Get(hpa.Spec.Target.Resource, scale.Status.Selector) // TODO: what to do on partial errors (like metrics obtained for 75% of pods). if err != nil { a.eventRecorder.Event(&hpa, "FailedGetMetrics", err.Error()) return fmt.Errorf("failed to get metrics for %s: %v", reference, err) } usageRatio := float64(currentConsumption.Quantity.MilliValue()) / float64(hpa.Spec.Target.Quantity.MilliValue()) desiredReplicas := int(math.Ceil(usageRatio * float64(currentReplicas))) if desiredReplicas < hpa.Spec.MinReplicas { desiredReplicas = hpa.Spec.MinReplicas } // TODO: remove when pod ideling is done. if desiredReplicas == 0 { desiredReplicas = 1 } if desiredReplicas > hpa.Spec.MaxReplicas { desiredReplicas = hpa.Spec.MaxReplicas } now := time.Now() rescale := false if desiredReplicas != currentReplicas { // Going down only if the usageRatio dropped significantly below the target // and there was no rescaling in the last downscaleForbiddenWindow. if desiredReplicas < currentReplicas && usageRatio < (1-tolerance) && (hpa.Status.LastScaleTimestamp == nil || hpa.Status.LastScaleTimestamp.Add(downscaleForbiddenWindow).Before(now)) { rescale = true } // Going up only if the usage ratio increased significantly above the target // and there was no rescaling in the last upscaleForbiddenWindow. if desiredReplicas > currentReplicas && usageRatio > (1+tolerance) && (hpa.Status.LastScaleTimestamp == nil || hpa.Status.LastScaleTimestamp.Add(upscaleForbiddenWindow).Before(now)) { rescale = true } } if rescale { scale.Spec.Replicas = desiredReplicas _, err = a.client.Experimental().Scales(hpa.Namespace).Update(hpa.Spec.ScaleRef.Kind, scale) if err != nil { a.eventRecorder.Eventf(&hpa, "FailedRescale", "New size: %d; error: %v", desiredReplicas, err.Error()) return fmt.Errorf("failed to rescale %s: %v", reference, err) } a.eventRecorder.Eventf(&hpa, "SuccessfulRescale", "New size: %d", desiredReplicas) glog.Infof("Successfull rescale of %s, old size: %d, new size: %d, usage ratio: %f", hpa.Name, currentReplicas, desiredReplicas, usageRatio) } else { desiredReplicas = currentReplicas } hpa.Status = extensions.HorizontalPodAutoscalerStatus{ CurrentReplicas: currentReplicas, DesiredReplicas: desiredReplicas, CurrentConsumption: currentConsumption, } if rescale { now := unversioned.NewTime(now) hpa.Status.LastScaleTimestamp = &now } _, err = a.client.Experimental().HorizontalPodAutoscalers(hpa.Namespace).Update(&hpa) if err != nil { a.eventRecorder.Event(&hpa, "FailedUpdateStatus", err.Error()) return fmt.Errorf("failed to update status for %s: %v", hpa.Name, err) } return nil }