Example #1
0
func (a *HorizontalController) updateStatus(hpa extensions.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int, cpuCurrentUtilization *int, cmStatus string, rescale bool) error {
	hpa.Status = extensions.HorizontalPodAutoscalerStatus{
		CurrentReplicas:                 currentReplicas,
		DesiredReplicas:                 desiredReplicas,
		CurrentCPUUtilizationPercentage: cpuCurrentUtilization,
		LastScaleTime:                   hpa.Status.LastScaleTime,
	}
	if cmStatus != "" {
		hpa.Annotations[HpaCustomMetricsStatusAnnotationName] = cmStatus
	}

	if rescale {
		now := unversioned.NewTime(time.Now())
		hpa.Status.LastScaleTime = &now
	}

	_, err := a.hpaNamespacer.HorizontalPodAutoscalers(hpa.Namespace).UpdateStatus(&hpa)
	if err != nil {
		a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedUpdateStatus", err.Error())
		return fmt.Errorf("failed to update status for %s: %v", hpa.Name, err)
	}
	return nil
}
Example #2
0
func (a *HorizontalController) reconcileAutoscaler(hpa extensions.HorizontalPodAutoscaler) error {
	reference := fmt.Sprintf("%s/%s/%s", hpa.Spec.ScaleRef.Kind, hpa.Namespace, hpa.Spec.ScaleRef.Name)

	scale, err := a.scaleNamespacer.Scales(hpa.Namespace).Get(hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Name)
	if err != nil {
		a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedGetScale", err.Error())
		return fmt.Errorf("failed to query scale subresource for %s: %v", reference, err)
	}
	currentReplicas := scale.Status.Replicas

	cpuDesiredReplicas := 0
	var cpuCurrentUtilization *int = nil
	cpuTimestamp := time.Time{}

	cmDesiredReplicas := 0
	cmStatus := ""
	cmTimestamp := time.Time{}

	if hpa.Spec.CPUUtilization != nil {
		cpuDesiredReplicas, cpuCurrentUtilization, cpuTimestamp, err = a.computeReplicasForCPUUtilization(hpa, scale)
		if err != nil {
			a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedComputeReplicas", err.Error())
			return fmt.Errorf("failed to compute desired number of replicas based on CPU utilization for %s: %v", reference, err)
		}
	}

	if cmAnnotation, cmAnnotationFound := hpa.Annotations[HpaCustomMetricsTargetAnnotationName]; cmAnnotationFound {
		cmDesiredReplicas, cmStatus, cmTimestamp, err = a.computeReplicasForCustomMetrics(hpa, scale, cmAnnotation)
		if err != nil {
			a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedComputeCMReplicas", err.Error())
			return fmt.Errorf("failed to compute desired number of replicas based on Custom Metrics for %s: %v", reference, err)
		}
	}

	desiredReplicas := 0
	timestamp := time.Time{}

	if cpuDesiredReplicas > desiredReplicas {
		desiredReplicas = cpuDesiredReplicas
		timestamp = cpuTimestamp
	}
	if cmDesiredReplicas > desiredReplicas {
		desiredReplicas = cmDesiredReplicas
		timestamp = cmTimestamp
	}

	if hpa.Spec.MinReplicas != nil && desiredReplicas < *hpa.Spec.MinReplicas {
		desiredReplicas = *hpa.Spec.MinReplicas
	}

	// TODO: remove when pod idling is done.
	if desiredReplicas == 0 {
		desiredReplicas = 1
	}

	if desiredReplicas > hpa.Spec.MaxReplicas {
		desiredReplicas = hpa.Spec.MaxReplicas
	}
	rescale := false

	if desiredReplicas != currentReplicas {
		// Going down only if the usageRatio dropped significantly below the target
		// and there was no rescaling in the last downscaleForbiddenWindow.
		if desiredReplicas < currentReplicas &&
			(hpa.Status.LastScaleTime == nil ||
				hpa.Status.LastScaleTime.Add(downscaleForbiddenWindow).Before(timestamp)) {
			rescale = true
		}

		// Going up only if the usage ratio increased significantly above the target
		// and there was no rescaling in the last upscaleForbiddenWindow.
		if desiredReplicas > currentReplicas &&
			(hpa.Status.LastScaleTime == nil ||
				hpa.Status.LastScaleTime.Add(upscaleForbiddenWindow).Before(timestamp)) {
			rescale = true
		}
	}

	if rescale {
		scale.Spec.Replicas = desiredReplicas
		_, err = a.scaleNamespacer.Scales(hpa.Namespace).Update(hpa.Spec.ScaleRef.Kind, scale)
		if err != nil {
			a.eventRecorder.Eventf(&hpa, api.EventTypeWarning, "FailedRescale", "New size: %d; error: %v", desiredReplicas, err.Error())
			return fmt.Errorf("failed to rescale %s: %v", reference, err)
		}
		a.eventRecorder.Eventf(&hpa, api.EventTypeNormal, "SuccessfulRescale", "New size: %d", desiredReplicas)
		glog.Infof("Successfull rescale of %s, old size: %d, new size: %d",
			hpa.Name, currentReplicas, desiredReplicas)
	} else {
		desiredReplicas = currentReplicas
	}

	hpa.Status = extensions.HorizontalPodAutoscalerStatus{
		CurrentReplicas:                 currentReplicas,
		DesiredReplicas:                 desiredReplicas,
		CurrentCPUUtilizationPercentage: cpuCurrentUtilization,
		LastScaleTime:                   hpa.Status.LastScaleTime,
	}
	if cmStatus != "" {
		hpa.Annotations[HpaCustomMetricsStatusAnnotationName] = cmStatus
	}

	if rescale {
		now := unversioned.NewTime(time.Now())
		hpa.Status.LastScaleTime = &now
	}

	_, err = a.hpaNamespacer.HorizontalPodAutoscalers(hpa.Namespace).UpdateStatus(&hpa)
	if err != nil {
		a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedUpdateStatus", err.Error())
		return fmt.Errorf("failed to update status for %s: %v", hpa.Name, err)
	}
	return nil
}
Example #3
0
func (a *HorizontalController) reconcileAutoscaler(hpa extensions.HorizontalPodAutoscaler) error {
	reference := fmt.Sprintf("%s/%s/%s", hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Namespace, hpa.Spec.ScaleRef.Name)

	scale, err := a.client.Extensions().Scales(hpa.Spec.ScaleRef.Namespace).Get(hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Name)
	if err != nil {
		a.eventRecorder.Event(&hpa, "FailedGetScale", err.Error())
		return fmt.Errorf("failed to query scale subresource for %s: %v", reference, err)
	}
	currentReplicas := scale.Status.Replicas

	desiredReplicas, currentUtilization, err := a.computeReplicasForCPUUtilization(hpa, scale)
	if err != nil {
		a.eventRecorder.Event(&hpa, "FailedComputeReplicas", err.Error())
		return fmt.Errorf("failed to compute desired number of replicas based on CPU utilization for %s: %v", reference, err)
	}

	if hpa.Spec.MinReplicas != nil && desiredReplicas < *hpa.Spec.MinReplicas {
		desiredReplicas = *hpa.Spec.MinReplicas
	}

	// TODO: remove when pod idling is done.
	if desiredReplicas == 0 {
		desiredReplicas = 1
	}

	if desiredReplicas > hpa.Spec.MaxReplicas {
		desiredReplicas = hpa.Spec.MaxReplicas
	}
	now := time.Now()
	rescale := false

	if desiredReplicas != currentReplicas {
		// Going down only if the usageRatio dropped significantly below the target
		// and there was no rescaling in the last downscaleForbiddenWindow.
		if desiredReplicas < currentReplicas &&
			(hpa.Status.LastScaleTime == nil ||
				hpa.Status.LastScaleTime.Add(downscaleForbiddenWindow).Before(now)) {
			rescale = true
		}

		// Going up only if the usage ratio increased significantly above the target
		// and there was no rescaling in the last upscaleForbiddenWindow.
		if desiredReplicas > currentReplicas &&
			(hpa.Status.LastScaleTime == nil ||
				hpa.Status.LastScaleTime.Add(upscaleForbiddenWindow).Before(now)) {
			rescale = true
		}
	}

	if rescale {
		scale.Spec.Replicas = desiredReplicas
		_, err = a.client.Extensions().Scales(hpa.Namespace).Update(hpa.Spec.ScaleRef.Kind, scale)
		if err != nil {
			a.eventRecorder.Eventf(&hpa, "FailedRescale", "New size: %d; error: %v", desiredReplicas, err.Error())
			return fmt.Errorf("failed to rescale %s: %v", reference, err)
		}
		a.eventRecorder.Eventf(&hpa, "SuccessfulRescale", "New size: %d", desiredReplicas)
		glog.Infof("Successfull rescale of %s, old size: %d, new size: %d",
			hpa.Name, currentReplicas, desiredReplicas)
	} else {
		desiredReplicas = currentReplicas
	}

	hpa.Status = extensions.HorizontalPodAutoscalerStatus{
		CurrentReplicas:                 currentReplicas,
		DesiredReplicas:                 desiredReplicas,
		CurrentCPUUtilizationPercentage: currentUtilization,
	}
	if rescale {
		now := unversioned.NewTime(now)
		hpa.Status.LastScaleTime = &now
	}

	_, err = a.client.Extensions().HorizontalPodAutoscalers(hpa.Namespace).UpdateStatus(&hpa)
	if err != nil {
		a.eventRecorder.Event(&hpa, "FailedUpdateStatus", err.Error())
		return fmt.Errorf("failed to update status for %s: %v", hpa.Name, err)
	}
	return nil
}
Example #4
0
func (a *HorizontalController) reconcileAutoscaler(hpa extensions.HorizontalPodAutoscaler) error {
	reference := fmt.Sprintf("%s/%s/%s", hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Namespace, hpa.Spec.ScaleRef.Name)

	scale, err := a.client.Experimental().Scales(hpa.Spec.ScaleRef.Namespace).Get(hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Name)
	if err != nil {
		a.eventRecorder.Event(&hpa, "FailedGetScale", err.Error())
		return fmt.Errorf("failed to query scale subresource for %s: %v", reference, err)
	}
	currentReplicas := scale.Status.Replicas
	currentConsumption, err := a.metricsClient.
		ResourceConsumption(hpa.Spec.ScaleRef.Namespace).
		Get(hpa.Spec.Target.Resource, scale.Status.Selector)

	// TODO: what to do on partial errors (like metrics obtained for 75% of pods).
	if err != nil {
		a.eventRecorder.Event(&hpa, "FailedGetMetrics", err.Error())
		return fmt.Errorf("failed to get metrics for %s: %v", reference, err)
	}

	usageRatio := float64(currentConsumption.Quantity.MilliValue()) / float64(hpa.Spec.Target.Quantity.MilliValue())
	desiredReplicas := int(math.Ceil(usageRatio * float64(currentReplicas)))

	if desiredReplicas < hpa.Spec.MinReplicas {
		desiredReplicas = hpa.Spec.MinReplicas
	}

	// TODO: remove when pod ideling is done.
	if desiredReplicas == 0 {
		desiredReplicas = 1
	}

	if desiredReplicas > hpa.Spec.MaxReplicas {
		desiredReplicas = hpa.Spec.MaxReplicas
	}
	now := time.Now()
	rescale := false

	if desiredReplicas != currentReplicas {
		// Going down only if the usageRatio dropped significantly below the target
		// and there was no rescaling in the last downscaleForbiddenWindow.
		if desiredReplicas < currentReplicas && usageRatio < (1-tolerance) &&
			(hpa.Status.LastScaleTimestamp == nil ||
				hpa.Status.LastScaleTimestamp.Add(downscaleForbiddenWindow).Before(now)) {
			rescale = true
		}

		// Going up only if the usage ratio increased significantly above the target
		// and there was no rescaling in the last upscaleForbiddenWindow.
		if desiredReplicas > currentReplicas && usageRatio > (1+tolerance) &&
			(hpa.Status.LastScaleTimestamp == nil ||
				hpa.Status.LastScaleTimestamp.Add(upscaleForbiddenWindow).Before(now)) {
			rescale = true
		}
	}

	if rescale {
		scale.Spec.Replicas = desiredReplicas
		_, err = a.client.Experimental().Scales(hpa.Namespace).Update(hpa.Spec.ScaleRef.Kind, scale)
		if err != nil {
			a.eventRecorder.Eventf(&hpa, "FailedRescale", "New size: %d; error: %v", desiredReplicas, err.Error())
			return fmt.Errorf("failed to rescale %s: %v", reference, err)
		}
		a.eventRecorder.Eventf(&hpa, "SuccessfulRescale", "New size: %d", desiredReplicas)
		glog.Infof("Successfull rescale of %s, old size: %d, new size: %d, usage ratio: %f",
			hpa.Name, currentReplicas, desiredReplicas, usageRatio)
	} else {
		desiredReplicas = currentReplicas
	}

	hpa.Status = extensions.HorizontalPodAutoscalerStatus{
		CurrentReplicas:    currentReplicas,
		DesiredReplicas:    desiredReplicas,
		CurrentConsumption: currentConsumption,
	}
	if rescale {
		now := unversioned.NewTime(now)
		hpa.Status.LastScaleTimestamp = &now
	}

	_, err = a.client.Experimental().HorizontalPodAutoscalers(hpa.Namespace).Update(&hpa)
	if err != nil {
		a.eventRecorder.Event(&hpa, "FailedUpdateStatus", err.Error())
		return fmt.Errorf("failed to update status for %s: %v", hpa.Name, err)
	}
	return nil
}