예제 #1
0
// Computes the desired number of replicas based on the CustomMetrics passed in cmAnnotation as json-serialized
// extensions.CustomMetricsTargetList.
// Returns number of replicas, status string (also json-serialized extensions.CustomMetricsCurrentStatusList),
// last timestamp of the metrics involved in computations or error, if occurred.
func (a *HorizontalController) computeReplicasForCustomMetrics(hpa extensions.HorizontalPodAutoscaler, scale *extensions.Scale,
	cmAnnotation string) (int, string, time.Time, error) {

	currentReplicas := scale.Status.Replicas
	replicas := 0
	timestamp := time.Time{}

	if cmAnnotation == "" {
		return 0, "", time.Time{}, nil
	}

	var targetList extensions.CustomMetricTargetList
	if err := json.Unmarshal([]byte(cmAnnotation), &targetList); err != nil {
		return 0, "", time.Time{}, fmt.Errorf("failed to parse custom metrics annotation: %v", err)
	}
	if len(targetList.Items) == 0 {
		return 0, "", time.Time{}, fmt.Errorf("no custom metrics in annotation")
	}

	statusList := extensions.CustomMetricCurrentStatusList{
		Items: make([]extensions.CustomMetricCurrentStatus, 0),
	}

	for _, customMetricTarget := range targetList.Items {
		value, currentTimestamp, err := a.metricsClient.GetCustomMetric(customMetricTarget.Name, hpa.Namespace, scale.Status.Selector)
		// TODO: what to do on partial errors (like metrics obtained for 75% of pods).
		if err != nil {
			a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedGetCustomMetrics", err.Error())
			return 0, "", time.Time{}, fmt.Errorf("failed to get custom metric value: %v", err)
		}
		floatTarget := float64(customMetricTarget.TargetValue.MilliValue()) / 1000.0
		usageRatio := *value / floatTarget

		replicaCountProposal := 0
		if math.Abs(1.0-usageRatio) > tolerance {
			replicaCountProposal = int(math.Ceil(usageRatio * float64(currentReplicas)))
		} else {
			replicaCountProposal = currentReplicas
		}
		if replicaCountProposal > replicas {
			timestamp = currentTimestamp
			replicas = replicaCountProposal
		}
		quantity, err := resource.ParseQuantity(fmt.Sprintf("%.3f", *value))
		if err != nil {
			return 0, "", time.Time{}, fmt.Errorf("failed to set custom metric value: %v", err)
		}
		statusList.Items = append(statusList.Items, extensions.CustomMetricCurrentStatus{
			Name:         customMetricTarget.Name,
			CurrentValue: *quantity,
		})
	}
	byteStatusList, err := json.Marshal(statusList)
	if err != nil {
		return 0, "", time.Time{}, fmt.Errorf("failed to serialize custom metric status: %v", err)
	}

	return replicas, string(byteStatusList), timestamp, nil

}
예제 #2
0
// computeReplicasForCustomMetrics computes the desired number of replicas based on the CustomMetrics passed in cmAnnotation
// as json-serialized extensions.CustomMetricsTargetList.
// Returns number of replicas, metric which required highest number of replicas,
// status string (also json-serialized extensions.CustomMetricsCurrentStatusList),
// last timestamp of the metrics involved in computations or error, if occurred.
func (a *HorizontalController) computeReplicasForCustomMetrics(hpa *autoscaling.HorizontalPodAutoscaler, scale *extensions.Scale,
	cmAnnotation string) (replicas int32, metric string, status string, timestamp time.Time, err error) {

	if cmAnnotation == "" {
		return
	}

	currentReplicas := scale.Status.Replicas

	var targetList extensions.CustomMetricTargetList
	if err := json.Unmarshal([]byte(cmAnnotation), &targetList); err != nil {
		return 0, "", "", time.Time{}, fmt.Errorf("failed to parse custom metrics annotation: %v", err)
	}
	if len(targetList.Items) == 0 {
		return 0, "", "", time.Time{}, fmt.Errorf("no custom metrics in annotation")
	}

	statusList := extensions.CustomMetricCurrentStatusList{
		Items: make([]extensions.CustomMetricCurrentStatus, 0),
	}

	for _, customMetricTarget := range targetList.Items {
		if scale.Status.Selector == nil {
			errMsg := "selector is required"
			a.eventRecorder.Event(hpa, api.EventTypeWarning, "SelectorRequired", errMsg)
			return 0, "", "", time.Time{}, fmt.Errorf("selector is required")
		}

		selector, err := unversioned.LabelSelectorAsSelector(scale.Status.Selector)
		if err != nil {
			errMsg := fmt.Sprintf("couldn't convert selector string to a corresponding selector object: %v", err)
			a.eventRecorder.Event(hpa, api.EventTypeWarning, "InvalidSelector", errMsg)
			return 0, "", "", time.Time{}, fmt.Errorf("couldn't convert selector string to a corresponding selector object: %v", err)
		}
		floatTarget := float64(customMetricTarget.TargetValue.MilliValue()) / 1000.0
		replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetMetricReplicas(currentReplicas, floatTarget, fmt.Sprintf("custom/%s", customMetricTarget.Name), hpa.Namespace, selector)
		if err != nil {
			lastScaleTime := getLastScaleTime(hpa)
			if time.Now().After(lastScaleTime.Add(upscaleForbiddenWindow)) {
				a.eventRecorder.Event(hpa, api.EventTypeWarning, "FailedGetCustomMetrics", err.Error())
			} else {
				a.eventRecorder.Event(hpa, api.EventTypeNormal, "CustomMetricsNotAvailableYet", err.Error())
			}

			return 0, "", "", time.Time{}, fmt.Errorf("failed to get custom metric value: %v", err)
		}

		if replicaCountProposal > replicas {
			timestamp = timestampProposal
			replicas = replicaCountProposal
			metric = fmt.Sprintf("Custom metric %s", customMetricTarget.Name)
		}
		quantity, err := resource.ParseQuantity(fmt.Sprintf("%.3f", utilizationProposal))
		if err != nil {
			return 0, "", "", time.Time{}, fmt.Errorf("failed to set custom metric value: %v", err)
		}
		statusList.Items = append(statusList.Items, extensions.CustomMetricCurrentStatus{
			Name:         customMetricTarget.Name,
			CurrentValue: quantity,
		})
	}
	byteStatusList, err := json.Marshal(statusList)
	if err != nil {
		return 0, "", "", time.Time{}, fmt.Errorf("failed to serialize custom metric status: %v", err)
	}

	return replicas, metric, string(byteStatusList), timestamp, nil
}
예제 #3
0
// Computes the desired number of replicas based on the CustomMetrics passed in cmAnnotation as json-serialized
// extensions.CustomMetricsTargetList.
// Returns number of replicas, metric which required highest number of replicas,
// status string (also json-serialized extensions.CustomMetricsCurrentStatusList),
// last timestamp of the metrics involved in computations or error, if occurred.
func (a *HorizontalController) computeReplicasForCustomMetrics(hpa *autoscaling.HorizontalPodAutoscaler, scale *extensions.Scale,
	cmAnnotation string) (replicas int32, metric string, status string, timestamp time.Time, err error) {

	currentReplicas := scale.Status.Replicas
	replicas = 0
	metric = ""
	status = ""
	timestamp = time.Time{}
	err = nil

	if cmAnnotation == "" {
		return
	}

	var targetList extensions.CustomMetricTargetList
	if err := json.Unmarshal([]byte(cmAnnotation), &targetList); err != nil {
		return 0, "", "", time.Time{}, fmt.Errorf("failed to parse custom metrics annotation: %v", err)
	}
	if len(targetList.Items) == 0 {
		return 0, "", "", time.Time{}, fmt.Errorf("no custom metrics in annotation")
	}

	statusList := extensions.CustomMetricCurrentStatusList{
		Items: make([]extensions.CustomMetricCurrentStatus, 0),
	}

	for _, customMetricTarget := range targetList.Items {
		if scale.Status.Selector == nil {
			errMsg := "selector is required"
			a.eventRecorder.Event(hpa, api.EventTypeWarning, "SelectorRequired", errMsg)
			return 0, "", "", time.Time{}, fmt.Errorf("selector is required")
		}

		selector, err := unversioned.LabelSelectorAsSelector(scale.Status.Selector)
		if err != nil {
			errMsg := fmt.Sprintf("couldn't convert selector string to a corresponding selector object: %v", err)
			a.eventRecorder.Event(hpa, api.EventTypeWarning, "InvalidSelector", errMsg)
			return 0, "", "", time.Time{}, fmt.Errorf("couldn't convert selector string to a corresponding selector object: %v", err)
		}
		value, currentTimestamp, err := a.metricsClient.GetCustomMetric(customMetricTarget.Name, hpa.Namespace, selector)
		// TODO: what to do on partial errors (like metrics obtained for 75% of pods).
		if err != nil {
			a.eventRecorder.Event(hpa, api.EventTypeWarning, "FailedGetCustomMetrics", err.Error())
			return 0, "", "", time.Time{}, fmt.Errorf("failed to get custom metric value: %v", err)
		}
		floatTarget := float64(customMetricTarget.TargetValue.MilliValue()) / 1000.0
		usageRatio := *value / floatTarget

		replicaCountProposal := int32(0)
		if math.Abs(1.0-usageRatio) > tolerance {
			replicaCountProposal = int32(math.Ceil(usageRatio * float64(currentReplicas)))
		} else {
			replicaCountProposal = currentReplicas
		}
		if replicaCountProposal > replicas {
			timestamp = currentTimestamp
			replicas = replicaCountProposal
			metric = fmt.Sprintf("Custom metric %s", customMetricTarget.Name)
		}
		quantity, err := resource.ParseQuantity(fmt.Sprintf("%.3f", *value))
		if err != nil {
			return 0, "", "", time.Time{}, fmt.Errorf("failed to set custom metric value: %v", err)
		}
		statusList.Items = append(statusList.Items, extensions.CustomMetricCurrentStatus{
			Name:         customMetricTarget.Name,
			CurrentValue: *quantity,
		})
	}
	byteStatusList, err := json.Marshal(statusList)
	if err != nil {
		return 0, "", "", time.Time{}, fmt.Errorf("failed to serialize custom metric status: %v", err)
	}

	return replicas, metric, string(byteStatusList), timestamp, nil
}