// Computes the desired number of replicas based on the CustomMetrics passed in cmAnnotation as json-serialized
// extensions.CustomMetricsTargetList.
// Returns number of replicas, status string (also json-serialized extensions.CustomMetricsCurrentStatusList),
// last timestamp of the metrics involved in computations or error, if occurred.
func (a *HorizontalController) computeReplicasForCustomMetrics(hpa extensions.HorizontalPodAutoscaler, scale *extensions.Scale,
cmAnnotation string) (int, string, time.Time, error) {
currentReplicas := scale.Status.Replicas
replicas := 0
timestamp := time.Time{}
if cmAnnotation == "" {
return 0, "", time.Time{}, nil
}
var targetList extensions.CustomMetricTargetList
if err := json.Unmarshal([]byte(cmAnnotation), &targetList); err != nil {
return 0, "", time.Time{}, fmt.Errorf("failed to parse custom metrics annotation: %v", err)
}
if len(targetList.Items) == 0 {
return 0, "", time.Time{}, fmt.Errorf("no custom metrics in annotation")
}
statusList := extensions.CustomMetricCurrentStatusList{
Items: make([]extensions.CustomMetricCurrentStatus, 0),
}
for _, customMetricTarget := range targetList.Items {
value, currentTimestamp, err := a.metricsClient.GetCustomMetric(customMetricTarget.Name, hpa.Namespace, scale.Status.Selector)
// TODO: what to do on partial errors (like metrics obtained for 75% of pods).
if err != nil {
a.eventRecorder.Event(&hpa, api.EventTypeWarning, "FailedGetCustomMetrics", err.Error())
return 0, "", time.Time{}, fmt.Errorf("failed to get custom metric value: %v", err)
}
floatTarget := float64(customMetricTarget.TargetValue.MilliValue()) / 1000.0
usageRatio := *value / floatTarget
replicaCountProposal := 0
if math.Abs(1.0-usageRatio) > tolerance {
replicaCountProposal = int(math.Ceil(usageRatio * float64(currentReplicas)))
} else {
replicaCountProposal = currentReplicas
}
if replicaCountProposal > replicas {
timestamp = currentTimestamp
replicas = replicaCountProposal
}
quantity, err := resource.ParseQuantity(fmt.Sprintf("%.3f", *value))
if err != nil {
return 0, "", time.Time{}, fmt.Errorf("failed to set custom metric value: %v", err)
}
statusList.Items = append(statusList.Items, extensions.CustomMetricCurrentStatus{
Name: customMetricTarget.Name,
CurrentValue: *quantity,
})
}
byteStatusList, err := json.Marshal(statusList)
if err != nil {
return 0, "", time.Time{}, fmt.Errorf("failed to serialize custom metric status: %v", err)
}
return replicas, string(byteStatusList), timestamp, nil
}
// computeReplicasForCustomMetrics computes the desired number of replicas based on the CustomMetrics passed in cmAnnotation
// as json-serialized extensions.CustomMetricsTargetList.
// Returns number of replicas, metric which required highest number of replicas,
// status string (also json-serialized extensions.CustomMetricsCurrentStatusList),
// last timestamp of the metrics involved in computations or error, if occurred.
func (a *HorizontalController) computeReplicasForCustomMetrics(hpa *autoscaling.HorizontalPodAutoscaler, scale *extensions.Scale,
cmAnnotation string) (replicas int32, metric string, status string, timestamp time.Time, err error) {
if cmAnnotation == "" {
return
}
currentReplicas := scale.Status.Replicas
var targetList extensions.CustomMetricTargetList
if err := json.Unmarshal([]byte(cmAnnotation), &targetList); err != nil {
return 0, "", "", time.Time{}, fmt.Errorf("failed to parse custom metrics annotation: %v", err)
}
if len(targetList.Items) == 0 {
return 0, "", "", time.Time{}, fmt.Errorf("no custom metrics in annotation")
}
statusList := extensions.CustomMetricCurrentStatusList{
Items: make([]extensions.CustomMetricCurrentStatus, 0),
}
for _, customMetricTarget := range targetList.Items {
if scale.Status.Selector == nil {
errMsg := "selector is required"
a.eventRecorder.Event(hpa, api.EventTypeWarning, "SelectorRequired", errMsg)
return 0, "", "", time.Time{}, fmt.Errorf("selector is required")
}
selector, err := unversioned.LabelSelectorAsSelector(scale.Status.Selector)
if err != nil {
errMsg := fmt.Sprintf("couldn't convert selector string to a corresponding selector object: %v", err)
a.eventRecorder.Event(hpa, api.EventTypeWarning, "InvalidSelector", errMsg)
return 0, "", "", time.Time{}, fmt.Errorf("couldn't convert selector string to a corresponding selector object: %v", err)
}
floatTarget := float64(customMetricTarget.TargetValue.MilliValue()) / 1000.0
replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetMetricReplicas(currentReplicas, floatTarget, fmt.Sprintf("custom/%s", customMetricTarget.Name), hpa.Namespace, selector)
if err != nil {
lastScaleTime := getLastScaleTime(hpa)
if time.Now().After(lastScaleTime.Add(upscaleForbiddenWindow)) {
a.eventRecorder.Event(hpa, api.EventTypeWarning, "FailedGetCustomMetrics", err.Error())
} else {
a.eventRecorder.Event(hpa, api.EventTypeNormal, "CustomMetricsNotAvailableYet", err.Error())
}
return 0, "", "", time.Time{}, fmt.Errorf("failed to get custom metric value: %v", err)
}
if replicaCountProposal > replicas {
timestamp = timestampProposal
replicas = replicaCountProposal
metric = fmt.Sprintf("Custom metric %s", customMetricTarget.Name)
}
quantity, err := resource.ParseQuantity(fmt.Sprintf("%.3f", utilizationProposal))
if err != nil {
return 0, "", "", time.Time{}, fmt.Errorf("failed to set custom metric value: %v", err)
}
statusList.Items = append(statusList.Items, extensions.CustomMetricCurrentStatus{
Name: customMetricTarget.Name,
CurrentValue: quantity,
})
}
byteStatusList, err := json.Marshal(statusList)
if err != nil {
return 0, "", "", time.Time{}, fmt.Errorf("failed to serialize custom metric status: %v", err)
}
return replicas, metric, string(byteStatusList), timestamp, nil
}
// Computes the desired number of replicas based on the CustomMetrics passed in cmAnnotation as json-serialized
// extensions.CustomMetricsTargetList.
// Returns number of replicas, metric which required highest number of replicas,
// status string (also json-serialized extensions.CustomMetricsCurrentStatusList),
// last timestamp of the metrics involved in computations or error, if occurred.
func (a *HorizontalController) computeReplicasForCustomMetrics(hpa *autoscaling.HorizontalPodAutoscaler, scale *extensions.Scale,
cmAnnotation string) (replicas int32, metric string, status string, timestamp time.Time, err error) {
currentReplicas := scale.Status.Replicas
replicas = 0
metric = ""
status = ""
timestamp = time.Time{}
err = nil
if cmAnnotation == "" {
return
}
var targetList extensions.CustomMetricTargetList
if err := json.Unmarshal([]byte(cmAnnotation), &targetList); err != nil {
return 0, "", "", time.Time{}, fmt.Errorf("failed to parse custom metrics annotation: %v", err)
}
if len(targetList.Items) == 0 {
return 0, "", "", time.Time{}, fmt.Errorf("no custom metrics in annotation")
}
statusList := extensions.CustomMetricCurrentStatusList{
Items: make([]extensions.CustomMetricCurrentStatus, 0),
}
for _, customMetricTarget := range targetList.Items {
if scale.Status.Selector == nil {
errMsg := "selector is required"
a.eventRecorder.Event(hpa, api.EventTypeWarning, "SelectorRequired", errMsg)
return 0, "", "", time.Time{}, fmt.Errorf("selector is required")
}
selector, err := unversioned.LabelSelectorAsSelector(scale.Status.Selector)
if err != nil {
errMsg := fmt.Sprintf("couldn't convert selector string to a corresponding selector object: %v", err)
a.eventRecorder.Event(hpa, api.EventTypeWarning, "InvalidSelector", errMsg)
return 0, "", "", time.Time{}, fmt.Errorf("couldn't convert selector string to a corresponding selector object: %v", err)
}
value, currentTimestamp, err := a.metricsClient.GetCustomMetric(customMetricTarget.Name, hpa.Namespace, selector)
// TODO: what to do on partial errors (like metrics obtained for 75% of pods).
if err != nil {
a.eventRecorder.Event(hpa, api.EventTypeWarning, "FailedGetCustomMetrics", err.Error())
return 0, "", "", time.Time{}, fmt.Errorf("failed to get custom metric value: %v", err)
}
floatTarget := float64(customMetricTarget.TargetValue.MilliValue()) / 1000.0
usageRatio := *value / floatTarget
replicaCountProposal := int32(0)
if math.Abs(1.0-usageRatio) > tolerance {
replicaCountProposal = int32(math.Ceil(usageRatio * float64(currentReplicas)))
} else {
replicaCountProposal = currentReplicas
}
if replicaCountProposal > replicas {
timestamp = currentTimestamp
replicas = replicaCountProposal
metric = fmt.Sprintf("Custom metric %s", customMetricTarget.Name)
}
quantity, err := resource.ParseQuantity(fmt.Sprintf("%.3f", *value))
if err != nil {
return 0, "", "", time.Time{}, fmt.Errorf("failed to set custom metric value: %v", err)
}
statusList.Items = append(statusList.Items, extensions.CustomMetricCurrentStatus{
Name: customMetricTarget.Name,
CurrentValue: *quantity,
})
}
byteStatusList, err := json.Marshal(statusList)
if err != nil {
return 0, "", "", time.Time{}, fmt.Errorf("failed to serialize custom metric status: %v", err)
}
return replicas, metric, string(byteStatusList), timestamp, nil
}