diff --git a/pkg/controller/ehpa/hpa.go b/pkg/controller/ehpa/hpa.go index 07d090c61..99abfe888 100644 --- a/pkg/controller/ehpa/hpa.go +++ b/pkg/controller/ehpa/hpa.go @@ -208,10 +208,30 @@ func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autosc var metricIdentifier string var averageValue *resource.Quantity switch metric.Type { - case autoscalingv2.ResourceMetricSourceType: - metricIdentifier = utils.GetMetricIdentifier(metric, metric.Resource.Name.String()) + case autoscalingv2.ResourceMetricSourceType, autoscalingv2.ContainerResourceMetricSourceType: + var averageUtilization *int32 + var containerName string + if metric.Resource != nil { + if metric.Resource.Target.AverageUtilization != nil { + averageUtilization = metric.Resource.Target.AverageUtilization + } + if metric.Resource.Target.AverageValue != nil { + averageValue = metric.Resource.Target.AverageValue + } + } + if metric.ContainerResource != nil { + containerName = metric.ContainerResource.Container + if metric.ContainerResource.Target.AverageUtilization != nil { + averageUtilization = metric.ContainerResource.Target.AverageUtilization + } + if metric.ContainerResource.Target.AverageValue != nil { + averageValue = metric.ContainerResource.Target.AverageValue + } + } + // When use AverageUtilization in EffectiveHorizontalPodAutoscaler's metricSpec, convert to AverageValue - if metric.Resource.Target.AverageUtilization != nil { + if averageUtilization != nil { + metricName := utils.GetMetricName(metric) scale, _, err := utils.GetScale(ctx, c.RestMapper, c.ScaleClient, ehpa.Namespace, ehpa.Spec.ScaleTargetRef) if err != nil { return nil, err @@ -231,24 +251,21 @@ func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autosc return nil, fmt.Errorf("failed to get available pods. ") } - requests, err := utils.CalculatePodRequests(availablePods, metric.Resource.Name) + requests, err := utils.CalculatePodRequests(availablePods, v1.ResourceName(metricName), containerName) if err != nil { return nil, err } - value := int64((float64(requests) * float64(*metric.Resource.Target.AverageUtilization) / 100) / float64(len(availablePods))) + value := int64((float64(requests) * float64(*averageUtilization) / 100) / float64(len(availablePods))) averageValue = resource.NewMilliQuantity(value, resource.DecimalSI) - } else { - averageValue = metric.Resource.Target.AverageValue } case autoscalingv2.ExternalMetricSourceType: - metricIdentifier = utils.GetMetricIdentifier(metric, metric.External.Metric.Name) averageValue = metric.External.Target.AverageValue case autoscalingv2.PodsMetricSourceType: - metricIdentifier = utils.GetMetricIdentifier(metric, metric.Pods.Metric.Name) averageValue = metric.Pods.Target.AverageValue } + metricIdentifier = utils.GetPredictionMetricIdentifier(metric) if metricIdentifier == "" { continue } diff --git a/pkg/controller/ehpa/predict.go b/pkg/controller/ehpa/predict.go index c1431df23..0418255fa 100644 --- a/pkg/controller/ehpa/predict.go +++ b/pkg/controller/ehpa/predict.go @@ -133,25 +133,12 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect var predictionMetrics []predictionapi.PredictionMetric for _, metric := range ehpa.Spec.Metrics { - var metricName string - //get metricIdentifier by metric.Type and metricName - var metricIdentifier string - switch metric.Type { - case autoscalingv2.ResourceMetricSourceType: - metricName = metric.Resource.Name.String() - metricIdentifier = utils.GetMetricIdentifier(metric, metric.Resource.Name.String()) - case autoscalingv2.ExternalMetricSourceType: - metricName = metric.External.Metric.Name - metricIdentifier = utils.GetMetricIdentifier(metric, metric.External.Metric.Name) - case autoscalingv2.PodsMetricSourceType: - metricName = metric.Pods.Metric.Name - metricIdentifier = utils.GetMetricIdentifier(metric, metric.Pods.Metric.Name) - } - + metricIdentifier := utils.GetPredictionMetricIdentifier(metric) if metricIdentifier == "" { continue } + metricName := utils.GetMetricName(metric) //get matchLabels var matchLabels []string var metricRule *prometheus_adapter.MetricRule @@ -159,7 +146,7 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect // Supreme priority: annotation expressionQuery := utils.GetExpressionQueryAnnotation(metricIdentifier, ehpa.Annotations) if expressionQuery == "" { - var nameReg string + var podNameReg string // get metricRule from prometheus-adapter switch metric.Type { case autoscalingv2.ResourceMetricSourceType: @@ -169,7 +156,23 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricName[%s]", metricName) } else { klog.V(4).Infof("Got MetricRulesResource prometheus-adapter-resource MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName) - nameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind) + podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind) + } + } + case autoscalingv2.ContainerResourceMetricSourceType: + if len(mrs.MetricRulesResource) > 0 { + metricRule = prometheus_adapter.MatchMetricRule(mrs.MetricRulesResource, metricName) + if metricRule == nil { + klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricName[%s]", metricName) + } else { + klog.V(4).Infof("Got MetricRulesResource prometheus-adapter-resource MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName) + podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind) + // Compared to ResourceMetricSourceType, there is an additional container-name field + containerLabel := "container" + if metricRule.ContainerLabel != "" { + containerLabel = metricRule.ContainerLabel + } + matchLabels = append(matchLabels, utils.MapSortToArray(map[string]string{containerLabel: metric.ContainerResource.Container})...) } } case autoscalingv2.PodsMetricSourceType: @@ -179,12 +182,9 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect klog.Errorf("Got MetricRulesCustomer prometheus-adapter-customer Failed MetricName[%s]", metricName) } else { klog.V(4).Infof("Got MetricRulesCustomer prometheus-adapter-customer MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName) - nameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind) - + podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind) if metric.Pods.Metric.Selector != nil { - for _, i := range utils.MapSortToArray(metric.Pods.Metric.Selector.MatchLabels) { - matchLabels = append(matchLabels, i) - } + matchLabels = append(matchLabels, utils.MapSortToArray(metric.Pods.Metric.Selector.MatchLabels)...) } } } @@ -196,18 +196,16 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect } else { klog.V(4).Infof("Got MetricRulesExternal prometheus-adapter-external MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName) if metric.External.Metric.Selector != nil { - for _, i := range utils.MapSortToArray(metric.External.Metric.Selector.MatchLabels) { - matchLabels = append(matchLabels, i) - } + matchLabels = append(matchLabels, utils.MapSortToArray(metric.External.Metric.Selector.MatchLabels)...) } } } } if metricRule != nil { - // Second priority: get default expressionQuery + // Second priority: get prometheus-adapter expressionQuery var err error - expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, nameReg, append(mrs.ExtensionLabels, matchLabels...)) + expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, podNameReg, append(mrs.ExtensionLabels, matchLabels...)) if err != nil { klog.Errorf("Got promSelector prometheus-adapter %v %v", metricRule, err) } else { diff --git a/pkg/prometheus-adapter/expression.go b/pkg/prometheus-adapter/expression.go index b38ae9c91..e53355311 100644 --- a/pkg/prometheus-adapter/expression.go +++ b/pkg/prometheus-adapter/expression.go @@ -47,12 +47,13 @@ type MetricRules struct { } type MetricRule struct { - MetricMatches string - SeriesName string - ResConverter naming.ResourceConverter - Template *template.Template - Namespaced bool - LabelMatchers []string + MetricMatches string + SeriesName string + ResConverter naming.ResourceConverter + Template *template.Template + Namespaced bool + LabelMatchers []string + ContainerLabel string } type QueryTemplateArgs struct { @@ -129,7 +130,7 @@ func SetExtensionLabels(extensionLabels string) { } } -// GetMetricRuleResourceFromRules produces a MetricNamer for each rule in the given config. +// GetMetricRulesFromResourceRules produces a MetricNamer for each rule in the given config. func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTMapper) (metricRules []MetricRule, metricRulesError []string, err error) { // get cpu MetricsQuery if cfg.CPU.ContainerQuery != "" { @@ -154,10 +155,11 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTM klog.Errorf("unable to parse metrics query template %q: %v", cfg.CPU.ContainerQuery, err) } else { metricRules = append(metricRules, MetricRule{ - MetricMatches: "cpu", - ResConverter: resConverter, - Template: templ, - Namespaced: true, + MetricMatches: "cpu", + ResConverter: resConverter, + Template: templ, + Namespaced: true, + ContainerLabel: cfg.CPU.ContainerLabel, }) } } @@ -190,10 +192,11 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTM klog.Errorf("unable to parse metrics query template %q: %v", cfg.Memory.ContainerQuery, err) } else { metricRules = append(metricRules, MetricRule{ - MetricMatches: "memory", - ResConverter: resConverter, - Template: templ, - Namespaced: true, + MetricMatches: "memory", + ResConverter: resConverter, + Template: templ, + Namespaced: true, + ContainerLabel: cfg.Memory.ContainerLabel, }) } } diff --git a/pkg/utils/ehpa.go b/pkg/utils/ehpa.go index dd8bd1107..9e09e647a 100644 --- a/pkg/utils/ehpa.go +++ b/pkg/utils/ehpa.go @@ -6,9 +6,8 @@ import ( "strings" autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1" - autoscalingv2 "k8s.io/api/autoscaling/v2beta2" - "github.com/gocrane/crane/pkg/known" + autoscalingv2 "k8s.io/api/autoscaling/v2beta2" ) func IsEHPAPredictionEnabled(ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler) bool { @@ -39,7 +38,7 @@ func IsEHPACronEnabled(ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler) bo // GetPredictionMetricName return metric name used by prediction func GetPredictionMetricName(sourceType autoscalingv2.MetricSourceType) (metricName string) { switch sourceType { - case autoscalingv2.ResourceMetricSourceType, autoscalingv2.PodsMetricSourceType, autoscalingv2.ExternalMetricSourceType: + case autoscalingv2.ResourceMetricSourceType, autoscalingv2.ContainerResourceMetricSourceType, autoscalingv2.PodsMetricSourceType, autoscalingv2.ExternalMetricSourceType: metricName = known.MetricNamePrediction } @@ -51,19 +50,36 @@ func GetCronMetricName() string { return known.MetricNameCron } -// GetGeneralPredictionMetricName return metric name used by prediction -func GetMetricIdentifier(metric autoscalingv2.MetricSpec, name string) string { +func GetMetricName(metric autoscalingv2.MetricSpec) string { + switch metric.Type { + case autoscalingv2.PodsMetricSourceType: + return metric.Pods.Metric.Name + case autoscalingv2.ResourceMetricSourceType: + return metric.Resource.Name.String() + case autoscalingv2.ContainerResourceMetricSourceType: + return metric.ContainerResource.Name.String() + case autoscalingv2.ExternalMetricSourceType: + return metric.External.Metric.Name + default: + return "" + } +} + +// GetPredictionMetricIdentifier return metric name used by prediction +func GetPredictionMetricIdentifier(metric autoscalingv2.MetricSpec) string { var prefix string switch metric.Type { case autoscalingv2.PodsMetricSourceType: prefix = "pods" case autoscalingv2.ResourceMetricSourceType: prefix = "resource" + case autoscalingv2.ContainerResourceMetricSourceType: + prefix = "container-resource" case autoscalingv2.ExternalMetricSourceType: prefix = "external" } - return fmt.Sprintf("%s.%s", prefix, name) + return fmt.Sprintf("%s.%s", prefix, GetMetricName(metric)) } // GetExpressionQueryAnnotation return metric query from annotation by metricName @@ -95,7 +111,7 @@ func IsExpressionQueryAnnotationEnabled(metricIdentifier string, annotations map return false } -// GetExpressionQuery return metric query +// GetExpressionQueryDefault return default metric query func GetExpressionQueryDefault(metric autoscalingv2.MetricSpec, namespace string, name string, kind string) string { var expressionQuery string switch metric.Type { @@ -106,6 +122,13 @@ func GetExpressionQueryDefault(metric autoscalingv2.MetricSpec, namespace string case "memory": expressionQuery = GetWorkloadMemUsageExpression(namespace, name, kind) } + case autoscalingv2.ContainerResourceMetricSourceType: + switch metric.ContainerResource.Name { + case "cpu": + expressionQuery = GetContainerCpuUsageExpression(namespace, name, kind, metric.ContainerResource.Container) + case "memory": + expressionQuery = GetContainerMemUsageExpression(namespace, name, kind, metric.ContainerResource.Container) + } case autoscalingv2.PodsMetricSourceType: var labels []string if metric.Pods.Metric.Selector != nil { diff --git a/pkg/utils/pod.go b/pkg/utils/pod.go index eef7ea3e0..929cfa0f2 100644 --- a/pkg/utils/pod.go +++ b/pkg/utils/pod.go @@ -110,11 +110,14 @@ func EvictPodWithGracePeriod(client clientset.Interface, pod *v1.Pod, gracePerio return client.CoreV1().Pods(pod.Namespace).EvictV1beta1(context.Background(), e) } -// CalculatePodRequests sum request total from pods -func CalculatePodRequests(pods []v1.Pod, resource v1.ResourceName) (int64, error) { +// CalculatePodRequests sum request total from pods. If the containerName is specified, the total amount of requests for that container will be calculated. +func CalculatePodRequests(pods []v1.Pod, resource v1.ResourceName, containerName string) (int64, error) { var requests int64 for _, pod := range pods { for _, c := range pod.Spec.Containers { + if containerName != "" && c.Name != containerName { + continue + } if containerRequest, ok := c.Resources.Requests[resource]; ok { requests += containerRequest.MilliValue() } else { diff --git a/pkg/utils/pod_test.go b/pkg/utils/pod_test.go index d1e64fd39..badeecdcf 100644 --- a/pkg/utils/pod_test.go +++ b/pkg/utils/pod_test.go @@ -61,9 +61,10 @@ func TestCalculatePodRequests(t *testing.T) { } tests := []struct { - description string - resource v1.ResourceName - expect int64 + description string + resource v1.ResourceName + containerName string + expect int64 }{ { description: "calculate cpu request total", @@ -75,10 +76,22 @@ func TestCalculatePodRequests(t *testing.T) { resource: v1.ResourceMemory, expect: 60000, }, + { + description: "calculate cpu request total of container1", + resource: v1.ResourceCPU, + containerName: "container1", + expect: 3000, + }, + { + description: "calculate memory request total of container1", + resource: v1.ResourceMemory, + containerName: "container1", + expect: 30000, + }, } for _, test := range tests { - requests, err := CalculatePodRequests(pods, test.resource) + requests, err := CalculatePodRequests(pods, test.resource, test.containerName) if err != nil { t.Errorf("Failed to calculatePodRequests: %v", err) }