Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add ContainerResource source type for ehpa prediction #859

Merged
merged 2 commits into from
Sep 13, 2023
Merged

add ContainerResource source type for ehpa prediction #859

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4a459d3
add ContainerResource source type for ehpa prediction
whitebear009 Aug 30, 2023
b5475b7
remove useless function
whitebear009 Aug 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 26 additions & 9 deletions pkg/controller/ehpa/hpa.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,10 +208,30 @@ func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autosc
var metricIdentifier string
var averageValue *resource.Quantity
switch metric.Type {
case autoscalingv2.ResourceMetricSourceType:
metricIdentifier = utils.GetMetricIdentifier(metric, metric.Resource.Name.String())
case autoscalingv2.ResourceMetricSourceType, autoscalingv2.ContainerResourceMetricSourceType:
var averageUtilization *int32
var containerName string
if metric.Resource != nil {
if metric.Resource.Target.AverageUtilization != nil {
averageUtilization = metric.Resource.Target.AverageUtilization
}
if metric.Resource.Target.AverageValue != nil {
averageValue = metric.Resource.Target.AverageValue
}
}
if metric.ContainerResource != nil {
containerName = metric.ContainerResource.Container
if metric.ContainerResource.Target.AverageUtilization != nil {
averageUtilization = metric.ContainerResource.Target.AverageUtilization
}
if metric.ContainerResource.Target.AverageValue != nil {
averageValue = metric.ContainerResource.Target.AverageValue
}
}

// When use AverageUtilization in EffectiveHorizontalPodAutoscaler's metricSpec, convert to AverageValue
if metric.Resource.Target.AverageUtilization != nil {
if averageUtilization != nil {
metricName := utils.GetMetricName(metric)
scale, _, err := utils.GetScale(ctx, c.RestMapper, c.ScaleClient, ehpa.Namespace, ehpa.Spec.ScaleTargetRef)
if err != nil {
return nil, err
Expand All @@ -231,24 +251,21 @@ func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autosc
return nil, fmt.Errorf("failed to get available pods. ")
}

requests, err := utils.CalculatePodRequests(availablePods, metric.Resource.Name)
requests, err := utils.CalculatePodRequests(availablePods, v1.ResourceName(metricName), containerName)
if err != nil {
return nil, err
}

value := int64((float64(requests) * float64(*metric.Resource.Target.AverageUtilization) / 100) / float64(len(availablePods)))
value := int64((float64(requests) * float64(*averageUtilization) / 100) / float64(len(availablePods)))
averageValue = resource.NewMilliQuantity(value, resource.DecimalSI)
} else {
averageValue = metric.Resource.Target.AverageValue
}
case autoscalingv2.ExternalMetricSourceType:
metricIdentifier = utils.GetMetricIdentifier(metric, metric.External.Metric.Name)
averageValue = metric.External.Target.AverageValue
case autoscalingv2.PodsMetricSourceType:
metricIdentifier = utils.GetMetricIdentifier(metric, metric.Pods.Metric.Name)
averageValue = metric.Pods.Target.AverageValue
}

metricIdentifier = utils.GetPredictionMetricIdentifier(metric)
if metricIdentifier == "" {
continue
}
Expand Down
52 changes: 25 additions & 27 deletions pkg/controller/ehpa/predict.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,33 +133,20 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect

var predictionMetrics []predictionapi.PredictionMetric
for _, metric := range ehpa.Spec.Metrics {
var metricName string
//get metricIdentifier by metric.Type and metricName
var metricIdentifier string
switch metric.Type {
case autoscalingv2.ResourceMetricSourceType:
metricName = metric.Resource.Name.String()
metricIdentifier = utils.GetMetricIdentifier(metric, metric.Resource.Name.String())
case autoscalingv2.ExternalMetricSourceType:
metricName = metric.External.Metric.Name
metricIdentifier = utils.GetMetricIdentifier(metric, metric.External.Metric.Name)
case autoscalingv2.PodsMetricSourceType:
metricName = metric.Pods.Metric.Name
metricIdentifier = utils.GetMetricIdentifier(metric, metric.Pods.Metric.Name)
}

metricIdentifier := utils.GetPredictionMetricIdentifier(metric)
if metricIdentifier == "" {
continue
}

metricName := utils.GetMetricName(metric)
//get matchLabels
var matchLabels []string
var metricRule *prometheus_adapter.MetricRule

// Supreme priority: annotation
expressionQuery := utils.GetExpressionQueryAnnotation(metricIdentifier, ehpa.Annotations)
if expressionQuery == "" {
var nameReg string
var podNameReg string
// get metricRule from prometheus-adapter
switch metric.Type {
case autoscalingv2.ResourceMetricSourceType:
Expand All @@ -169,7 +156,23 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricName[%s]", metricName)
} else {
klog.V(4).Infof("Got MetricRulesResource prometheus-adapter-resource MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
nameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
}
}
case autoscalingv2.ContainerResourceMetricSourceType:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would you help to provide a sample and explain how to convert it? Also please show how to config it in prometheus-adapter mode. tks.

Copy link
Contributor Author

@whitebear009 whitebear009 Sep 12, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here is a sample:

metrics:
  - type: ContainerResource
    containerResource:
      container: app
      name: cpu
      target:
        type: Utilization
        averageUtilization: 50

the default expression will convert it to promql like irate(container_cpu_usage_seconds_total{container!="POD",namespace="kube-system",pod=~"pod1|pod2",container="app"}[3m]) in file https://github.com/gocrane/crane/pull/859/files#diff-58fd591c9f408018596b9c5b19515f115c97a0243aeadcab4984c3d5bed6edc9R128

In prometheus-adapter mode, its config is basicly the same with pod resource metrics. The only difference is that compared to pod resource metrics, there is an additional field representing container name. This field name defaults to "container" and can be modified through the containerLabel of prometheus-adapter(https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/pkg/config/config.go#L110) (this is also the original purpose of this field, because compared with namespace/pod, container is not K8s resources, so a separate field is required)

if len(mrs.MetricRulesResource) > 0 {
metricRule = prometheus_adapter.MatchMetricRule(mrs.MetricRulesResource, metricName)
if metricRule == nil {
klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricName[%s]", metricName)
} else {
klog.V(4).Infof("Got MetricRulesResource prometheus-adapter-resource MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
// Compared to ResourceMetricSourceType, there is an additional container-name field
containerLabel := "container"
if metricRule.ContainerLabel != "" {
containerLabel = metricRule.ContainerLabel
}
matchLabels = append(matchLabels, utils.MapSortToArray(map[string]string{containerLabel: metric.ContainerResource.Container})...)
}
}
case autoscalingv2.PodsMetricSourceType:
Expand All @@ -179,12 +182,9 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
klog.Errorf("Got MetricRulesCustomer prometheus-adapter-customer Failed MetricName[%s]", metricName)
} else {
klog.V(4).Infof("Got MetricRulesCustomer prometheus-adapter-customer MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
nameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)

podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
if metric.Pods.Metric.Selector != nil {
for _, i := range utils.MapSortToArray(metric.Pods.Metric.Selector.MatchLabels) {
matchLabels = append(matchLabels, i)
}
matchLabels = append(matchLabels, utils.MapSortToArray(metric.Pods.Metric.Selector.MatchLabels)...)
}
}
}
Expand All @@ -196,18 +196,16 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
} else {
klog.V(4).Infof("Got MetricRulesExternal prometheus-adapter-external MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
if metric.External.Metric.Selector != nil {
for _, i := range utils.MapSortToArray(metric.External.Metric.Selector.MatchLabels) {
matchLabels = append(matchLabels, i)
}
matchLabels = append(matchLabels, utils.MapSortToArray(metric.External.Metric.Selector.MatchLabels)...)
}
}
}
}

if metricRule != nil {
// Second priority: get default expressionQuery
// Second priority: get prometheus-adapter expressionQuery
var err error
expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, nameReg, append(mrs.ExtensionLabels, matchLabels...))
expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, podNameReg, append(mrs.ExtensionLabels, matchLabels...))
if err != nil {
klog.Errorf("Got promSelector prometheus-adapter %v %v", metricRule, err)
} else {
Expand Down
33 changes: 18 additions & 15 deletions pkg/prometheus-adapter/expression.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,13 @@ type MetricRules struct {
}

type MetricRule struct {
MetricMatches string
SeriesName string
ResConverter naming.ResourceConverter
Template *template.Template
Namespaced bool
LabelMatchers []string
MetricMatches string
SeriesName string
ResConverter naming.ResourceConverter
Template *template.Template
Namespaced bool
LabelMatchers []string
ContainerLabel string
}

type QueryTemplateArgs struct {
Expand Down Expand Up @@ -129,7 +130,7 @@ func SetExtensionLabels(extensionLabels string) {
}
}

// GetMetricRuleResourceFromRules produces a MetricNamer for each rule in the given config.
// GetMetricRulesFromResourceRules produces a MetricNamer for each rule in the given config.
func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTMapper) (metricRules []MetricRule, metricRulesError []string, err error) {
// get cpu MetricsQuery
if cfg.CPU.ContainerQuery != "" {
Expand All @@ -154,10 +155,11 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTM
klog.Errorf("unable to parse metrics query template %q: %v", cfg.CPU.ContainerQuery, err)
} else {
metricRules = append(metricRules, MetricRule{
MetricMatches: "cpu",
ResConverter: resConverter,
Template: templ,
Namespaced: true,
MetricMatches: "cpu",
ResConverter: resConverter,
Template: templ,
Namespaced: true,
ContainerLabel: cfg.CPU.ContainerLabel,
})
}
}
Expand Down Expand Up @@ -190,10 +192,11 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTM
klog.Errorf("unable to parse metrics query template %q: %v", cfg.Memory.ContainerQuery, err)
} else {
metricRules = append(metricRules, MetricRule{
MetricMatches: "memory",
ResConverter: resConverter,
Template: templ,
Namespaced: true,
MetricMatches: "memory",
ResConverter: resConverter,
Template: templ,
Namespaced: true,
ContainerLabel: cfg.Memory.ContainerLabel,
})
}
}
Expand Down
37 changes: 30 additions & 7 deletions pkg/utils/ehpa.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@ import (
"strings"

autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1"
autoscalingv2 "k8s.io/api/autoscaling/v2beta2"

"github.com/gocrane/crane/pkg/known"
autoscalingv2 "k8s.io/api/autoscaling/v2beta2"
)

func IsEHPAPredictionEnabled(ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler) bool {
Expand Down Expand Up @@ -39,7 +38,7 @@ func IsEHPACronEnabled(ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler) bo
// GetPredictionMetricName return metric name used by prediction
func GetPredictionMetricName(sourceType autoscalingv2.MetricSourceType) (metricName string) {
switch sourceType {
case autoscalingv2.ResourceMetricSourceType, autoscalingv2.PodsMetricSourceType, autoscalingv2.ExternalMetricSourceType:
case autoscalingv2.ResourceMetricSourceType, autoscalingv2.ContainerResourceMetricSourceType, autoscalingv2.PodsMetricSourceType, autoscalingv2.ExternalMetricSourceType:
metricName = known.MetricNamePrediction
}

Expand All @@ -51,19 +50,36 @@ func GetCronMetricName() string {
return known.MetricNameCron
}

// GetGeneralPredictionMetricName return metric name used by prediction
func GetMetricIdentifier(metric autoscalingv2.MetricSpec, name string) string {
func GetMetricName(metric autoscalingv2.MetricSpec) string {
switch metric.Type {
case autoscalingv2.PodsMetricSourceType:
return metric.Pods.Metric.Name
case autoscalingv2.ResourceMetricSourceType:
return metric.Resource.Name.String()
case autoscalingv2.ContainerResourceMetricSourceType:
return metric.ContainerResource.Name.String()
case autoscalingv2.ExternalMetricSourceType:
return metric.External.Metric.Name
default:
return ""
}
}

// GetPredictionMetricIdentifier return metric name used by prediction
func GetPredictionMetricIdentifier(metric autoscalingv2.MetricSpec) string {
var prefix string
switch metric.Type {
case autoscalingv2.PodsMetricSourceType:
prefix = "pods"
case autoscalingv2.ResourceMetricSourceType:
prefix = "resource"
case autoscalingv2.ContainerResourceMetricSourceType:
prefix = "container-resource"
case autoscalingv2.ExternalMetricSourceType:
prefix = "external"
}

return fmt.Sprintf("%s.%s", prefix, name)
return fmt.Sprintf("%s.%s", prefix, GetMetricName(metric))
}

// GetExpressionQueryAnnotation return metric query from annotation by metricName
Expand Down Expand Up @@ -95,7 +111,7 @@ func IsExpressionQueryAnnotationEnabled(metricIdentifier string, annotations map
return false
}

// GetExpressionQuery return metric query
// GetExpressionQueryDefault return default metric query
func GetExpressionQueryDefault(metric autoscalingv2.MetricSpec, namespace string, name string, kind string) string {
var expressionQuery string
switch metric.Type {
Expand All @@ -106,6 +122,13 @@ func GetExpressionQueryDefault(metric autoscalingv2.MetricSpec, namespace string
case "memory":
expressionQuery = GetWorkloadMemUsageExpression(namespace, name, kind)
}
case autoscalingv2.ContainerResourceMetricSourceType:
switch metric.ContainerResource.Name {
case "cpu":
expressionQuery = GetContainerCpuUsageExpression(namespace, name, kind, metric.ContainerResource.Container)
case "memory":
expressionQuery = GetContainerMemUsageExpression(namespace, name, kind, metric.ContainerResource.Container)
}
case autoscalingv2.PodsMetricSourceType:
var labels []string
if metric.Pods.Metric.Selector != nil {
Expand Down
7 changes: 5 additions & 2 deletions pkg/utils/pod.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,11 +110,14 @@ func EvictPodWithGracePeriod(client clientset.Interface, pod *v1.Pod, gracePerio
return client.CoreV1().Pods(pod.Namespace).EvictV1beta1(context.Background(), e)
}

// CalculatePodRequests sum request total from pods
func CalculatePodRequests(pods []v1.Pod, resource v1.ResourceName) (int64, error) {
// CalculatePodRequests sum request total from pods. If the containerName is specified, the total amount of requests for that container will be calculated.
func CalculatePodRequests(pods []v1.Pod, resource v1.ResourceName, containerName string) (int64, error) {
var requests int64
for _, pod := range pods {
for _, c := range pod.Spec.Containers {
if containerName != "" && c.Name != containerName {
continue
}
if containerRequest, ok := c.Resources.Requests[resource]; ok {
requests += containerRequest.MilliValue()
} else {
Expand Down
21 changes: 17 additions & 4 deletions pkg/utils/pod_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,10 @@ func TestCalculatePodRequests(t *testing.T) {
}

tests := []struct {
description string
resource v1.ResourceName
expect int64
description string
resource v1.ResourceName
containerName string
expect int64
}{
{
description: "calculate cpu request total",
Expand All @@ -75,10 +76,22 @@ func TestCalculatePodRequests(t *testing.T) {
resource: v1.ResourceMemory,
expect: 60000,
},
{
description: "calculate cpu request total of container1",
resource: v1.ResourceCPU,
containerName: "container1",
expect: 3000,
},
{
description: "calculate memory request total of container1",
resource: v1.ResourceMemory,
containerName: "container1",
expect: 30000,
},
}

for _, test := range tests {
requests, err := CalculatePodRequests(pods, test.resource)
requests, err := CalculatePodRequests(pods, test.resource, test.containerName)
if err != nil {
t.Errorf("Failed to calculatePodRequests: %v", err)
}
Expand Down
Loading