diff --git a/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1/types.go b/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1/types.go index 05f9fcbe5a49..cde79b1d6a5e 100644 --- a/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1/types.go +++ b/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1/types.go @@ -200,4 +200,8 @@ const ( // ProvisioningClassBestEffortAtomicScaleUp denotes that CA try to provision the capacity // in an atomic manner. ProvisioningClassBestEffortAtomicScaleUp string = "best-effort-atomic-scale-up.autoscaling.x-k8s.io" + // ProvisioningRequestPodAnnotationKey is a key used to annotate pods consuming provisioning request. + ProvisioningRequestPodAnnotationKey = "autoscaling.x-k8s.io/consume-provisioning-request" + // ProvisioningClassPodAnnotationKey is a key used to add annotation about Provisioning Class + ProvisioningClassPodAnnotationKey = "autoscaling.x-k8s.io/provisioning-class-name" ) diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index 5c81dd53fa55..e470b6958e73 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -46,7 +46,7 @@ require ( k8s.io/api v0.31.0-alpha.2 k8s.io/apimachinery v0.31.0-alpha.2 k8s.io/apiserver v0.31.0-alpha.2 - k8s.io/autoscaler/cluster-autoscaler/apis v0.0.0-20240606111128-34690b19c19b + k8s.io/autoscaler/cluster-autoscaler/apis v0.0.0-20240627115740-d52e4b9665d7 k8s.io/client-go v0.31.0-alpha.2 k8s.io/cloud-provider v0.30.0-alpha.3 k8s.io/cloud-provider-aws v1.27.0 diff --git a/cluster-autoscaler/processors/provreq/pods_filter.go b/cluster-autoscaler/processors/provreq/pods_filter.go index a75f62dfe67d..fb61ce25f3f7 100644 --- a/cluster-autoscaler/processors/provreq/pods_filter.go +++ b/cluster-autoscaler/processors/provreq/pods_filter.go @@ -22,16 +22,14 @@ import ( apiv1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1" "k8s.io/autoscaler/cluster-autoscaler/context" "k8s.io/autoscaler/cluster-autoscaler/processors/pods" + provreqpods "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/pods" "k8s.io/autoscaler/cluster-autoscaler/utils/klogx" ) -const ( - // ProvisioningRequestPodAnnotationKey is an annotation on pod that indicate that pod was created by ProvisioningRequest. - ProvisioningRequestPodAnnotationKey = "cluster-autoscaler.kubernetes.io/consume-provisioning-request" - maxProvReqEvent = 50 -) +const maxProvReqEvent = 50 // EventManager is an interface for handling events for provisioning request. type EventManager interface { @@ -102,6 +100,9 @@ func provisioningRequestName(pod *v1.Pod) (string, bool) { if pod == nil || pod.Annotations == nil { return "", false } - provReqName, found := pod.Annotations[ProvisioningRequestPodAnnotationKey] + provReqName, found := pod.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey] + if !found { + provReqName, found = pod.Annotations[provreqpods.DeprecatedProvisioningRequestPodAnnotationKey] + } return provReqName, found } diff --git a/cluster-autoscaler/processors/provreq/pods_filter_test.go b/cluster-autoscaler/processors/provreq/pods_filter_test.go index 07c224dadac3..80977fcdd30c 100644 --- a/cluster-autoscaler/processors/provreq/pods_filter_test.go +++ b/cluster-autoscaler/processors/provreq/pods_filter_test.go @@ -24,17 +24,19 @@ import ( "github.com/stretchr/testify/assert" apiv1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1" "k8s.io/autoscaler/cluster-autoscaler/context" + "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/pods" . "k8s.io/autoscaler/cluster-autoscaler/utils/test" "k8s.io/client-go/tools/record" ) func TestProvisioningRequestPodsFilter(t *testing.T) { prPod1 := BuildTestPod("pr-pod-1", 500, 10) - prPod1.Annotations[ProvisioningRequestPodAnnotationKey] = "pr-class" + prPod1.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey] = "pr-class" prPod2 := BuildTestPod("pr-pod-2", 500, 10) - prPod2.Annotations[ProvisioningRequestPodAnnotationKey] = "pr-class-2" + prPod2.Annotations[pods.DeprecatedProvisioningRequestPodAnnotationKey] = "pr-class-2" pod1 := BuildTestPod("pod-1", 500, 10) pod2 := BuildTestPod("pod-2", 500, 10) @@ -91,7 +93,7 @@ func TestEventManager(t *testing.T) { for i := 0; i < 10; i++ { prPod := BuildTestPod(fmt.Sprintf("pr-pod-%d", i), 10, 10) - prPod.Annotations[ProvisioningRequestPodAnnotationKey] = "pr-class" + prPod.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey] = "pr-class" unscheduledPods = append(unscheduledPods, prPod) } got, err := prFilter.Process(ctx, unscheduledPods) diff --git a/cluster-autoscaler/proposals/provisioning-request.md b/cluster-autoscaler/proposals/provisioning-request.md index 0540cd7d8dc0..b412d6645273 100644 --- a/cluster-autoscaler/proposals/provisioning-request.md +++ b/cluster-autoscaler/proposals/provisioning-request.md @@ -184,10 +184,13 @@ not required in ProvReq’s template, though can be specified): ```yaml annotations: - "cluster-autoscaler.kubernetes.io/provisioning-class-name": "provreq-class-name" - "cluster-autoscaler.kubernetes.io/consume-provisioning-request": "provreq-name" + "autoscaling.x-k8s.io/provisioning-class-name": "provreq-class-name" + "autoscaling.x-k8s.io/consume-provisioning-request": "provreq-name" ``` +Previous prosoal included annotations with prefix `cluster-autoscaler.kubernetes.io` +but were deprecated as part of API reivew. + If those are provided for the pods that consume the ProvReq with `check-capacity.kubernetes.io` class, the CA will not provision the capacity, even if it was needed (as some other pods might have been scheduled on it) and will result in visibility events passed to the ProvReq and pods. diff --git a/cluster-autoscaler/provisioningrequest/orchestrator/wrapper_orchestrator.go b/cluster-autoscaler/provisioningrequest/orchestrator/wrapper_orchestrator.go index 5c7f792e55b2..c74e4a1148ab 100644 --- a/cluster-autoscaler/provisioningrequest/orchestrator/wrapper_orchestrator.go +++ b/cluster-autoscaler/provisioningrequest/orchestrator/wrapper_orchestrator.go @@ -19,13 +19,13 @@ package orchestrator import ( appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1" "k8s.io/autoscaler/cluster-autoscaler/clusterstate" "k8s.io/autoscaler/cluster-autoscaler/context" "k8s.io/autoscaler/cluster-autoscaler/core/scaleup" "k8s.io/autoscaler/cluster-autoscaler/core/scaleup/orchestrator" "k8s.io/autoscaler/cluster-autoscaler/estimator" ca_processors "k8s.io/autoscaler/cluster-autoscaler/processors" - "k8s.io/autoscaler/cluster-autoscaler/processors/provreq" "k8s.io/autoscaler/cluster-autoscaler/processors/status" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/taints" @@ -87,7 +87,7 @@ func (o *WrapperOrchestrator) ScaleUp( func splitOut(unschedulablePods []*apiv1.Pod) (provReqPods, regularPods []*apiv1.Pod) { for _, pod := range unschedulablePods { - if _, ok := pod.Annotations[provreq.ProvisioningRequestPodAnnotationKey]; ok { + if _, ok := pod.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey]; ok { provReqPods = append(provReqPods, pod) } else { regularPods = append(regularPods, pod) diff --git a/cluster-autoscaler/provisioningrequest/orchestrator/wrapper_orchestrator_test.go b/cluster-autoscaler/provisioningrequest/orchestrator/wrapper_orchestrator_test.go index bf31f59f8b1b..814a213b340f 100644 --- a/cluster-autoscaler/provisioningrequest/orchestrator/wrapper_orchestrator_test.go +++ b/cluster-autoscaler/provisioningrequest/orchestrator/wrapper_orchestrator_test.go @@ -22,11 +22,11 @@ import ( "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1" "k8s.io/autoscaler/cluster-autoscaler/clusterstate" "k8s.io/autoscaler/cluster-autoscaler/context" "k8s.io/autoscaler/cluster-autoscaler/estimator" ca_processors "k8s.io/autoscaler/cluster-autoscaler/processors" - "k8s.io/autoscaler/cluster-autoscaler/processors/provreq" "k8s.io/autoscaler/cluster-autoscaler/processors/status" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/taints" @@ -53,7 +53,7 @@ func TestWrapperScaleUp(t *testing.T) { BuildTestPod("pr-pod-2", 1, 100), } for _, pod := range provReqPods { - pod.Annotations[provreq.ProvisioningRequestPodAnnotationKey] = "true" + pod.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey] = "true" } unschedulablePods := append(regularPods, provReqPods...) _, err := o.ScaleUp(unschedulablePods, nil, nil, nil, false) diff --git a/cluster-autoscaler/provisioningrequest/pods/pods.go b/cluster-autoscaler/provisioningrequest/pods/pods.go index 712f85453ab1..99ee740f6945 100644 --- a/cluster-autoscaler/provisioningrequest/pods/pods.go +++ b/cluster-autoscaler/provisioningrequest/pods/pods.go @@ -23,15 +23,16 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1" "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/provreqwrapper" "k8s.io/kubernetes/pkg/controller" ) const ( - // ProvisioningRequestPodAnnotationKey is a key used to annotate pods consuming provisioning request. - ProvisioningRequestPodAnnotationKey = "cluster-autoscaler.kubernetes.io/consume-provisioning-request" - // ProvisioningClassPodAnnotationKey is a key used to add annotation about Provisioning Class - ProvisioningClassPodAnnotationKey = "cluster-autoscaler.kubernetes.io/provisioning-class-name" + // DeprecatedProvisioningRequestPodAnnotationKey is a key used to annotate pods consuming provisioning request. + DeprecatedProvisioningRequestPodAnnotationKey = "cluster-autoscaler.kubernetes.io/consume-provisioning-request" + // DeprecatedProvisioningClassPodAnnotationKey is a key used to add annotation about Provisioning Class + DeprecatedProvisioningClassPodAnnotationKey = "cluster-autoscaler.kubernetes.io/provisioning-class-name" ) // PodsForProvisioningRequest returns a list of pods for which Provisioning @@ -77,8 +78,8 @@ func populatePodFields(pr *provreqwrapper.ProvisioningRequest, pod *v1.Pod, i, j if pod.Annotations == nil { pod.Annotations = make(map[string]string) } - pod.Annotations[ProvisioningRequestPodAnnotationKey] = pr.Name - pod.Annotations[ProvisioningClassPodAnnotationKey] = pr.Spec.ProvisioningClassName + pod.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey] = pr.Name + pod.Annotations[v1beta1.ProvisioningClassPodAnnotationKey] = pr.Spec.ProvisioningClassName pod.UID = types.UID(fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)) pod.CreationTimestamp = pr.CreationTimestamp } diff --git a/cluster-autoscaler/provisioningrequest/pods/pods_test.go b/cluster-autoscaler/provisioningrequest/pods/pods_test.go index e33cc83e9422..c8c3e95c5261 100644 --- a/cluster-autoscaler/provisioningrequest/pods/pods_test.go +++ b/cluster-autoscaler/provisioningrequest/pods/pods_test.go @@ -42,8 +42,8 @@ func TestPodsForProvisioningRequest(t *testing.T) { Namespace: "test-namespace", UID: types.UID(fmt.Sprintf("test-namespace/%s", name)), Annotations: map[string]string{ - ProvisioningRequestPodAnnotationKey: prName, - ProvisioningClassPodAnnotationKey: testProvisioningClassName, + v1beta1.ProvisioningRequestPodAnnotationKey: prName, + v1beta1.ProvisioningClassPodAnnotationKey: testProvisioningClassName, }, Labels: map[string]string{}, Finalizers: []string{},