diff --git a/pkg/controllers/provisioning/nodepool_test.go b/pkg/controllers/provisioning/nodepool_test.go deleted file mode 100644 index 7bc411d5d7..0000000000 --- a/pkg/controllers/provisioning/nodepool_test.go +++ /dev/null @@ -1,1502 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provisioning_test - -import ( - "fmt" - - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "knative.dev/pkg/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/aws/karpenter-core/pkg/apis/v1beta1" - "github.com/aws/karpenter-core/pkg/cloudprovider/fake" - "github.com/aws/karpenter-core/pkg/test" - . "github.com/aws/karpenter-core/pkg/test/expectations" -) - -var _ = Describe("NodePool/Provisioning", func() { - It("should provision nodes", func() { - ExpectApplied(ctx, env.Client, test.NodePool()) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - nodes := &v1.NodeList{} - Expect(env.Client.List(ctx, nodes)).To(Succeed()) - Expect(len(nodes.Items)).To(Equal(1)) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should ignore NodePools that are deleting", func() { - nodePool := test.NodePool() - ExpectApplied(ctx, env.Client, nodePool) - ExpectDeletionTimestampSet(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - nodes := &v1.NodeList{} - Expect(env.Client.List(ctx, nodes)).To(Succeed()) - Expect(len(nodes.Items)).To(Equal(0)) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should provision nodes for pods with supported node selectors", func() { - nodePool := test.NodePool() - schedulable := []*v1.Pod{ - // Constrained by nodepool - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: nodePool.Name}}), - // Constrained by zone - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), - // Constrained by instanceType - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), - // Constrained by architecture - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), - // Constrained by operatingSystem - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), - } - unschedulable := []*v1.Pod{ - // Ignored, matches another nodepool - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: "unknown"}}), - // Ignored, invalid zone - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}), - // Ignored, invalid instance type - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "unknown"}}), - // Ignored, invalid architecture - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "unknown"}}), - // Ignored, invalid operating system - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: "unknown"}}), - // Ignored, invalid capacity type - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.CapacityTypeLabelKey: "unknown"}}), - // Ignored, label selector does not match - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{"foo": "bar"}}), - } - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) - for _, pod := range schedulable { - ExpectScheduled(ctx, env.Client, pod) - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, unschedulable...) - for _, pod := range unschedulable { - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should provision nodes for pods with supported node affinities", func() { - nodePool := test.NodePool() - schedulable := []*v1.Pod{ - // Constrained by nodepool - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1beta1.NodePoolLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{nodePool.Name}}}}), - // Constrained by zone - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}}}), - // Constrained by instanceType - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type"}}}}), - // Constrained by architecture - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{"arm64"}}}}), - // Constrained by operatingSystem - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{string(v1.Linux)}}}}), - } - unschedulable := []*v1.Pod{ - // Ignored, matches another nodepool - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1beta1.NodePoolLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, invalid zone - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, invalid instance type - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, invalid architecture - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, invalid operating system - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, invalid capacity type - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, label selector does not match - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: "foo", Operator: v1.NodeSelectorOpIn, Values: []string{"bar"}}}}), - } - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) - for _, pod := range schedulable { - ExpectScheduled(ctx, env.Client, pod) - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, unschedulable...) - for _, pod := range unschedulable { - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should provision nodes for accelerators", func() { - ExpectApplied(ctx, env.Client, test.NodePool()) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{fake.ResourceGPUVendorA: resource.MustParse("1")}}, - }), - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{fake.ResourceGPUVendorB: resource.MustParse("1")}}, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } - }) - It("should provision multiple nodes when maxPods is set", func() { - // Kubelet is actually not observed here, the scheduler is relying on the - // pods resource value which is statically set in the fake cloudprovider - ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Kubelet: &v1beta1.KubeletConfiguration{MaxPods: ptr.Int32(1)}, - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"single-pod-instance-type"}, - }, - }, - }, - }, - }, - })) - pods := []*v1.Pod{ - test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod(), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodes := &v1.NodeList{} - Expect(env.Client.List(ctx, nodes)).To(Succeed()) - Expect(len(nodes.Items)).To(Equal(3)) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } - }) - It("should schedule all pods on one inflight node when node is in deleting state", func() { - nodePool := test.NodePool() - its, err := cloudProvider.GetInstanceTypes(ctx, nodePool) - Expect(err).To(BeNil()) - node := test.Node(test.NodeOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelInstanceTypeStable: its[0].Name, - }, - Finalizers: []string{v1beta1.TerminationFinalizer}, - }}, - ) - ExpectApplied(ctx, env.Client, node, nodePool) - ExpectReconcileSucceeded(ctx, nodeController, client.ObjectKeyFromObject(node)) - - // Schedule 3 pods to the node that currently exists - for i := 0; i < 3; i++ { - pod := test.UnschedulablePod() - ExpectApplied(ctx, env.Client, pod) - ExpectManualBinding(ctx, env.Client, pod, node) - } - - // Node shouldn't fully delete since it has a finalizer - Expect(env.Client.Delete(ctx, node)).To(Succeed()) - ExpectReconcileSucceeded(ctx, nodeController, client.ObjectKeyFromObject(node)) - - // Provision without a binding since some pods will already be bound - // Should all schedule to the new node, ignoring the old node - bindings := ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, test.UnschedulablePod(), test.UnschedulablePod()) - nodes := &v1.NodeList{} - Expect(env.Client.List(ctx, nodes)).To(Succeed()) - Expect(len(nodes.Items)).To(Equal(2)) - - // Scheduler should attempt to schedule all the pods to the new node - for _, n := range bindings { - Expect(n.Node.Name).ToNot(Equal(node.Name)) - } - }) - Context("Resource Limits", func() { - It("should not schedule when limits are exceeded", func() { - ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("20")}), - }, - Status: v1beta1.NodePoolStatus{ - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100"), - }, - }, - })) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule if limits would be met", func() { - ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}), - }, - })) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - // requires a 2 CPU node, but leaves room for overhead - v1.ResourceCPU: resource.MustParse("1.75"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // A 2 CPU node can be launched - ExpectScheduled(ctx, env.Client, pod) - }) - It("should partially schedule if limits would be exceeded", func() { - ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("3")}), - }, - })) - - // prevent these pods from scheduling on the same node - opts := test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "foo"}, - }, - PodAntiRequirements: []v1.PodAffinityTerm{ - { - TopologyKey: v1.LabelHostname, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "foo", - }, - }, - }, - }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1.5"), - }}} - pods := []*v1.Pod{ - test.UnschedulablePod(opts), - test.UnschedulablePod(opts), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - scheduledPodCount := 0 - unscheduledPodCount := 0 - pod0 := ExpectPodExists(ctx, env.Client, pods[0].Name, pods[0].Namespace) - pod1 := ExpectPodExists(ctx, env.Client, pods[1].Name, pods[1].Namespace) - if pod0.Spec.NodeName == "" { - unscheduledPodCount++ - } else { - scheduledPodCount++ - } - if pod1.Spec.NodeName == "" { - unscheduledPodCount++ - } else { - scheduledPodCount++ - } - Expect(scheduledPodCount).To(Equal(1)) - Expect(unscheduledPodCount).To(Equal(1)) - }) - It("should not schedule if limits would be exceeded", func() { - ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}), - }, - })) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2.1"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if limits would be exceeded (GPU)", func() { - ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Limits: v1beta1.Limits(v1.ResourceList{v1.ResourcePods: resource.MustParse("1")}), - }, - })) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // only available instance type has 2 GPUs which would exceed the limit - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule to a provisioner after a scheduling round if limits would be exceeded", func() { - ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}), - }, - })) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - // requires a 2 CPU node, but leaves room for overhead - v1.ResourceCPU: resource.MustParse("1.75"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // A 2 CPU node can be launched - ExpectScheduled(ctx, env.Client, pod) - - // This pod requests over the existing limit (would add to 3.5 CPUs) so this should fail - pod = test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - // requires a 2 CPU node, but leaves room for overhead - v1.ResourceCPU: resource.MustParse("1.75"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - }) - Context("Daemonsets and Node Overhead", func() { - It("should account for overhead", func() { - ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) - }) - It("should account for overhead (with startup taint)", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - StartupTaints: []v1.Taint{{Key: "foo.com/taint", Effect: v1.TaintEffectNoSchedule}}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool, test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) - }) - It("should not schedule if overhead is too large", func() { - ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}}, - }}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should account for overhead using daemonset pod spec instead of daemonset spec", func() { - nodePool := test.NodePool() - // Create a daemonset with large resource requests - daemonset := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}}, - }}, - ) - ExpectApplied(ctx, env.Client, nodePool, daemonset) - // Create the actual daemonSet pod with lower resource requests and expect to use the pod - daemonsetPod := test.UnschedulablePod( - test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps/v1", - Kind: "DaemonSet", - Name: daemonset.Name, - UID: daemonset.UID, - Controller: ptr.Bool(true), - BlockOwnerDeletion: ptr.Bool(true), - }, - }, - }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }) - ExpectApplied(ctx, env.Client, nodePool, daemonsetPod) - ExpectReconcileSucceeded(ctx, daemonsetController, client.ObjectKeyFromObject(daemonset)) - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: nodePool.Name}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - - // We expect a smaller instance since the daemonset pod is smaller then daemonset spec - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) - }) - It("should not schedule if resource requests are not defined and limits (requests) are too large", func() { - ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - }}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule based on the max resource requests of containers and initContainers", func() { - ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("2Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - }}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) - }) - It("should not schedule if combined max resources are too large for any node", func() { - ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("1Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - }}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if initContainer resources are too large", func() { - ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, - }, - }}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should be able to schedule pods if resource requests and limits are not defined", func() { - ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{}}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should ignore daemonsets without matching tolerations", func() { - ExpectApplied(ctx, env.Client, - test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoSchedule}}, - }, - }, - }, - }), - test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - Tolerations: []v1.Toleration{{Operator: v1.TolerationOperator(v1.NodeSelectorOpExists)}}, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("2"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("2Gi"))) - }) - It("should ignore daemonsets with an invalid selector", func() { - ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - NodeSelector: map[string]string{"node": "invalid"}, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("2"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("2Gi"))) - }) - It("should account daemonsets with NotIn operator and unspecified key", func() { - ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{{Key: "foo", Operator: v1.NodeSelectorOpNotIn, Values: []string{"bar"}}}, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}}, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) - }) - It("should account for daemonset spec affinity", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - ObjectMeta: v1beta1.ObjectMeta{ - Labels: map[string]string{ - "foo": "voo", - }, - }, - }, - Limits: v1beta1.Limits(v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - }), - }, - }) - nodePoolDaemonset := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - ObjectMeta: v1beta1.ObjectMeta{ - Labels: map[string]string{ - "foo": "bar", - }, - }, - }, - }, - }) - // Create a daemonset with large resource requests - daemonset := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: "foo", - Operator: v1.NodeSelectorOpIn, - Values: []string{"bar"}, - }, - }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}}, - }}, - ) - ExpectApplied(ctx, env.Client, nodePoolDaemonset, daemonset) - // Create the actual daemonSet pod with lower resource requests and expect to use the pod - daemonsetPod := test.UnschedulablePod( - test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps/v1", - Kind: "DaemonSet", - Name: daemonset.Name, - UID: daemonset.UID, - Controller: ptr.Bool(true), - BlockOwnerDeletion: ptr.Bool(true), - }, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: metav1.ObjectNameField, - Operator: v1.NodeSelectorOpIn, - Values: []string{"node-name"}, - }, - }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}}, - }) - ExpectApplied(ctx, env.Client, daemonsetPod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, daemonsetPod) - ExpectReconcileSucceeded(ctx, daemonsetController, client.ObjectKeyFromObject(daemonset)) - - //Deploy pod - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - NodeSelector: map[string]string{ - "foo": "voo", - }, - }) - ExpectApplied(ctx, env.Client, nodePool, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - }) - Context("Annotations", func() { - It("should annotate nodes", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - ObjectMeta: v1beta1.ObjectMeta{ - Annotations: map[string]string{v1beta1.DoNotDisruptAnnotationKey: "true"}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Annotations).To(HaveKeyWithValue(v1beta1.DoNotDisruptAnnotationKey, "true")) - }) - }) - Context("Labels", func() { - It("should label nodes", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - ObjectMeta: v1beta1.ObjectMeta{ - Labels: map[string]string{"test-key-1": "test-value-1"}, - }, - Spec: v1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ - {Key: "test-key-2", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value-2"}}, - {Key: "test-key-3", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value-3"}}, - {Key: "test-key-4", Operator: v1.NodeSelectorOpLt, Values: []string{"4"}}, - {Key: "test-key-5", Operator: v1.NodeSelectorOpGt, Values: []string{"5"}}, - {Key: "test-key-6", Operator: v1.NodeSelectorOpExists}, - {Key: "test-key-7", Operator: v1.NodeSelectorOpDoesNotExist}, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) - Expect(node.Labels).To(HaveKeyWithValue("test-key-1", "test-value-1")) - Expect(node.Labels).To(HaveKeyWithValue("test-key-2", "test-value-2")) - Expect(node.Labels).To(And(HaveKey("test-key-3"), Not(HaveValue(Equal("test-value-3"))))) - Expect(node.Labels).To(And(HaveKey("test-key-4"), Not(HaveValue(Equal("test-value-4"))))) - Expect(node.Labels).To(And(HaveKey("test-key-5"), Not(HaveValue(Equal("test-value-5"))))) - Expect(node.Labels).To(HaveKey("test-key-6")) - Expect(node.Labels).ToNot(HaveKey("test-key-7")) - }) - It("should label nodes with labels in the LabelDomainExceptions list", func() { - for domain := range v1beta1.LabelDomainExceptions { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - ObjectMeta: v1beta1.ObjectMeta{ - Labels: map[string]string{domain + "/test": "test-value"}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) - } - }) - - }) - Context("Taints", func() { - It("should schedule pods that tolerate taints", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Taints: []v1.Taint{{Key: "nvidia.com/gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{Tolerations: []v1.Toleration{ - { - Key: "nvidia.com/gpu", - Operator: v1.TolerationOpEqual, - Value: "true", - Effect: v1.TaintEffectNoSchedule, - }, - }}), - test.UnschedulablePod( - test.PodOptions{Tolerations: []v1.Toleration{ - { - Key: "nvidia.com/gpu", - Operator: v1.TolerationOpExists, - Effect: v1.TaintEffectNoSchedule, - }, - }}), - test.UnschedulablePod( - test.PodOptions{Tolerations: []v1.Toleration{ - { - Key: "nvidia.com/gpu", - Operator: v1.TolerationOpExists, - }, - }}), - test.UnschedulablePod( - test.PodOptions{Tolerations: []v1.Toleration{ - { - Operator: v1.TolerationOpExists, - }, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } - }) - }) - Context("NodeClaim Creation", func() { - It("should create a nodeclaim request with expected requirements", func() { - nodePool := test.NodePool() - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: lo.Keys(instanceTypeMap), - }, - v1.NodeSelectorRequirement{ - Key: v1beta1.NodePoolLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{nodePool.Name}, - }, - ) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a nodeclaim request with additional expected requirements", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: "custom-requirement-key", - Operator: v1.NodeSelectorOpIn, - Values: []string{"value"}, - }, - { - Key: "custom-requirement-key2", - Operator: v1.NodeSelectorOpIn, - Values: []string{"value"}, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: lo.Keys(instanceTypeMap), - }, - v1.NodeSelectorRequirement{ - Key: v1beta1.NodePoolLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{nodePool.Name}, - }, - v1.NodeSelectorRequirement{ - Key: "custom-requirement-key", - Operator: v1.NodeSelectorOpIn, - Values: []string{"value"}, - }, - v1.NodeSelectorRequirement{ - Key: "custom-requirement-key2", - Operator: v1.NodeSelectorOpIn, - Values: []string{"value"}, - }, - ) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a nodeclaim request restricting instance types on architecture", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"arm64"}, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - - // Expect a more restricted set of instance types - ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], - v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"arm64"}, - }, - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"arm-instance-type"}, - }, - ) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a nodeclaim request restricting instance types on operating system", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"ios"}, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - - // Expect a more restricted set of instance types - ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], - v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"ios"}, - }, - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"arm-instance-type"}, - }, - ) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a nodeclaim request restricting instance types based on pod resource requests", func() { - nodePool := test.NodePool() - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - Limits: v1.ResourceList{ - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - - // Expect a more restricted set of instance types - ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"gpu-vendor-instance-type"}, - }, - ) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a nodeclaim request with the correct owner reference", func() { - nodePool := test.NodePool() - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - Expect(cloudProvider.CreateCalls[0].OwnerReferences).To(ContainElement( - metav1.OwnerReference{ - APIVersion: "karpenter.sh/v1beta1", - Kind: "NodePool", - Name: nodePool.Name, - UID: nodePool.UID, - BlockOwnerDeletion: lo.ToPtr(true), - }, - )) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a nodeclaim request propagating the nodeClass reference", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - NodeClassRef: &v1beta1.NodeClassReference{ - APIVersion: "cloudprovider.karpenter.sh/v1beta1", - Kind: "CloudProvider", - Name: "default", - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - Expect(cloudProvider.CreateCalls[0].Spec.NodeClassRef).To(Equal( - &v1beta1.NodeClassReference{ - APIVersion: "cloudprovider.karpenter.sh/v1beta1", - Kind: "CloudProvider", - Name: "default", - }, - )) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a nodeclaim with resource requests", func() { - ExpectApplied(ctx, env.Client, test.NodePool()) - pod := test.UnschedulablePod( - test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Mi"), - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - Limits: v1.ResourceList{ - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - Expect(cloudProvider.CreateCalls[0].Spec.Resources.Requests).To(HaveLen(4)) - ExpectNodeClaimRequests(cloudProvider.CreateCalls[0], v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Mi"), - fake.ResourceGPUVendorA: resource.MustParse("1"), - v1.ResourcePods: resource.MustParse("1"), - }) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a nodeclaim with resource requests with daemon overhead", func() { - ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Mi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Mi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - ExpectNodeClaimRequests(cloudProvider.CreateCalls[0], v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("2Mi"), - v1.ResourcePods: resource.MustParse("2"), - }) - ExpectScheduled(ctx, env.Client, pod) - }) - }) - Context("Volume Topology Requirements", func() { - var storageClass *storagev1.StorageClass - BeforeEach(func() { - storageClass = test.StorageClass(test.StorageClassOptions{Zones: []string{"test-zone-2", "test-zone-3"}}) - }) - It("should not schedule if invalid pvc", func() { - ExpectApplied(ctx, env.Client, test.NodePool()) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{"invalid"}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule with an empty storage class", func() { - storageClass := "" - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass}) - ExpectApplied(ctx, env.Client, test.NodePool(), persistentVolumeClaim) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule valid pods when a pod with an invalid pvc is encountered (pvc)", func() { - ExpectApplied(ctx, env.Client, test.NodePool()) - invalidPod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{"invalid"}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, invalidPod) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, invalidPod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule valid pods when a pod with an invalid pvc is encountered (storage class)", func() { - invalidStorageClass := "invalid-storage-class" - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &invalidStorageClass}) - ExpectApplied(ctx, env.Client, test.NodePool(), persistentVolumeClaim) - invalidPod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, invalidPod) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, invalidPod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule valid pods when a pod with an invalid pvc is encountered (volume name)", func() { - invalidVolumeName := "invalid-volume-name" - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: invalidVolumeName}) - ExpectApplied(ctx, env.Client, test.NodePool(), persistentVolumeClaim) - invalidPod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, invalidPod) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, invalidPod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule to storage class zones if volume does not exist", func() { - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass.Name}) - ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}, - }}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should schedule to storage class zones if volume does not exist (ephemeral volume)", func() { - pod := test.UnschedulablePod(test.PodOptions{ - EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ - { - StorageClassName: &storageClass.Name, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}, - }}, - }) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), - }, - StorageClassName: &storageClass.Name, - }) - ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should not schedule if storage class zones are incompatible", func() { - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass.Name}) - ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, - }}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if storage class zones are incompatible (ephemeral volume)", func() { - pod := test.UnschedulablePod(test.PodOptions{ - EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ - { - StorageClassName: &storageClass.Name, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, - }}, - }) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), - }, - StorageClassName: &storageClass.Name, - }) - ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule to volume zones if volume already bound", func() { - persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) - ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim, persistentVolume) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should schedule to volume zones if volume already bound (ephemeral volume)", func() { - pod := test.UnschedulablePod(test.PodOptions{ - EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ - { - StorageClassName: &storageClass.Name, - }, - }, - }) - persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), - }, - VolumeName: persistentVolume.Name, - StorageClassName: &storageClass.Name, - }) - ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, pod, persistentVolumeClaim, persistentVolume) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should not schedule if volume zones are incompatible", func() { - persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) - ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim, persistentVolume) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, - }}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if volume zones are incompatible (ephemeral volume)", func() { - pod := test.UnschedulablePod(test.PodOptions{ - EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ - { - StorageClassName: &storageClass.Name, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, - }}, - }) - persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), - }, - VolumeName: persistentVolume.Name, - StorageClassName: &storageClass.Name, - }) - ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, pod, persistentVolumeClaim, persistentVolume) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not relax an added volume topology zone node-selector away", func() { - persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) - ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim, persistentVolume) - - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: "example.com/label", - Operator: v1.NodeSelectorOpIn, - Values: []string{"unsupported"}, - }, - }, - }) - - // Add the second capacity type that is OR'd with the first. Previously we only added the volume topology requirement - // to a single node selector term which would sometimes get relaxed away. Now we add it to all of them to AND - // it with each existing term. - pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, - v1.NodeSelectorTerm{ - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1beta1.CapacityTypeOnDemand}, - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - }) - Context("Preferential Fallback", func() { - Context("Required", func() { - It("should not relax the final term", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, // Should not be relaxed - }}, - }}}} - // Don't relax - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should relax multiple terms", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, // OR operator, never get to this one - }}, - }}}} - // Success - ExpectApplied(ctx, env.Client, test.NodePool()) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1")) - }) - }) - Context("Preferences", func() { - It("should relax all node affinity terms", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - }}} - // Success - ExpectApplied(ctx, env.Client, test.NodePool()) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should relax to use lighter weights", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 100, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, - }}, - }, - { - Weight: 50, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ // OR operator, never get to this one - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - }}, - }, - }}} - // Success - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should tolerate PreferNoSchedule taint only after trying to relax Affinity terms", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - }}} - // Success - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Spec.Taints).To(ContainElement(v1.Taint{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule})) - }) - }) - }) - Context("Multiple NodePools", func() { - It("should schedule to an explicitly selected NodePool", func() { - nodePool := test.NodePool() - ExpectApplied(ctx, env.Client, nodePool, test.NodePool()) - pod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: nodePool.Name}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1beta1.NodePoolLabelKey]).To(Equal(nodePool.Name)) - }) - It("should schedule to a NodePool by labels", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - ObjectMeta: v1beta1.ObjectMeta{ - Labels: map[string]string{"foo": "bar"}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool, test.NodePool()) - pod := test.UnschedulablePod(test.PodOptions{NodeSelector: nodePool.Spec.Template.Labels}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1beta1.NodePoolLabelKey]).To(Equal(nodePool.Name)) - }) - It("should not match NodePool with PreferNoSchedule taint when other NodePool match", func() { - nodePool := test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool, test.NodePool()) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1beta1.NodePoolLabelKey]).ToNot(Equal(nodePool.Name)) - }) - Context("Weighted nodePools", func() { - It("should schedule to the provisioner with the highest priority always", func() { - nodePools := []client.Object{ - test.NodePool(), - test.NodePool(v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Weight: ptr.Int32(20)}}), - test.NodePool(v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Weight: ptr.Int32(100)}}), - } - ExpectApplied(ctx, env.Client, nodePools...) - pods := []*v1.Pod{ - test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod(), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1beta1.NodePoolLabelKey]).To(Equal(nodePools[2].GetName())) - } - }) - It("should schedule to explicitly selected provisioner even if other nodePools are higher priority", func() { - targetedNodePool := test.NodePool() - nodePools := []client.Object{ - targetedNodePool, - test.NodePool(v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Weight: ptr.Int32(20)}}), - test.NodePool(v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Weight: ptr.Int32(100)}}), - } - ExpectApplied(ctx, env.Client, nodePools...) - pod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: targetedNodePool.Name}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1beta1.NodePoolLabelKey]).To(Equal(targetedNodePool.Name)) - }) - }) - }) -}) diff --git a/pkg/controllers/provisioning/provisioner_test.go b/pkg/controllers/provisioning/provisioner_test.go deleted file mode 100644 index a1a7d127a0..0000000000 --- a/pkg/controllers/provisioning/provisioner_test.go +++ /dev/null @@ -1,1401 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provisioning_test - -import ( - "encoding/json" - "fmt" - - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "knative.dev/pkg/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter-core/pkg/apis/v1beta1" - "github.com/aws/karpenter-core/pkg/cloudprovider/fake" - "github.com/aws/karpenter-core/pkg/test" - nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" - - . "github.com/aws/karpenter-core/pkg/test/expectations" -) - -var _ = Describe("Provisioner/Provisioning", func() { - It("should provision nodes", func() { - ExpectApplied(ctx, env.Client, test.Provisioner()) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - nodes := &v1.NodeList{} - Expect(env.Client.List(ctx, nodes)).To(Succeed()) - Expect(len(nodes.Items)).To(Equal(1)) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should continue with provisioning when at least a provisioner doesn't have resolved instance types", func() { - provNotDefined := test.Provisioner() - provNotDefined.Spec.ProviderRef = nil - ExpectApplied(ctx, env.Client, test.Provisioner(), provNotDefined) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - nodes := &v1.NodeList{} - Expect(env.Client.List(ctx, nodes)).To(Succeed()) - Expect(len(nodes.Items)).To(Equal(1)) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should ignore provisioners that are deleting", func() { - provisioner := test.Provisioner() - ExpectApplied(ctx, env.Client, provisioner) - ExpectDeletionTimestampSet(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - nodes := &v1.NodeList{} - Expect(env.Client.List(ctx, nodes)).To(Succeed()) - Expect(len(nodes.Items)).To(Equal(0)) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should provision nodes for pods with supported node selectors", func() { - provisioner := test.Provisioner() - schedulable := []*v1.Pod{ - // Constrained by provisioner - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: provisioner.Name}}), - // Constrained by zone - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), - // Constrained by instanceType - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), - // Constrained by architecture - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), - // Constrained by operatingSystem - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), - } - unschedulable := []*v1.Pod{ - // Ignored, matches another provisioner - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: "unknown"}}), - // Ignored, invalid zone - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}), - // Ignored, invalid instance type - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "unknown"}}), - // Ignored, invalid architecture - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "unknown"}}), - // Ignored, invalid operating system - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: "unknown"}}), - // Ignored, invalid capacity type - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.LabelCapacityType: "unknown"}}), - // Ignored, label selector does not match - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{"foo": "bar"}}), - } - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) - for _, pod := range schedulable { - ExpectScheduled(ctx, env.Client, pod) - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, unschedulable...) - for _, pod := range unschedulable { - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should provision nodes for pods with supported node affinities", func() { - provisioner := test.Provisioner() - schedulable := []*v1.Pod{ - // Constrained by provisioner - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1alpha5.ProvisionerNameLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{provisioner.Name}}}}), - // Constrained by zone - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}}}), - // Constrained by instanceType - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type"}}}}), - // Constrained by architecture - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{"arm64"}}}}), - // Constrained by operatingSystem - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{string(v1.Linux)}}}}), - } - unschedulable := []*v1.Pod{ - // Ignored, matches another provisioner - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1alpha5.ProvisionerNameLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, invalid zone - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, invalid instance type - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, invalid architecture - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, invalid operating system - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, invalid capacity type - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), - // Ignored, label selector does not match - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: "foo", Operator: v1.NodeSelectorOpIn, Values: []string{"bar"}}}}), - } - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) - for _, pod := range schedulable { - ExpectScheduled(ctx, env.Client, pod) - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, unschedulable...) - for _, pod := range unschedulable { - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should provision nodes for accelerators", func() { - ExpectApplied(ctx, env.Client, test.Provisioner()) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{fake.ResourceGPUVendorA: resource.MustParse("1")}}, - }), - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{fake.ResourceGPUVendorB: resource.MustParse("1")}}, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } - }) - It("should provision multiple nodes when maxPods is set", func() { - // Kubelet is actually not observed here, the scheduler is relying on the - // pods resource value which is statically set in the fake cloudprovider - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ - Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(1)}, - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"single-pod-instance-type"}, - }, - }, - })) - pods := []*v1.Pod{ - test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod(), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodes := &v1.NodeList{} - Expect(env.Client.List(ctx, nodes)).To(Succeed()) - Expect(len(nodes.Items)).To(Equal(3)) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } - }) - It("should schedule all pods on one inflight node while node is in deleting state", func() { - provisioner := test.Provisioner() - its, err := cloudProvider.GetInstanceTypes(ctx, nodepoolutil.New(provisioner)) - Expect(err).To(BeNil()) - node := test.Node(test.NodeOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1alpha5.ProvisionerNameLabelKey: provisioner.Name, - v1.LabelInstanceTypeStable: its[0].Name, - }, - Finalizers: []string{v1alpha5.TerminationFinalizer}, - }}, - ) - ExpectApplied(ctx, env.Client, node, provisioner) - ExpectReconcileSucceeded(ctx, nodeController, client.ObjectKeyFromObject(node)) - - // Schedule 3 pods to the node that currently exists - for i := 0; i < 3; i++ { - pod := test.UnschedulablePod() - ExpectApplied(ctx, env.Client, pod) - ExpectManualBinding(ctx, env.Client, pod, node) - } - - // Node shouldn't fully delete since it has a finalizer - Expect(env.Client.Delete(ctx, node)).To(Succeed()) - ExpectReconcileSucceeded(ctx, nodeController, client.ObjectKeyFromObject(node)) - - // Provision without a binding since some pods will already be bound - // Should all schedule to the new node, ignoring the old node - bindings := ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, test.UnschedulablePod(), test.UnschedulablePod()) - nodes := &v1.NodeList{} - Expect(env.Client.List(ctx, nodes)).To(Succeed()) - Expect(len(nodes.Items)).To(Equal(2)) - - // Scheduler should attempt to schedule all the pods to the new node - for _, n := range bindings { - Expect(n.Node.Name).ToNot(Equal(node.Name)) - } - }) - Context("Resource Limits", func() { - It("should not schedule when limits are exceeded", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("20")}, - Status: v1alpha5.ProvisionerStatus{ - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100"), - }, - }, - })) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule if limits would be met", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - })) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - // requires a 2 CPU node, but leaves room for overhead - v1.ResourceCPU: resource.MustParse("1.75"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // A 2 CPU node can be launched - ExpectScheduled(ctx, env.Client, pod) - }) - It("should partially schedule if limits would be exceeded", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3")}, - })) - - // prevent these pods from scheduling on the same node - opts := test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "foo"}, - }, - PodAntiRequirements: []v1.PodAffinityTerm{ - { - TopologyKey: v1.LabelHostname, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "foo", - }, - }, - }, - }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1.5"), - }}} - pods := []*v1.Pod{ - test.UnschedulablePod(opts), - test.UnschedulablePod(opts), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - scheduledPodCount := 0 - unscheduledPodCount := 0 - pod0 := ExpectPodExists(ctx, env.Client, pods[0].Name, pods[0].Namespace) - pod1 := ExpectPodExists(ctx, env.Client, pods[1].Name, pods[1].Namespace) - if pod0.Spec.NodeName == "" { - unscheduledPodCount++ - } else { - scheduledPodCount++ - } - if pod1.Spec.NodeName == "" { - unscheduledPodCount++ - } else { - scheduledPodCount++ - } - Expect(scheduledPodCount).To(Equal(1)) - Expect(unscheduledPodCount).To(Equal(1)) - }) - It("should not schedule if limits would be exceeded", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - })) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2.1"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if limits would be exceeded (GPU)", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ - Limits: v1.ResourceList{v1.ResourcePods: resource.MustParse("1")}, - })) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // only available instance type has 2 GPUs which would exceed the limit - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule to a provisioner after a scheduling round if limits would be exceeded", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - })) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - // requires a 2 CPU node, but leaves room for overhead - v1.ResourceCPU: resource.MustParse("1.75"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // A 2 CPU node can be launched - ExpectScheduled(ctx, env.Client, pod) - - // This pod requests over the existing limit (would add to 3.5 CPUs) so this should fail - pod = test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - // requires a 2 CPU node, but leaves room for overhead - v1.ResourceCPU: resource.MustParse("1.75"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - }) - Context("Daemonsets and Node Overhead", func() { - It("should account for overhead", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) - }) - It("should account for overhead (with startup taint)", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{ - StartupTaints: []v1.Taint{{Key: "foo.com/taint", Effect: v1.TaintEffectNoSchedule}}, - }) - - ExpectApplied(ctx, env.Client, provisioner, test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) - }) - It("should not schedule if overhead is too large", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}}, - }}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should account for overhead using daemonset pod spec instead of daemonset spec", func() { - provisioner := test.Provisioner() - // Create a daemonset with large resource requests - daemonset := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}}, - }}, - ) - ExpectApplied(ctx, env.Client, provisioner, daemonset) - // Create the actual daemonSet pod with lower resource requests and expect to use the pod - daemonsetPod := test.UnschedulablePod( - test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps/v1", - Kind: "DaemonSet", - Name: daemonset.Name, - UID: daemonset.UID, - Controller: ptr.Bool(true), - BlockOwnerDeletion: ptr.Bool(true), - }, - }, - }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }) - ExpectApplied(ctx, env.Client, provisioner, daemonsetPod) - ExpectReconcileSucceeded(ctx, daemonsetController, client.ObjectKeyFromObject(daemonset)) - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: provisioner.Name}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - - // We expect a smaller instance since the daemonset pod is smaller then daemonset spec - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) - }) - It("should not schedule if resource requests are not defined and limits (requests) are too large", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - }}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule based on the max resource requests of containers and initContainers", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("2Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - }}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) - }) - It("should not schedule if combined max resources are too large for any node", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("1Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - }}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if initContainer resources are too large", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, - }, - }}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should be able to schedule pods if resource requests and limits are not defined", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{}}, - )) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should ignore daemonsets without matching tolerations", func() { - ExpectApplied(ctx, env.Client, - test.Provisioner(test.ProvisionerOptions{Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoSchedule}}}), - test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - Tolerations: []v1.Toleration{{Operator: v1.TolerationOperator(v1.NodeSelectorOpExists)}}, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("2"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("2Gi"))) - }) - It("should ignore daemonsets with an invalid selector", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - NodeSelector: map[string]string{"node": "invalid"}, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("2"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("2Gi"))) - }) - It("should account daemonsets with NotIn operator and unspecified key", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{{Key: "foo", Operator: v1.NodeSelectorOpNotIn, Values: []string{"bar"}}}, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}}, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity - Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) - Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) - }) - It("should account for daemonset spec affinity", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{ - Labels: map[string]string{ - "foo": "voo", - }, - Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - }, - }) - provisionerDaemonset := test.Provisioner(test.ProvisionerOptions{ - Labels: map[string]string{ - "foo": "bar", - }, - }) - // Create a daemonset with large resource requests - daemonset := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: "foo", - Operator: v1.NodeSelectorOpIn, - Values: []string{"bar"}, - }, - }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}}, - }}, - ) - ExpectApplied(ctx, env.Client, provisionerDaemonset, daemonset) - // Create the actual daemonSet pod with lower resource requests and expect to use the pod - daemonsetPod := test.UnschedulablePod( - test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps/v1", - Kind: "DaemonSet", - Name: daemonset.Name, - UID: daemonset.UID, - Controller: ptr.Bool(true), - BlockOwnerDeletion: ptr.Bool(true), - }, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: metav1.ObjectNameField, - Operator: v1.NodeSelectorOpIn, - Values: []string{"node-name"}, - }, - }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}}, - }) - ExpectApplied(ctx, env.Client, daemonsetPod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, daemonsetPod) - ExpectReconcileSucceeded(ctx, daemonsetController, client.ObjectKeyFromObject(daemonset)) - - //Deploy pod - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - NodeSelector: map[string]string{ - "foo": "voo", - }, - }) - ExpectApplied(ctx, env.Client, provisioner, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - }) - Context("Annotations", func() { - It("should annotate nodes", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{ - Annotations: map[string]string{v1alpha5.DoNotConsolidateNodeAnnotationKey: "true"}, - }) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Annotations).To(HaveKeyWithValue(v1alpha5.DoNotConsolidateNodeAnnotationKey, "true")) - }) - }) - Context("Labels", func() { - It("should label nodes", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{ - Labels: map[string]string{"test-key-1": "test-value-1"}, - Requirements: []v1.NodeSelectorRequirement{ - {Key: "test-key-2", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value-2"}}, - {Key: "test-key-3", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value-3"}}, - {Key: "test-key-4", Operator: v1.NodeSelectorOpLt, Values: []string{"4"}}, - {Key: "test-key-5", Operator: v1.NodeSelectorOpGt, Values: []string{"5"}}, - {Key: "test-key-6", Operator: v1.NodeSelectorOpExists}, - {Key: "test-key-7", Operator: v1.NodeSelectorOpDoesNotExist}, - }, - }) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) - Expect(node.Labels).To(HaveKeyWithValue("test-key-1", "test-value-1")) - Expect(node.Labels).To(HaveKeyWithValue("test-key-2", "test-value-2")) - Expect(node.Labels).To(And(HaveKey("test-key-3"), Not(HaveValue(Equal("test-value-3"))))) - Expect(node.Labels).To(And(HaveKey("test-key-4"), Not(HaveValue(Equal("test-value-4"))))) - Expect(node.Labels).To(And(HaveKey("test-key-5"), Not(HaveValue(Equal("test-value-5"))))) - Expect(node.Labels).To(HaveKey("test-key-6")) - Expect(node.Labels).ToNot(HaveKey("test-key-7")) - }) - It("should label nodes with labels in the LabelDomainExceptions list", func() { - for domain := range v1alpha5.LabelDomainExceptions { - provisioner := test.Provisioner(test.ProvisionerOptions{Labels: map[string]string{domain + "/test": "test-value"}}) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) - } - }) - - }) - Context("Taints", func() { - It("should schedule pods that tolerate taints", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{Taints: []v1.Taint{{Key: "nvidia.com/gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}}) - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{Tolerations: []v1.Toleration{ - { - Key: "nvidia.com/gpu", - Operator: v1.TolerationOpEqual, - Value: "true", - Effect: v1.TaintEffectNoSchedule, - }, - }}), - test.UnschedulablePod( - test.PodOptions{Tolerations: []v1.Toleration{ - { - Key: "nvidia.com/gpu", - Operator: v1.TolerationOpExists, - Effect: v1.TaintEffectNoSchedule, - }, - }}), - test.UnschedulablePod( - test.PodOptions{Tolerations: []v1.Toleration{ - { - Key: "nvidia.com/gpu", - Operator: v1.TolerationOpExists, - }, - }}), - test.UnschedulablePod( - test.PodOptions{Tolerations: []v1.Toleration{ - { - Operator: v1.TolerationOpExists, - }, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } - }) - }) - Context("Machine Creation", func() { - It("should create a machine request with expected requirements", func() { - provisioner := test.Provisioner() - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: lo.Keys(instanceTypeMap), - }, - v1.NodeSelectorRequirement{ - Key: v1alpha5.ProvisionerNameLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{provisioner.Name}, - }, - ) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a machine request with additional expected requirements", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: "custom-requirement-key", - Operator: v1.NodeSelectorOpIn, - Values: []string{"value"}, - }, - { - Key: "custom-requirement-key2", - Operator: v1.NodeSelectorOpIn, - Values: []string{"value"}, - }, - }, - }) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: lo.Keys(instanceTypeMap), - }, - v1.NodeSelectorRequirement{ - Key: v1alpha5.ProvisionerNameLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{provisioner.Name}, - }, - v1.NodeSelectorRequirement{ - Key: "custom-requirement-key", - Operator: v1.NodeSelectorOpIn, - Values: []string{"value"}, - }, - v1.NodeSelectorRequirement{ - Key: "custom-requirement-key2", - Operator: v1.NodeSelectorOpIn, - Values: []string{"value"}, - }, - ) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a machine request restricting instance types on architecture", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"arm64"}, - }, - }, - }) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - - // Expect a more restricted set of instance types - ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], - v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"arm64"}, - }, - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"arm-instance-type"}, - }, - ) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a machine request restricting instance types on operating system", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"ios"}, - }, - }, - }) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - - // Expect a more restricted set of instance types - ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], - v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"ios"}, - }, - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"arm-instance-type"}, - }, - ) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a machine request restricting instance types based on pod resource requests", func() { - provisioner := test.Provisioner() - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - Limits: v1.ResourceList{ - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - - // Expect a more restricted set of instance types - ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"gpu-vendor-instance-type"}, - }, - ) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a machine request with the correct owner reference", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{}) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - Expect(cloudProvider.CreateCalls[0].OwnerReferences).To(ContainElement( - metav1.OwnerReference{ - APIVersion: "karpenter.sh/v1alpha5", - Kind: "Provisioner", - Name: provisioner.Name, - UID: provisioner.UID, - BlockOwnerDeletion: lo.ToPtr(true), - }, - )) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a machine request propagating the provider reference", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ - ProviderRef: &v1alpha5.MachineTemplateRef{ - APIVersion: "cloudprovider.karpenter.sh/v1alpha1", - Kind: "CloudProvider", - Name: "default", - }, - })) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - Expect(cloudProvider.CreateCalls[0].Spec.NodeClassRef).To(Equal( - &v1beta1.NodeClassReference{ - APIVersion: "cloudprovider.karpenter.sh/v1alpha1", - Kind: "CloudProvider", - Name: "default", - }, - )) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a machine request with the karpenter.sh/compatibility/provider annotation", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ - Provider: map[string]string{ - "providerField1": "value", - "providerField2": "value", - }, - })) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - Expect(cloudProvider.CreateCalls[0].Annotations).To(HaveKey(v1alpha5.ProviderCompatabilityAnnotationKey)) - - // Deserialize the provider into the expected format - provider := map[string]string{} - Expect(json.Unmarshal([]byte(cloudProvider.CreateCalls[0].Annotations[v1alpha5.ProviderCompatabilityAnnotationKey]), &provider)).To(Succeed()) - Expect(provider).To(HaveKeyWithValue("providerField1", "value")) - Expect(provider).To(HaveKeyWithValue("providerField2", "value")) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a machine with resource requests", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ - Provider: map[string]string{ - "providerField1": "value", - "providerField2": "value", - }, - })) - pod := test.UnschedulablePod( - test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Mi"), - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - Limits: v1.ResourceList{ - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - Expect(cloudProvider.CreateCalls[0].Spec.Resources.Requests).To(HaveLen(4)) - ExpectNodeClaimRequests(cloudProvider.CreateCalls[0], v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Mi"), - fake.ResourceGPUVendorA: resource.MustParse("1"), - v1.ResourcePods: resource.MustParse("1"), - }) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should create a machine with resource requests with daemon overhead", func() { - ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Mi")}}, - }}, - )) - pod := test.UnschedulablePod( - test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Mi")}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - Expect(cloudProvider.CreateCalls).To(HaveLen(1)) - ExpectNodeClaimRequests(cloudProvider.CreateCalls[0], v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("2Mi"), - v1.ResourcePods: resource.MustParse("2"), - }) - ExpectScheduled(ctx, env.Client, pod) - }) - }) - Context("Volume Topology Requirements", func() { - var storageClass *storagev1.StorageClass - BeforeEach(func() { - storageClass = test.StorageClass(test.StorageClassOptions{Zones: []string{"test-zone-2", "test-zone-3"}}) - }) - It("should not schedule if invalid pvc", func() { - ExpectApplied(ctx, env.Client, test.Provisioner()) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{"invalid"}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule with an empty storage class", func() { - storageClass := "" - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass}) - ExpectApplied(ctx, env.Client, test.Provisioner(), persistentVolumeClaim) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule valid pods when a pod with an invalid pvc is encountered (pvc)", func() { - ExpectApplied(ctx, env.Client, test.Provisioner()) - invalidPod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{"invalid"}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, invalidPod) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, invalidPod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule valid pods when a pod with an invalid pvc is encountered (storage class)", func() { - invalidStorageClass := "invalid-storage-class" - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &invalidStorageClass}) - ExpectApplied(ctx, env.Client, test.Provisioner(), persistentVolumeClaim) - invalidPod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, invalidPod) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, invalidPod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule valid pods when a pod with an invalid pvc is encountered (volume name)", func() { - invalidVolumeName := "invalid-volume-name" - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: invalidVolumeName}) - ExpectApplied(ctx, env.Client, test.Provisioner(), persistentVolumeClaim) - invalidPod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, invalidPod) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, invalidPod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule to storage class zones if volume does not exist", func() { - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass.Name}) - ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}, - }}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should schedule to storage class zones if volume does not exist (ephemeral volume)", func() { - pod := test.UnschedulablePod(test.PodOptions{ - EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ - { - StorageClassName: &storageClass.Name, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}, - }}, - }) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), - }, - StorageClassName: &storageClass.Name, - }) - ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should not schedule if storage class zones are incompatible", func() { - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass.Name}) - ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, - }}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if storage class zones are incompatible (ephemeral volume)", func() { - pod := test.UnschedulablePod(test.PodOptions{ - EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ - { - StorageClassName: &storageClass.Name, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, - }}, - }) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), - }, - StorageClassName: &storageClass.Name, - }) - ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule to volume zones if volume already bound", func() { - persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) - ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim, persistentVolume) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should schedule to volume zones if volume already bound (ephemeral volume)", func() { - pod := test.UnschedulablePod(test.PodOptions{ - EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ - { - StorageClassName: &storageClass.Name, - }, - }, - }) - persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), - }, - VolumeName: persistentVolume.Name, - StorageClassName: &storageClass.Name, - }) - ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, pod, persistentVolumeClaim, persistentVolume) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should not schedule if volume zones are incompatible", func() { - persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) - ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim, persistentVolume) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, - }}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if volume zones are incompatible (ephemeral volume)", func() { - pod := test.UnschedulablePod(test.PodOptions{ - EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ - { - StorageClassName: &storageClass.Name, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, - }}, - }) - persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), - }, - VolumeName: persistentVolume.Name, - StorageClassName: &storageClass.Name, - }) - ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, pod, persistentVolumeClaim, persistentVolume) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not relax an added volume topology zone node-selector away", func() { - persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) - persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) - ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim, persistentVolume) - - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: "example.com/label", - Operator: v1.NodeSelectorOpIn, - Values: []string{"unsupported"}, - }, - }, - }) - - // Add the second capacity type that is OR'd with the first. Previously we only added the volume topology requirement - // to a single node selector term which would sometimes get relaxed away. Now we add it to all of them to AND - // it with each existing term. - pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, - v1.NodeSelectorTerm{ - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeOnDemand}, - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - }) - Context("Preferential Fallback", func() { - Context("Required", func() { - It("should not relax the final term", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, // Should not be relaxed - }}, - }}}} - // Don't relax - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}}})) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should relax multiple terms", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, // OR operator, never get to this one - }}, - }}}} - // Success - ExpectApplied(ctx, env.Client, test.Provisioner()) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1")) - }) - }) - Context("Preferences", func() { - It("should relax all node affinity terms", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - }}} - // Success - ExpectApplied(ctx, env.Client, test.Provisioner()) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should relax to use lighter weights", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 100, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, - }}, - }, - { - Weight: 50, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ // OR operator, never get to this one - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - }}, - }, - }}} - // Success - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}}})) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should tolerate PreferNoSchedule taint only after trying to relax Affinity terms", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - }}} - // Success - ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}}})) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Spec.Taints).To(ContainElement(v1.Taint{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule})) - }) - }) - }) - Context("Multiple Provisioners", func() { - It("should schedule to an explicitly selected provisioner", func() { - provisioner := test.Provisioner() - ExpectApplied(ctx, env.Client, provisioner, test.Provisioner()) - pod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: provisioner.Name}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner.Name)) - }) - It("should schedule to a provisioner by labels", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{Labels: map[string]string{"foo": "bar"}}) - ExpectApplied(ctx, env.Client, provisioner, test.Provisioner()) - pod := test.UnschedulablePod(test.PodOptions{NodeSelector: provisioner.Spec.Labels}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner.Name)) - }) - It("should not match provisioner with PreferNoSchedule taint when other provisioner match", func() { - provisioner := test.Provisioner(test.ProvisionerOptions{Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}}}) - ExpectApplied(ctx, env.Client, provisioner, test.Provisioner()) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).ToNot(Equal(provisioner.Name)) - }) - Context("Weighted Provisioners", func() { - It("should schedule to the provisioner with the highest priority always", func() { - provisioners := []client.Object{ - test.Provisioner(), - test.Provisioner(test.ProvisionerOptions{Weight: ptr.Int32(20)}), - test.Provisioner(test.ProvisionerOptions{Weight: ptr.Int32(100)}), - } - ExpectApplied(ctx, env.Client, provisioners...) - pods := []*v1.Pod{ - test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod(), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioners[2].GetName())) - } - }) - It("should schedule to explicitly selected provisioner even if other provisioners are higher priority", func() { - targetedProvisioner := test.Provisioner() - provisioners := []client.Object{ - targetedProvisioner, - test.Provisioner(test.ProvisionerOptions{Weight: ptr.Int32(20)}), - test.Provisioner(test.ProvisionerOptions{Weight: ptr.Int32(100)}), - } - ExpectApplied(ctx, env.Client, provisioners...) - pod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: targetedProvisioner.Name}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(targetedProvisioner.Name)) - }) - }) - }) -}) diff --git a/pkg/controllers/provisioning/scheduling/nodepool_instance_selection_test.go b/pkg/controllers/provisioning/scheduling/instance_selection_test.go similarity index 100% rename from pkg/controllers/provisioning/scheduling/nodepool_instance_selection_test.go rename to pkg/controllers/provisioning/scheduling/instance_selection_test.go diff --git a/pkg/controllers/provisioning/scheduling/nodepool_test.go b/pkg/controllers/provisioning/scheduling/nodepool_test.go deleted file mode 100644 index 71fad2846f..0000000000 --- a/pkg/controllers/provisioning/scheduling/nodepool_test.go +++ /dev/null @@ -1,3180 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheduling_test - -import ( - "fmt" - "math/rand" //nolint:gosec - "time" - - "github.com/samber/lo" - nodev1 "k8s.io/api/node/v1" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - cloudproviderapi "k8s.io/cloud-provider/api" - "k8s.io/csi-translation-lib/plugins" - "knative.dev/pkg/ptr" - - v1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/aws/karpenter-core/pkg/apis/v1beta1" - "github.com/aws/karpenter-core/pkg/cloudprovider" - "github.com/aws/karpenter-core/pkg/cloudprovider/fake" - "github.com/aws/karpenter-core/pkg/controllers/state" - pscheduling "github.com/aws/karpenter-core/pkg/scheduling" - "github.com/aws/karpenter-core/pkg/test" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "github.com/aws/karpenter-core/pkg/test/expectations" -) - -var _ = Context("NodePool", func() { - var nodePool *v1beta1.NodePool - BeforeEach(func() { - nodePool = test.NodePool(v1beta1.NodePool{ - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1beta1.CapacityTypeSpot, v1beta1.CapacityTypeOnDemand}, - }, - }, - }, - }, - }, - }) - }) - - Describe("Custom Constraints", func() { - Context("NodePool with Labels", func() { - It("should schedule unconstrained pods that don't have matching node selectors", func() { - nodePool.Spec.Template.Labels = map[string]string{"test-key": "test-value"} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have conflicting node selectors", func() { - nodePool.Spec.Template.Labels = map[string]string{"test-key": "test-value"} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{"test-key": "different-value"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule pods that have node selectors with undefined key", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{"test-key": "test-value"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that have matching requirements", func() { - nodePool.Spec.Template.Labels = map[string]string{"test-key": "test-value"} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have conflicting requirements", func() { - nodePool.Spec.Template.Labels = map[string]string{"test-key": "test-value"} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - }) - Context("Well Known Labels", func() { - It("should use NodePool constraints", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should use node selectors", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should not schedule nodes with a hostname selector", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelHostname: "red-node"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule the pod if nodeselector unknown", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if node selector outside of NodePool constraints", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should schedule compatible requirements with Operator=Gt", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpGt, Values: []string{"8"}, - }} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "16")) - }) - It("should schedule compatible requirements with Operator=Lt", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpLt, Values: []string{"8"}, - }} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "2")) - }) - It("should not schedule incompatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should not schedule incompatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "unknown"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should schedule incompatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should schedule incompatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible node selectors, preferences and requirements", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}, - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should combine multidimensional node selectors, preferences and requirements", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeSelector: map[string]string{ - v1.LabelTopologyZone: "test-zone-3", - v1.LabelInstanceTypeStable: "arm-instance-type", - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type", "arm-instance-type"}}, - }, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, - }, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "arm-instance-type")) - }) - }) - Context("Constraints Validation", func() { - It("should not schedule pods that have node selectors with restricted labels", func() { - ExpectApplied(ctx, env.Client, nodePool) - for label := range v1beta1.RestrictedLabels { - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should not schedule pods that have node selectors with restricted domains", func() { - ExpectApplied(ctx, env.Client, nodePool) - for domain := range v1beta1.RestrictedLabelDomains { - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should schedule pods that have node selectors with label in restricted domains exceptions list", func() { - var requirements []v1.NodeSelectorRequirement - for domain := range v1beta1.LabelDomainExceptions { - requirements = append(requirements, v1.NodeSelectorRequirement{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}) - } - nodePool.Spec.Template.Spec.Requirements = requirements - ExpectApplied(ctx, env.Client, nodePool) - for domain := range v1beta1.LabelDomainExceptions { - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) - } - }) - It("should schedule pods that have node selectors with label in wellknown label list", func() { - schedulable := []*v1.Pod{ - // Constrained by zone - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), - // Constrained by instanceType - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), - // Constrained by architecture - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), - // Constrained by operatingSystem - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), - // Constrained by capacity type - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.CapacityTypeLabelKey: "spot"}}), - } - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) - for _, pod := range schedulable { - ExpectScheduled(ctx, env.Client, pod) - } - }) - }) - Context("Scheduling Logic", func() { - It("should not schedule pods that have node selectors with In operator and undefined key", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that have node selectors with NotIn operator and undefined key", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).ToNot(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have node selectors with Exists operator and undefined key", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpExists}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that with DoesNotExists operator and undefined key", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).ToNot(HaveKey("test-key")) - }) - It("should schedule unconstrained pods that don't have matching node selectors", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should schedule pods that have node selectors with matching value and In operator", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have node selectors with matching value and NotIn operator", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule the pod with Exists operator and defined key", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpExists}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should not schedule the pod with DoesNotExists operator and defined key", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule pods that have node selectors with different value and In operator", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that have node selectors with different value and NotIn operator", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should schedule compatible pods to the same node", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}), - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - node1 := ExpectScheduled(ctx, env.Client, pods[0]) - node2 := ExpectScheduled(ctx, env.Client, pods[1]) - Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node2.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should schedule incompatible pods to the different node", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}), - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - node1 := ExpectScheduled(ctx, env.Client, pods[0]) - node2 := ExpectScheduled(ctx, env.Client, pods[1]) - Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node2.Labels).To(HaveKeyWithValue("test-key", "another-value")) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("Exists operator should not overwrite the existing value", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"non-existent-zone"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - }) - Context("Well Known Labels", func() { - It("should use NodePool constraints", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should use node selectors", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should not schedule nodes with a hostname selector", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelHostname: "red-node"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule the pod if nodeselector unknown", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if node selector outside of NodePool constraints", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should schedule compatible requirements with Operator=Gt", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpGt, Values: []string{"8"}, - }} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "16")) - }) - It("should schedule compatible requirements with Operator=Lt", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpLt, Values: []string{"8"}, - }} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "2")) - }) - It("should not schedule incompatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should not schedule incompatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "unknown"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should schedule incompatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should schedule incompatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible node selectors, preferences and requirements", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}, - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should combine multidimensional node selectors, preferences and requirements", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeSelector: map[string]string{ - v1.LabelTopologyZone: "test-zone-3", - v1.LabelInstanceTypeStable: "arm-instance-type", - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type", "arm-instance-type"}}, - }, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, - }, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "arm-instance-type")) - }) - }) - Context("Constraints Validation", func() { - It("should not schedule pods that have node selectors with restricted labels", func() { - ExpectApplied(ctx, env.Client, nodePool) - for label := range v1beta1.RestrictedLabels { - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should not schedule pods that have node selectors with restricted domains", func() { - ExpectApplied(ctx, env.Client, nodePool) - for domain := range v1beta1.RestrictedLabelDomains { - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should schedule pods that have node selectors with label in restricted domains exceptions list", func() { - var requirements []v1.NodeSelectorRequirement - for domain := range v1beta1.LabelDomainExceptions { - requirements = append(requirements, v1.NodeSelectorRequirement{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}) - } - nodePool.Spec.Template.Spec.Requirements = requirements - ExpectApplied(ctx, env.Client, nodePool) - for domain := range v1beta1.LabelDomainExceptions { - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) - } - }) - It("should schedule pods that have node selectors with label in wellknown label list", func() { - schedulable := []*v1.Pod{ - // Constrained by zone - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), - // Constrained by instanceType - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), - // Constrained by architecture - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), - // Constrained by operatingSystem - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), - // Constrained by capacity type - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.CapacityTypeLabelKey: "spot"}}), - } - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) - for _, pod := range schedulable { - ExpectScheduled(ctx, env.Client, pod) - } - }) - }) - Context("Scheduling Logic", func() { - It("should not schedule pods that have node selectors with In operator and undefined key", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that have node selectors with NotIn operator and undefined key", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).ToNot(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have node selectors with Exists operator and undefined key", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpExists}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that with DoesNotExists operator and undefined key", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).ToNot(HaveKey("test-key")) - }) - It("should schedule unconstrained pods that don't have matching node selectors", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should schedule pods that have node selectors with matching value and In operator", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have node selectors with matching value and NotIn operator", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule the pod with Exists operator and defined key", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpExists}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should not schedule the pod with DoesNotExists operator and defined key", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule pods that have node selectors with different value and In operator", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that have node selectors with different value and NotIn operator", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should schedule compatible pods to the same node", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}), - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - node1 := ExpectScheduled(ctx, env.Client, pods[0]) - node2 := ExpectScheduled(ctx, env.Client, pods[1]) - Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node2.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should schedule incompatible pods to the different node", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}), - test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - node1 := ExpectScheduled(ctx, env.Client, pods[0]) - node2 := ExpectScheduled(ctx, env.Client, pods[1]) - Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node2.Labels).To(HaveKeyWithValue("test-key", "another-value")) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("Exists operator should not overwrite the existing value", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"non-existent-zone"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - }) - }) - - Describe("Preferential Fallback", func() { - Context("Required", func() { - It("should not relax the final term", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type"}}, - } - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, // Should not be relaxed - }}, - }}}} - // Don't relax - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should relax multiple terms", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, // OR operator, never get to this one - }}, - }}}} - // Success - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1")) - }) - }) - Context("Preferred", func() { - It("should relax all terms", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - }}} - // Success - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should relax to use lighter weights", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 100, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, - }}, - }, - { - Weight: 50, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ // OR operator, never get to this one - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - }}, - }, - }}} - // Success - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should schedule even if preference is conflicting with requirement", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-3"}}, - }}, - }, - }, - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, // Should not be relaxed - }}, - }}, - }} - // Success - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should schedule even if preference requirements are conflicting", func() { - pod := test.UnschedulablePod(test.PodOptions{NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"invalid"}}, - }}) - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - }) - }) - - Describe("Instance Type Compatibility", func() { - It("should not schedule if requesting more resources than any instance type has", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("512"), - }}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should launch pods with different archs on different instances", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1beta1.ArchitectureArm64, v1beta1.ArchitectureAmd64}, - }} - nodeNames := sets.NewString() - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: v1beta1.ArchitectureAmd64}, - }), - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: v1beta1.ArchitectureArm64}, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodeNames.Insert(node.Name) - } - Expect(nodeNames.Len()).To(Equal(2)) - }) - It("should exclude instance types that are not supported by the pod constraints (node affinity/instance type)", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1beta1.ArchitectureAmd64}, - }} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"arm-instance-type"}, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // arm instance type conflicts with the nodePool limitation of AMD only - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should exclude instance types that are not supported by the pod constraints (node affinity/operating system)", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1beta1.ArchitectureAmd64}, - }} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"ios"}, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // there's an instance with an OS of ios, but it has an arm processor so the provider requirements will - // exclude it - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should exclude instance types that are not supported by the provider constraints (arch)", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1beta1.ArchitectureAmd64}, - }} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: resource.MustParse("14")}}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // only the ARM instance has enough CPU, but it's not allowed per the nodePool - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should launch pods with different operating systems on different instances", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1beta1.ArchitectureArm64, v1beta1.ArchitectureAmd64}, - }} - nodeNames := sets.NewString() - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}, - }), - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Windows)}, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodeNames.Insert(node.Name) - } - Expect(nodeNames.Len()).To(Equal(2)) - }) - It("should launch pods with different instance type node selectors on different instances", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1beta1.ArchitectureArm64, v1beta1.ArchitectureAmd64}, - }} - nodeNames := sets.NewString() - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelInstanceType: "small-instance-type"}, - }), - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodeNames.Insert(node.Name) - } - Expect(nodeNames.Len()).To(Equal(2)) - }) - It("should launch pods with different zone selectors on different instances", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1beta1.ArchitectureArm64, v1beta1.ArchitectureAmd64}, - }} - nodeNames := sets.NewString() - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}, - }), - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodeNames.Insert(node.Name) - } - Expect(nodeNames.Len()).To(Equal(2)) - }) - It("should launch pods with resources that aren't on any single instance type on different instances", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - const fakeGPU1 = "karpenter.sh/super-great-gpu" - const fakeGPU2 = "karpenter.sh/even-better-gpu" - cloudProvider.InstanceTypes[0].Capacity[fakeGPU1] = resource.MustParse("25") - cloudProvider.InstanceTypes[1].Capacity[fakeGPU2] = resource.MustParse("25") - - nodeNames := sets.NewString() - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{fakeGPU1: resource.MustParse("1")}, - }, - }), - // Should pack onto a different instance since no instance type has both GPUs - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{fakeGPU2: resource.MustParse("1")}, - }, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodeNames.Insert(node.Name) - } - Expect(nodeNames.Len()).To(Equal(2)) - }) - It("should fail to schedule a pod with resources requests that aren't on a single instance type", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - const fakeGPU1 = "karpenter.sh/super-great-gpu" - const fakeGPU2 = "karpenter.sh/even-better-gpu" - cloudProvider.InstanceTypes[0].Capacity[fakeGPU1] = resource.MustParse("25") - cloudProvider.InstanceTypes[1].Capacity[fakeGPU2] = resource.MustParse("25") - - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - fakeGPU1: resource.MustParse("1"), - fakeGPU2: resource.MustParse("1")}, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - Context("Provider Specific Labels", func() { - It("should filter instance types that match labels", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{fake.LabelInstanceSize: "large"}}), - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{fake.LabelInstanceSize: "small"}}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - node := ExpectScheduled(ctx, env.Client, pods[0]) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "fake-it-4")) - node = ExpectScheduled(ctx, env.Client, pods[1]) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "fake-it-0")) - }) - It("should not schedule with incompatible labels", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{ - fake.LabelInstanceSize: "large", - v1.LabelInstanceTypeStable: cloudProvider.InstanceTypes[0].Name, - }}), - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{ - fake.LabelInstanceSize: "small", - v1.LabelInstanceTypeStable: cloudProvider.InstanceTypes[4].Name, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - ExpectNotScheduled(ctx, env.Client, pods[0]) - ExpectNotScheduled(ctx, env.Client, pods[1]) - }) - It("should schedule optional labels", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // Only some instance types have this key - {Key: fake.ExoticInstanceLabelKey, Operator: v1.NodeSelectorOpExists}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKey(fake.ExoticInstanceLabelKey)) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, cloudProvider.InstanceTypes[4].Name)) - }) - It("should schedule without optional labels if disallowed", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - ExpectApplied(ctx, env.Client, test.NodePool()) - pod := test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // Only some instance types have this key - {Key: fake.ExoticInstanceLabelKey, Operator: v1.NodeSelectorOpDoesNotExist}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).ToNot(HaveKey(fake.ExoticInstanceLabelKey)) - }) - }) - }) - - Describe("Binpacking", func() { - It("should schedule a small pod on the smallest instance", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("100M"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) - }) - It("should schedule a small pod on the smallest possible instance type", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("2000M"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) - }) - It("should take pod runtime class into consideration", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1"), - }, - }}) - // the pod has overhead of 2 CPUs - runtimeClass := &nodev1.RuntimeClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-runtime-class", - }, - Handler: "default", - Overhead: &nodev1.Overhead{ - PodFixed: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - }, - }, - } - pod.Spec.RuntimeClassName = &runtimeClass.Name - ExpectApplied(ctx, env.Client, runtimeClass) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - // overhead of 2 + request of 1 = at least 3 CPUs, so it won't fit on small-instance-type which it otherwise - // would - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) - }) - It("should schedule multiple small pods on the smallest possible instance type", func() { - opts := test.PodOptions{ - Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("10M"), - }, - }} - pods := test.Pods(5, opts) - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := sets.NewString() - for _, p := range pods { - node := ExpectScheduled(ctx, env.Client, p) - nodeNames.Insert(node.Name) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) - } - Expect(nodeNames).To(HaveLen(1)) - }) - It("should create new nodes when a node is at capacity", func() { - opts := test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, - Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1.8G"), - }, - }} - ExpectApplied(ctx, env.Client, nodePool) - pods := test.Pods(40, opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := sets.NewString() - for _, p := range pods { - node := ExpectScheduled(ctx, env.Client, p) - nodeNames.Insert(node.Name) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) - } - Expect(nodeNames).To(HaveLen(20)) - }) - It("should pack small and large pods together", func() { - largeOpts := test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, - Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1.8G"), - }, - }} - smallOpts := test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, - Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("400M"), - }, - }} - - // Two large pods are all that will fit on the default-instance type (the largest instance type) which will create - // twenty nodes. This leaves just enough room on each of those nodes for one additional small pod per node, so we - // should only end up with 20 nodes total. - provPods := append(test.Pods(40, largeOpts), test.Pods(20, smallOpts)...) - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, provPods...) - nodeNames := sets.NewString() - for _, p := range provPods { - node := ExpectScheduled(ctx, env.Client, p) - nodeNames.Insert(node.Name) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) - } - Expect(nodeNames).To(HaveLen(20)) - }) - It("should pack nodes tightly", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - var nodes []*v1.Node - ExpectApplied(ctx, env.Client, nodePool) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.5")}, - }, - }), - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodes = append(nodes, node) - } - Expect(nodes).To(HaveLen(2)) - // the first pod consumes nearly all CPU of the largest instance type with no room for the second pod, the - // second pod is much smaller in terms of resources and should get a smaller node - Expect(nodes[0].Labels[v1.LabelInstanceTypeStable]).ToNot(Equal(nodes[1].Labels[v1.LabelInstanceTypeStable])) - }) - It("should handle zero-quantity resource requests", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")}, - Limits: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")}, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // requesting a resource of quantity zero of a type unsupported by any instance is fine - ExpectScheduled(ctx, env.Client, pod) - }) - It("should not schedule pods that exceed every instance type's capacity", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("2Ti"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should create new nodes when a node is at capacity due to pod limits per node", func() { - opts := test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, - Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1m"), - v1.ResourceCPU: resource.MustParse("1m"), - }, - }} - ExpectApplied(ctx, env.Client, nodePool) - pods := test.Pods(25, opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := sets.NewString() - // all of the test instance types support 5 pods each, so we use the 5 instances of the smallest one for our 25 pods - for _, p := range pods { - node := ExpectScheduled(ctx, env.Client, p) - nodeNames.Insert(node.Name) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) - } - Expect(nodeNames).To(HaveLen(5)) - }) - It("should take into account initContainer resource requests when binpacking", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourceCPU: resource.MustParse("1"), - }, - }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourceCPU: resource.MustParse("2"), - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) - }) - It("should not schedule pods when initContainer resource requests are greater than available instance types", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourceCPU: resource.MustParse("1"), - }, - }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1Ti"), - v1.ResourceCPU: resource.MustParse("2"), - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should select for valid instance types, regardless of price", func() { - // capacity sizes and prices don't correlate here, regardless we should filter and see that all three instance types - // are valid before preferring the cheapest one 'large' - cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "medium", - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("2Gi"), - }, - Offerings: []cloudprovider.Offering{ - { - CapacityType: v1beta1.CapacityTypeOnDemand, - Zone: "test-zone-1a", - Price: 3.00, - Available: true, - }, - }, - }), - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "small", - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi"), - }, - Offerings: []cloudprovider.Offering{ - { - CapacityType: v1beta1.CapacityTypeOnDemand, - Zone: "test-zone-1a", - Price: 2.00, - Available: true, - }, - }, - }), - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "large", - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("4"), - v1.ResourceMemory: resource.MustParse("4Gi"), - }, - Offerings: []cloudprovider.Offering{ - { - CapacityType: v1beta1.CapacityTypeOnDemand, - Zone: "test-zone-1a", - Price: 1.00, - Available: true, - }, - }, - }), - } - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1m"), - v1.ResourceMemory: resource.MustParse("1Mi"), - }, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - // large is the cheapest, so we should pick it, but the other two types are also valid options - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("large")) - // all three options should be passed to the cloud provider - possibleInstanceType := sets.NewString(pscheduling.NewNodeSelectorRequirements(cloudProvider.CreateCalls[0].Spec.Requirements...).Get(v1.LabelInstanceTypeStable).Values()...) - Expect(possibleInstanceType).To(Equal(sets.NewString("small", "medium", "large"))) - }) - }) - - Describe("In-Flight Nodes", func() { - It("should not launch a second node if there is an in-flight node that can support the pod", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - secondPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should not launch a second node if there is an in-flight node that can support the pod (node selectors)", func() { - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2"}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - // the node gets created in test-zone-2 - secondPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1", "test-zone-2"}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - // test-zone-2 is in the intersection of their node selectors and the node has capacity, so we shouldn't create a new node - node2 := ExpectScheduled(ctx, env.Client, secondPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - Expect(node1.Name).To(Equal(node2.Name)) - - // the node gets created in test-zone-2 - thirdPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1", "test-zone-3"}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, thirdPod) - // node is in test-zone-2, so this pod needs a new node - node3 := ExpectScheduled(ctx, env.Client, thirdPod) - Expect(node1.Name).ToNot(Equal(node3.Name)) - }) - It("should launch a second node if a pod won't fit on the existingNodes node", func() { - ExpectApplied(ctx, env.Client, nodePool) - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1001m"), - }, - }} - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - // the node will have 2000m CPU, so these two pods can't both fit on it - opts.ResourceRequirements.Limits[v1.ResourceCPU] = resource.MustParse("1") - secondPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("should launch a second node if a pod isn't compatible with the existingNodes node (node selector)", func() { - ExpectApplied(ctx, env.Client, nodePool) - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - secondPod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("should launch a second node if an in-flight node is terminating", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod(opts) - bindings := ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - ExpectScheduled(ctx, env.Client, initialPod) - - // delete the node/nodeclaim - nodeClaim1 := bindings.Get(initialPod).NodeClaim - node1 := bindings.Get(initialPod).Node - nodeClaim1.Finalizers = nil - node1.Finalizers = nil - ExpectApplied(ctx, env.Client, nodeClaim1, node1) - ExpectDeleted(ctx, env.Client, nodeClaim1, node1) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - ExpectReconcileSucceeded(ctx, nodeClaimStateController, client.ObjectKeyFromObject(nodeClaim1)) - - secondPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - Context("Topology", func() { - It("should balance pods across zones with in-flight nodes", func() { - labels := map[string]string{"foo": "bar"} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) - - // reconcile our nodes with the cluster state so they'll show up as in-flight - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - for _, node := range nodeList.Items { - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKey{Name: node.Name}) - } - - firstRoundNumNodes := len(nodeList.Items) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 5)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 3)) - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - - // shouldn't create any new nodes as the in-flight ones can support the pods - Expect(nodeList.Items).To(HaveLen(firstRoundNumNodes)) - }) - It("should balance pods across hostnames with in-flight nodes", func() { - labels := map[string]string{"foo": "bar"} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1)) - - // reconcile our nodes with the cluster state so they'll show up as in-flight - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - for _, node := range nodeList.Items { - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKey{Name: node.Name}) - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 5)..., - ) - // we prefer to launch new nodes to satisfy the topology spread even though we could technically schedule against existingNodes - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1, 1, 1, 1, 1, 1)) - }) - }) - Context("Taints", func() { - It("should assume pod will schedule to a tainted node with no taints", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("8"), - }, - }} - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - node1.Spec.Taints = nil - ExpectApplied(ctx, env.Client, node1) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - secondPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should not assume pod will schedule to a tainted node", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("8"), - }, - }} - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod(opts) - bindings := ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - ExpectScheduled(ctx, env.Client, initialPod) - - nodeClaim1 := bindings.Get(initialPod).NodeClaim - node1 := bindings.Get(initialPod).Node - nodeClaim1.StatusConditions().MarkTrue(v1beta1.Initialized) - node1.Labels = lo.Assign(node1.Labels, map[string]string{v1beta1.NodeInitializedLabelKey: "true"}) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - // and taint it - node1.Spec.Taints = append(node1.Spec.Taints, v1.Taint{ - Key: "foo.com/taint", - Value: "tainted", - Effect: v1.TaintEffectNoSchedule, - }) - ExpectApplied(ctx, env.Client, nodeClaim1, node1) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - secondPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("should assume pod will schedule to a tainted node with a custom startup taint", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("8"), - }, - }} - nodePool.Spec.Template.Spec.StartupTaints = append(nodePool.Spec.Template.Spec.StartupTaints, v1.Taint{ - Key: "foo.com/taint", - Value: "tainted", - Effect: v1.TaintEffectNoSchedule, - }) - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - // startup taint + node not ready taint = 2 - Expect(node1.Spec.Taints).To(HaveLen(2)) - Expect(node1.Spec.Taints).To(ContainElement(v1.Taint{ - Key: "foo.com/taint", - Value: "tainted", - Effect: v1.TaintEffectNoSchedule, - })) - ExpectApplied(ctx, env.Client, node1) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - secondPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should not assume pod will schedule to a node with startup taints after initialization", func() { - startupTaint := v1.Taint{Key: "ignore-me", Value: "nothing-to-see-here", Effect: v1.TaintEffectNoSchedule} - nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{startupTaint} - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod() - bindings := ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - ExpectScheduled(ctx, env.Client, initialPod) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - - // Mark it initialized which only occurs once the startup taint was removed and re-apply only the startup taint. - // We also need to add resource capacity as after initialization we assume that kubelet has recorded them. - - nodeClaim1 := bindings.Get(initialPod).NodeClaim - node1 := bindings.Get(initialPod).Node - nodeClaim1.StatusConditions().MarkTrue(v1beta1.Initialized) - node1.Labels = lo.Assign(node1.Labels, map[string]string{v1beta1.NodeInitializedLabelKey: "true"}) - - node1.Spec.Taints = []v1.Taint{startupTaint} - node1.Status.Capacity = v1.ResourceList{v1.ResourcePods: resource.MustParse("10")} - ExpectApplied(ctx, env.Client, nodeClaim1, node1) - - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - // we should launch a new node since the startup taint is there, but was gone at some point - secondPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("should consider a tainted NotReady node as in-flight even if initialized", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: resource.MustParse("10m")}, - }} - ExpectApplied(ctx, env.Client, nodePool) - - // Schedule to New NodeClaim - pod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node1 := ExpectScheduled(ctx, env.Client, pod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - // Mark Initialized - node1.Labels[v1beta1.NodeInitializedLabelKey] = "true" - node1.Spec.Taints = []v1.Taint{ - {Key: v1.TaintNodeNotReady, Effect: v1.TaintEffectNoSchedule}, - {Key: v1.TaintNodeUnreachable, Effect: v1.TaintEffectNoSchedule}, - {Key: cloudproviderapi.TaintExternalCloudProvider, Effect: v1.TaintEffectNoSchedule, Value: "true"}, - } - ExpectApplied(ctx, env.Client, node1) - // Schedule to In Flight NodeClaim - pod = test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) - - Expect(node1.Name).To(Equal(node2.Name)) - }) - }) - Context("Daemonsets", func() { - It("should track daemonset usage separately so we know how many DS resources are remaining to be scheduled", func() { - ds := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - ) - ExpectApplied(ctx, env.Client, nodePool, ds) - Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(ds), ds)).To(Succeed()) - - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("8"), - }, - }} - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - - // create our daemonset pod and manually bind it to the node - dsPod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("2Gi"), - }}, - }) - dsPod.OwnerReferences = append(dsPod.OwnerReferences, metav1.OwnerReference{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - Name: ds.Name, - UID: ds.UID, - Controller: ptr.Bool(true), - BlockOwnerDeletion: ptr.Bool(true), - }) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - ExpectApplied(ctx, env.Client, nodePool, dsPod) - cluster.ForEachNode(func(f *state.StateNode) bool { - dsRequests := f.DaemonSetRequests() - available := f.Available() - Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 0)) - // no pods so we have the full (16 cpu - 100m overhead) - Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 15.9)) - return true - }) - ExpectManualBinding(ctx, env.Client, dsPod, node1) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(dsPod)) - - cluster.ForEachNode(func(f *state.StateNode) bool { - dsRequests := f.DaemonSetRequests() - available := f.Available() - Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 1)) - // only the DS pod is bound, so available is reduced by one and the DS requested is incremented by one - Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 14.9)) - return true - }) - - opts = test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("14.9"), - }, - }} - // this pod should schedule on the existingNodes node as the daemonset pod has already bound, meaning that the - // remaining daemonset resources should be zero leaving 14.9 CPUs for the pod - secondPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should handle unexpected daemonset pods binding to the node", func() { - ds1 := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - NodeSelector: map[string]string{ - "my-node-label": "value", - }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - ) - ds2 := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1m"), - }}}}) - ExpectApplied(ctx, env.Client, nodePool, ds1, ds2) - Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(ds1), ds1)).To(Succeed()) - - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("8"), - }, - }} - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - // this label appears on the node for some reason that Karpenter can't track - node1.Labels["my-node-label"] = "value" - ExpectApplied(ctx, env.Client, node1) - - // create our daemonset pod and manually bind it to the node - dsPod := test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{ - "my-node-label": "value", - }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("2Gi"), - }}, - }) - dsPod.OwnerReferences = append(dsPod.OwnerReferences, metav1.OwnerReference{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - Name: ds1.Name, - UID: ds1.UID, - Controller: ptr.Bool(true), - BlockOwnerDeletion: ptr.Bool(true), - }) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - ExpectApplied(ctx, env.Client, nodePool, dsPod) - cluster.ForEachNode(func(f *state.StateNode) bool { - dsRequests := f.DaemonSetRequests() - available := f.Available() - Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 0)) - // no pods, so we have the full (16 CPU - 100m overhead) - Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 15.9)) - return true - }) - ExpectManualBinding(ctx, env.Client, dsPod, node1) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(dsPod)) - - cluster.ForEachNode(func(f *state.StateNode) bool { - dsRequests := f.DaemonSetRequests() - available := f.Available() - Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 1)) - // only the DS pod is bound, so available is reduced by one and the DS requested is incremented by one - Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 14.9)) - return true - }) - - opts = test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("15.5"), - }, - }} - // This pod should not schedule on the inflight node as it requires more CPU than we have. This verifies - // we don't reintroduce a bug where more daemonsets scheduled than anticipated due to unexepected labels - // appearing on the node which caused us to compute a negative amount of resources remaining for daemonsets - // which in turn caused us to mis-calculate the amount of resources that were free on the node. - secondPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - // must create a new node - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - - }) - // nolint:gosec - It("should pack in-flight nodes before launching new nodes", func() { - cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "medium", - Resources: v1.ResourceList{ - // enough CPU for four pods + a bit of overhead - v1.ResourceCPU: resource.MustParse("4.25"), - v1.ResourcePods: resource.MustParse("4"), - }, - }), - } - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1"), - }, - }} - - ExpectApplied(ctx, env.Client, nodePool) - - // scheduling in multiple batches random sets of pods - for i := 0; i < 10; i++ { - initialPods := test.UnschedulablePods(opts, rand.Intn(10)) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPods...) - for _, pod := range initialPods { - node := ExpectScheduled(ctx, env.Client, pod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - } - } - - // due to the in-flight node support, we should pack existing nodes before launching new node. The end result - // is that we should only have some spare capacity on our final node - nodesWithCPUFree := 0 - cluster.ForEachNode(func(n *state.StateNode) bool { - available := n.Available() - if available.Cpu().AsApproximateFloat64() >= 1 { - nodesWithCPUFree++ - } - return true - }) - Expect(nodesWithCPUFree).To(BeNumerically("<=", 1)) - }) - It("should not launch a second node if there is an in-flight node that can support the pod (#2011)", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(opts) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pod) - var nodes v1.NodeList - Expect(env.Client.List(ctx, &nodes)).To(Succeed()) - Expect(nodes.Items).To(HaveLen(1)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodes.Items[0])) - - pod.Status.Conditions = []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}} - ExpectApplied(ctx, env.Client, pod) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pod) - Expect(env.Client.List(ctx, &nodes)).To(Succeed()) - // shouldn't create a second node - Expect(nodes.Items).To(HaveLen(1)) - }) - It("should order initialized nodes for scheduling un-initialized nodes when all other nodes are inflight", func() { - ExpectApplied(ctx, env.Client, nodePool) - - var nodeClaims []*v1beta1.NodeClaim - var node *v1.Node - //nolint:gosec - elem := rand.Intn(100) // The nodeclaim/node that will be marked as initialized - for i := 0; i < 100; i++ { - nc := test.NodeClaim(v1beta1.NodeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1beta1.NodePoolLabelKey: nodePool.Name, - }, - }, - }) - ExpectApplied(ctx, env.Client, nc) - if i == elem { - nc, node = ExpectNodeClaimDeployed(ctx, env.Client, cluster, cloudProvider, nc) - } else { - var err error - nc, err = ExpectNodeClaimDeployedNoNode(ctx, env.Client, cluster, cloudProvider, nc) - Expect(err).ToNot(HaveOccurred()) - } - nodeClaims = append(nodeClaims, nc) - } - - // Make one of the nodes and nodeClaims initialized - ExpectMakeNodeClaimsInitialized(ctx, env.Client, nodeClaims[elem]) - ExpectMakeNodesInitialized(ctx, env.Client, node) - ExpectReconcileSucceeded(ctx, nodeClaimStateController, client.ObjectKeyFromObject(nodeClaims[elem])) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - scheduledNode := ExpectScheduled(ctx, env.Client, pod) - - // Expect that the scheduled node is equal to node3 since it's initialized - Expect(scheduledNode.Name).To(Equal(node.Name)) - }) - }) - - Describe("Existing Nodes", func() { - It("should schedule a pod to an existing node unowned by Karpenter", func() { - node := test.Node(test.NodeOptions{ - Allocatable: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10"), - v1.ResourceMemory: resource.MustParse("10Gi"), - v1.ResourcePods: resource.MustParse("110"), - }, - }) - ExpectApplied(ctx, env.Client, node) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - scheduledNode := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).To(Equal(scheduledNode.Name)) - }) - It("should schedule multiple pods to an existing node unowned by Karpenter", func() { - node := test.Node(test.NodeOptions{ - Allocatable: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10"), - v1.ResourceMemory: resource.MustParse("100Gi"), - v1.ResourcePods: resource.MustParse("110"), - }, - }) - ExpectApplied(ctx, env.Client, node) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - ExpectApplied(ctx, env.Client, nodePool) - pods := test.UnschedulablePods(opts, 100) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - - for _, pod := range pods { - scheduledNode := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).To(Equal(scheduledNode.Name)) - } - }) - It("should order initialized nodes for scheduling un-initialized nodes", func() { - ExpectApplied(ctx, env.Client, nodePool) - - var nodeClaims []*v1beta1.NodeClaim - var nodes []*v1.Node - for i := 0; i < 100; i++ { - nc := test.NodeClaim(v1beta1.NodeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1beta1.NodePoolLabelKey: nodePool.Name, - }, - }, - }) - ExpectApplied(ctx, env.Client, nc) - nc, n := ExpectNodeClaimDeployed(ctx, env.Client, cluster, cloudProvider, nc) - nodeClaims = append(nodeClaims, nc) - nodes = append(nodes, n) - } - - // Make one of the nodes and nodeClaims initialized - elem := rand.Intn(100) //nolint:gosec - ExpectMakeNodeClaimsInitialized(ctx, env.Client, nodeClaims[elem]) - ExpectMakeNodesInitialized(ctx, env.Client, nodes[elem]) - ExpectReconcileSucceeded(ctx, nodeClaimStateController, client.ObjectKeyFromObject(nodeClaims[elem])) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(nodes[elem])) - - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - scheduledNode := ExpectScheduled(ctx, env.Client, pod) - - // Expect that the scheduled node is equal to the ready node since it's initialized - Expect(scheduledNode.Name).To(Equal(nodes[elem].Name)) - }) - It("should consider a pod incompatible with an existing node but compatible with NodePool", func() { - nodeClaim, node := test.NodeClaimAndNode(v1beta1.NodeClaim{ - Status: v1beta1.NodeClaimStatus{ - Allocatable: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10"), - v1.ResourceMemory: resource.MustParse("10Gi"), - v1.ResourcePods: resource.MustParse("110"), - }, - }, - }) - ExpectApplied(ctx, env.Client, nodeClaim, node) - ExpectMakeNodeClaimsInitialized(ctx, env.Client, nodeClaim) - ExpectMakeNodesInitialized(ctx, env.Client, node) - - ExpectReconcileSucceeded(ctx, nodeClaimStateController, client.ObjectKeyFromObject(nodeClaim)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1"}, - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - Context("Daemonsets", func() { - It("should not subtract daemonset overhead that is not strictly compatible with an existing node", func() { - nodeClaim, node := test.NodeClaimAndNode(v1beta1.NodeClaim{ - Status: v1beta1.NodeClaimStatus{ - Allocatable: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourcePods: resource.MustParse("110"), - }, - }, - }) - // This DaemonSet is not compatible with the existing NodeClaim/Node - ds := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100"), - v1.ResourceMemory: resource.MustParse("100Gi")}, - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1"}, - }, - }, - }}, - ) - ExpectApplied(ctx, env.Client, nodePool, nodeClaim, node, ds) - ExpectMakeNodeClaimsInitialized(ctx, env.Client, nodeClaim) - ExpectMakeNodesInitialized(ctx, env.Client, node) - - ExpectReconcileSucceeded(ctx, nodeClaimStateController, client.ObjectKeyFromObject(nodeClaim)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi")}, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - scheduledNode := ExpectScheduled(ctx, env.Client, pod) - Expect(scheduledNode.Name).To(Equal(node.Name)) - - // Add another pod and expect that pod not to schedule against a nodePool since we will model the DS against the nodePool - // In this case, the DS overhead will take over the entire capacity for every "theoretical node" so we can't schedule a new pod to any new Node - pod2 := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi")}, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod2) - ExpectNotScheduled(ctx, env.Client, pod2) - }) - }) - }) - - Describe("No Pre-Binding", func() { - It("should not bind pods to nodes", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - - var nodeList v1.NodeList - // shouldn't have any nodes - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(0)) - - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod(opts) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - ExpectNotScheduled(ctx, env.Client, initialPod) - - // should launch a single node - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(1)) - node1 := &nodeList.Items[0] - - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - secondPod := test.UnschedulablePod(opts) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - ExpectNotScheduled(ctx, env.Client, secondPod) - // shouldn't create a second node as it can bind to the existingNodes node - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(1)) - }) - It("should handle resource zeroing of extended resources by kubelet", func() { - // Issue #1459 - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - }} - - var nodeList v1.NodeList - // shouldn't have any nodes - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(0)) - - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod(opts) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - ExpectNotScheduled(ctx, env.Client, initialPod) - - // should launch a single node - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(1)) - node1 := &nodeList.Items[0] - - // simulate kubelet zeroing out the extended resources on the node at startup - node1.Status.Capacity = map[v1.ResourceName]resource.Quantity{ - fake.ResourceGPUVendorA: resource.MustParse("0"), - } - node1.Status.Allocatable = map[v1.ResourceName]resource.Quantity{ - fake.ResourceGPUVendorB: resource.MustParse("0"), - } - - ExpectApplied(ctx, env.Client, node1) - - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - secondPod := test.UnschedulablePod(opts) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - ExpectNotScheduled(ctx, env.Client, secondPod) - // shouldn't create a second node as it can bind to the existingNodes node - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(1)) - }) - It("should respect self pod affinity without pod binding (zone)", func() { - // Issue #1975 - affLabels := map[string]string{"security": "s2"} - - pods := test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: affLabels, - }, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }}, - }, 2) - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pods[0]) - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - for i := range nodeList.Items { - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodeList.Items[i])) - } - // the second pod can schedule against the in-flight node, but for that to work we need to be careful - // in how we fulfill the self-affinity by taking the existing node's domain as a preference over any - // random viable domain - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pods[1]) - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(1)) - }) - }) - - Describe("VolumeUsage", func() { - BeforeEach(func() { - cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ - fake.NewInstanceType( - fake.InstanceTypeOptions{ - Name: "instance-type", - Resources: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1024"), - v1.ResourcePods: resource.MustParse("1024"), - }, - }), - } - nodePool.Spec.Limits = nil - }) - It("should launch multiple nodes if required due to volume limits", func() { - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: csiProvider, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(10), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, - Provisioner: ptr.String(csiProvider), - Zones: []string{"test-zone-1"}}) - ExpectApplied(ctx, env.Client, sc) - - var pods []*v1.Pod - for i := 0; i < 6; i++ { - pvcA := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - StorageClassName: ptr.String("my-storage-class"), - ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("my-claim-a-%d", i)}, - }) - pvcB := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - StorageClassName: ptr.String("my-storage-class"), - ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("my-claim-b-%d", i)}, - }) - ExpectApplied(ctx, env.Client, pvcA, pvcB) - pods = append(pods, test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{pvcA.Name, pvcB.Name}, - })) - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - // we need to create a new node as the in-flight one can only contain 5 pods due to the CSINode volume limit - Expect(nodeList.Items).To(HaveLen(2)) - }) - It("should launch a single node if all pods use the same PVC", func() { - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: csiProvider, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(10), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, - Provisioner: ptr.String(csiProvider), - Zones: []string{"test-zone-1"}}) - ExpectApplied(ctx, env.Client, sc) - - pv := test.PersistentVolume(test.PersistentVolumeOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, - Zones: []string{"test-zone-1"}}) - - pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, - StorageClassName: ptr.String("my-storage-class"), - VolumeName: pv.Name, - }) - ExpectApplied(ctx, env.Client, pv, pvc) - - var pods []*v1.Pod - for i := 0; i < 100; i++ { - pods = append(pods, test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{pvc.Name}, - })) - } - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - // 100 of the same PVC should all be schedulable on the same node - Expect(nodeList.Items).To(HaveLen(1)) - }) - It("should not fail for NFS volumes", func() { - ExpectApplied(ctx, env.Client, nodePool) - initialPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pv := test.PersistentVolume(test.PersistentVolumeOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, - StorageClassName: "nfs", - Zones: []string{"test-zone-1"}}) - pv.Spec.NFS = &v1.NFSVolumeSource{ - Server: "fake.server", - Path: "/some/path", - } - pv.Spec.CSI = nil - - pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, - VolumeName: pv.Name, - StorageClassName: ptr.String(""), - }) - ExpectApplied(ctx, env.Client, pv, pvc) - - var pods []*v1.Pod - for i := 0; i < 5; i++ { - pods = append(pods, test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{pvc.Name, pvc.Name}, - })) - } - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - // 5 of the same PVC should all be schedulable on the same node - Expect(nodeList.Items).To(HaveLen(1)) - }) - It("should launch nodes for pods with ephemeral volume using the specified storage class name", func() { - // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-storage-class", - }, - Provisioner: ptr.String(csiProvider), - Zones: []string{"test-zone-1"}}) - // Create another default storage class that shouldn't be used and has no associated limits - sc2 := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String("other-provider"), - Zones: []string{"test-zone-1"}}) - - initialPod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has a specified storage class, so it should use the one specified - initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - StorageClassName: lo.ToPtr(sc.Name), - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool, sc, sc2, initialPod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: csiProvider, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(1), - }, - }, - { - Name: "other-provider", - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(10), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has a specified storage class, so it should use the one specified - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - StorageClassName: lo.ToPtr(sc.Name), - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).ToNot(Equal(node2.Name)) - }) - It("should launch nodes for pods with ephemeral volume using a default storage class", func() { - // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String(csiProvider), - Zones: []string{"test-zone-1"}}) - - initialPod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one - initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool, sc, initialPod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: csiProvider, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(1), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - - ExpectApplied(ctx, env.Client, sc, nodePool, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).ToNot(Equal(node2.Name)) - }) - It("should launch nodes for pods with ephemeral volume using the newest storage class", func() { - if env.Version.Minor() < 26 { - Skip("Multiple default storage classes is only available in K8s >= 1.26.x") - } - // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String("other-provider"), - Zones: []string{"test-zone-1"}}) - sc2 := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "newer-default-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String(csiProvider), - Zones: []string{"test-zone-1"}}) - - ExpectApplied(ctx, env.Client, sc) - // Wait a few seconds to apply the second storage class to get a newer creationTimestamp - time.Sleep(time.Second * 2) - ExpectApplied(ctx, env.Client, sc2) - - initialPod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one - initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool, sc, initialPod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: csiProvider, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(1), - }, - }, - { - Name: "other-provider", - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(10), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, sc, nodePool, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).ToNot(Equal(node2.Name)) - }) - It("should not launch nodes for pods with ephemeral volume using a non-existent storage classes", func() { - ExpectApplied(ctx, env.Client, nodePool) - pod := test.UnschedulablePod(test.PodOptions{}) - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - StorageClassName: ptr.String("non-existent"), - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - // no nodes should be created as the storage class doesn't eixst - Expect(nodeList.Items).To(HaveLen(0)) - }) - Context("CSIMigration", func() { - It("should launch nodes for pods with non-dynamic PVC using a migrated PVC/PV", func() { - // We should assume that this PVC/PV is using CSI driver implicitly to limit pod scheduling - // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "in-tree-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String(plugins.AWSEBSInTreePluginName), - Zones: []string{"test-zone-1"}}) - pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - StorageClassName: ptr.String(sc.Name), - }) - ExpectApplied(ctx, env.Client, nodePool, sc, pvc) - initialPod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{pvc.Name}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: plugins.AWSEBSDriverName, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(1), - }, - }, - }, - }, - } - pv := test.PersistentVolume(test.PersistentVolumeOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-volume", - }, - Zones: []string{"test-zone-1"}, - UseAWSInTreeDriver: true, - }) - ExpectApplied(ctx, env.Client, csiNode, pvc, pv) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pvc2 := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - StorageClassName: ptr.String(sc.Name), - }) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{pvc2.Name}, - }) - ExpectApplied(ctx, env.Client, pvc2, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).ToNot(Equal(node2.Name)) - }) - It("should launch nodes for pods with ephemeral volume using a migrated PVC/PV", func() { - // We should assume that this PVC/PV is using CSI driver implicitly to limit pod scheduling - // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "in-tree-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String(plugins.AWSEBSInTreePluginName), - Zones: []string{"test-zone-1"}}) - - initialPod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that references the in-tree storage provider - initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - StorageClassName: ptr.String(sc.Name), - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodePool, sc, initialPod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: plugins.AWSEBSDriverName, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(1), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that reference the in-tree storage provider - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - StorageClassName: ptr.String(sc.Name), - }, - }, - }, - }, - }) - // Pod should not schedule to the first node since we should realize that we have hit our volume limits - ExpectApplied(ctx, env.Client, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).ToNot(Equal(node2.Name)) - }) - }) - }) -}) diff --git a/pkg/controllers/provisioning/scheduling/provisioner_instance_selection_test.go b/pkg/controllers/provisioning/scheduling/provisioner_instance_selection_test.go deleted file mode 100644 index d6f20b28da..0000000000 --- a/pkg/controllers/provisioning/scheduling/provisioner_instance_selection_test.go +++ /dev/null @@ -1,596 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheduling_test - -import ( - "fmt" - "math/rand" - - "github.com/mitchellh/hashstructure/v2" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/util/sets" - - "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter-core/pkg/cloudprovider" - "github.com/aws/karpenter-core/pkg/cloudprovider/fake" - "github.com/aws/karpenter-core/pkg/test" - . "github.com/aws/karpenter-core/pkg/test/expectations" - "github.com/aws/karpenter-core/pkg/utils/resources" -) - -var _ = Describe("Instance Type Selection", func() { - var provisioner *v1alpha5.Provisioner - var minPrice float64 - var instanceTypeMap map[string]*cloudprovider.InstanceType - nodePrice := func(n *v1.Node) float64 { - of, _ := instanceTypeMap[n.Labels[v1.LabelInstanceTypeStable]].Offerings.Get(n.Labels[v1alpha5.LabelCapacityType], n.Labels[v1.LabelTopologyZone]) - return of.Price - } - - BeforeEach(func() { - provisioner = test.Provisioner(test.ProvisionerOptions{Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeSpot, v1alpha5.CapacityTypeOnDemand}, - }, - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, - }, - }}) - cloudProvider.InstanceTypes = fake.InstanceTypesAssorted() - instanceTypeMap = getInstanceTypeMap(cloudProvider.InstanceTypes) - minPrice = getMinPrice(cloudProvider.InstanceTypes) - - // add some randomness to instance type ordering to ensure we sort everywhere we need to - rand.Shuffle(len(cloudProvider.InstanceTypes), func(i, j int) { - cloudProvider.InstanceTypes[i], cloudProvider.InstanceTypes[j] = cloudProvider.InstanceTypes[j], cloudProvider.InstanceTypes[i] - }) - }) - - // This set of tests ensure that we schedule on the cheapest valid instance type while also ensuring that all of the - // instance types passed to the cloud provider are also valid per provisioner and node selector requirements. In some - // ways they repeat some other tests, but the testing regarding checking against all possible instance types - // passed to the cloud provider is unique. - It("should schedule on one of the cheapest instances", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - }) - It("should schedule on one of the cheapest instances (pod arch = amd64)", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - // ensure that the entire list of instance types match the label - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureAmd64) - }) - It("should schedule on one of the cheapest instances (pod arch = arm64)", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureArm64) - }) - It("should schedule on one of the cheapest instances (prov arch = amd64)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureAmd64) - }) - It("should schedule on one of the cheapest instances (prov arch = arm64)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64}, - }, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureArm64) - }) - It("should schedule on one of the cheapest instances (prov os = windows)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Windows)}, - }, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) - }) - It("should schedule on one of the cheapest instances (pod os = windows)", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Windows)}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) - }) - It("should schedule on one of the cheapest instances (prov os = windows)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Windows)}, - }, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) - }) - It("should schedule on one of the cheapest instances (pod os = linux)", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) - }) - It("should schedule on one of the cheapest instances (pod os = linux)", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) - }) - It("should schedule on one of the cheapest instances (prov zone = test-zone-2)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2"}, - }, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelTopologyZone, "test-zone-2") - }) - It("should schedule on one of the cheapest instances (pod zone = test-zone-2)", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2"}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelTopologyZone, "test-zone-2") - }) - It("should schedule on one of the cheapest instances (prov ct = spot)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeSpot}, - }, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1alpha5.LabelCapacityType, v1alpha5.CapacityTypeSpot) - }) - It("should schedule on one of the cheapest instances (pod ct = spot)", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeSpot}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1alpha5.LabelCapacityType, v1alpha5.CapacityTypeSpot) - }) - It("should schedule on one of the cheapest instances (prov ct = ondemand, prov zone = test-zone-1)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeOnDemand}, - }, - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1"}, - }, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithOffering(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1alpha5.CapacityTypeOnDemand, "test-zone-1") - }) - It("should schedule on one of the cheapest instances (pod ct = spot, pod zone = test-zone-1)", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeSpot}, - }, - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1"}, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithOffering(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-1") - }) - It("should schedule on one of the cheapest instances (prov ct = spot, pod zone = test-zone-2)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeSpot}, - }, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2"}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithOffering(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-2") - }) - It("should schedule on one of the cheapest instances (prov ct = ondemand/test-zone-1/arm64/windows)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64}, - }, - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Windows)}, - }, - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeOnDemand}, - }, - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1"}, - }, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithOffering(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1alpha5.CapacityTypeOnDemand, "test-zone-1") - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelArchStable, "arm64") - }) - It("should schedule on one of the cheapest instances (prov = spot/test-zone-2, pod = amd64/linux)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }, - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, - }, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeSpot}, - }, - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2"}, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithOffering(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-2") - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelArchStable, "amd64") - }) - It("should schedule on one of the cheapest instances (pod ct = spot/test-zone-2/amd64/linux)", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }, - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, - }, - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeSpot}, - }, - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2"}, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(nodePrice(node)).To(Equal(minPrice)) - ExpectInstancesWithOffering(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-2") - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) - ExpectInstancesWithLabel(supportedInstanceTypes(cloudProvider.CreateCalls[0]), v1.LabelArchStable, "amd64") - }) - It("should not schedule if no instance type matches selector (pod arch = arm)", func() { - // remove all Arm instance types - cloudProvider.InstanceTypes = filterInstanceTypes(cloudProvider.InstanceTypes, func(i *cloudprovider.InstanceType) bool { - return i.Requirements.Get(v1.LabelArchStable).Has(v1alpha5.ArchitectureAmd64) - }) - - Expect(len(cloudProvider.InstanceTypes)).To(BeNumerically(">", 0)) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64}, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - Expect(cloudProvider.CreateCalls).To(HaveLen(0)) - }) - It("should not schedule if no instance type matches selector (pod arch = arm zone=test-zone-2)", func() { - // remove all Arm instance types in zone-2 - cloudProvider.InstanceTypes = filterInstanceTypes(cloudProvider.InstanceTypes, func(i *cloudprovider.InstanceType) bool { - for _, off := range i.Offerings { - if off.Zone == "test-zone-2" { - return i.Requirements.Get(v1.LabelArchStable).Has(v1alpha5.ArchitectureAmd64) - } - } - return true - }) - Expect(len(cloudProvider.InstanceTypes)).To(BeNumerically(">", 0)) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64}, - }, - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2"}, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - Expect(cloudProvider.CreateCalls).To(HaveLen(0)) - }) - It("should not schedule if no instance type matches selector (prov arch = arm / pod zone=test-zone-2)", func() { - // remove all Arm instance types in zone-2 - cloudProvider.InstanceTypes = filterInstanceTypes(cloudProvider.InstanceTypes, func(i *cloudprovider.InstanceType) bool { - for _, off := range i.Offerings { - if off.Zone == "test-zone-2" { - return i.Requirements.Get(v1.LabelArchStable).Has(v1alpha5.ArchitectureAmd64) - } - } - return true - }) - - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64}, - }, - } - Expect(len(cloudProvider.InstanceTypes)).To(BeNumerically(">", 0)) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2"}, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - Expect(cloudProvider.CreateCalls).To(HaveLen(0)) - }) - It("should schedule on an instance with enough resources", func() { - // this is a pretty thorough exercise of scheduling, so we also check an invariant that scheduling doesn't - // modify the instance type's Overhead() or Resources() maps so they can return the same map every time instead - // of re-alllocating a new one per call - resourceHashes := map[string]uint64{} - overheadHashes := map[string]uint64{} - for _, it := range cloudProvider.InstanceTypes { - var err error - resourceHashes[it.Name], err = hashstructure.Hash(it.Capacity, hashstructure.FormatV2, nil) - Expect(err).To(BeNil()) - overheadHashes[it.Name], err = hashstructure.Hash(it.Overhead.Total(), hashstructure.FormatV2, nil) - Expect(err).To(BeNil()) - } - ExpectApplied(ctx, env.Client, provisioner) - // these values are constructed so that three of these pods can always fit on at least one of our instance types - for _, cpu := range []float64{0.1, 1.0, 2, 2.5, 4, 8, 16} { - for _, mem := range []float64{0.1, 1.0, 2, 4, 8, 16, 32} { - cluster.Reset() - cloudProvider.CreateCalls = nil - opts := test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%0.1f", cpu)), - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%0.1fGi", mem)), - }}} - pods := []*v1.Pod{ - test.UnschedulablePod(opts), test.UnschedulablePod(opts), test.UnschedulablePod(opts), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := sets.NewString() - for _, p := range pods { - node := ExpectScheduled(ctx, env.Client, p) - nodeNames.Insert(node.Name) - } - // should fit on one node - Expect(nodeNames).To(HaveLen(1)) - totalPodResources := resources.RequestsForPods(pods...) - for _, it := range supportedInstanceTypes(cloudProvider.CreateCalls[0]) { - totalReserved := resources.Merge(totalPodResources, it.Overhead.Total()) - // the total pod resources in CPU and memory + instance overhead should always be less than the - // resources available on every viable instance has - Expect(totalReserved.Cpu().Cmp(it.Capacity[v1.ResourceCPU])).To(Equal(-1)) - Expect(totalReserved.Memory().Cmp(it.Capacity[v1.ResourceMemory])).To(Equal(-1)) - } - } - } - for _, it := range cloudProvider.InstanceTypes { - resourceHash, err := hashstructure.Hash(it.Capacity, hashstructure.FormatV2, nil) - Expect(err).To(BeNil()) - overheadHash, err := hashstructure.Hash(it.Overhead.Total(), hashstructure.FormatV2, nil) - Expect(err).To(BeNil()) - Expect(resourceHash).To(Equal(resourceHashes[it.Name]), fmt.Sprintf("expected %s Resources() to not be modified by scheduling", it.Name)) - Expect(overheadHash).To(Equal(overheadHashes[it.Name]), fmt.Sprintf("expected %s Overhead() to not be modified by scheduling", it.Name)) - } - }) - It("should schedule on cheaper on-demand instance even when spot price ordering would place other instance types first", func() { - cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "test-instance1", - Architecture: "amd64", - OperatingSystems: sets.New(string(v1.Linux)), - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi"), - }, - Offerings: []cloudprovider.Offering{ - {CapacityType: v1alpha5.CapacityTypeOnDemand, Zone: "test-zone-1a", Price: 1.0, Available: true}, - {CapacityType: v1alpha5.CapacityTypeSpot, Zone: "test-zone-1a", Price: 0.2, Available: true}, - }, - }), - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "test-instance2", - Architecture: "amd64", - OperatingSystems: sets.New(string(v1.Linux)), - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi"), - }, - Offerings: []cloudprovider.Offering{ - {CapacityType: v1alpha5.CapacityTypeOnDemand, Zone: "test-zone-1a", Price: 1.3, Available: true}, - {CapacityType: v1alpha5.CapacityTypeSpot, Zone: "test-zone-1a", Price: 0.1, Available: true}, - }, - }), - } - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{"on-demand"}, - }, - } - - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("test-instance1")) - }) -}) diff --git a/pkg/controllers/provisioning/scheduling/provisioner_test.go b/pkg/controllers/provisioning/scheduling/provisioner_test.go deleted file mode 100644 index 46b8625bab..0000000000 --- a/pkg/controllers/provisioning/scheduling/provisioner_test.go +++ /dev/null @@ -1,3175 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheduling_test - -import ( - "fmt" - "math/rand" - "time" - - "github.com/samber/lo" - nodev1 "k8s.io/api/node/v1" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - cloudproviderapi "k8s.io/cloud-provider/api" - "k8s.io/csi-translation-lib/plugins" - "knative.dev/pkg/ptr" - - v1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter-core/pkg/cloudprovider" - "github.com/aws/karpenter-core/pkg/cloudprovider/fake" - "github.com/aws/karpenter-core/pkg/controllers/state" - pscheduling "github.com/aws/karpenter-core/pkg/scheduling" - "github.com/aws/karpenter-core/pkg/test" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "github.com/aws/karpenter-core/pkg/test/expectations" -) - -var _ = Context("Provisioner", func() { - var provisioner *v1alpha5.Provisioner - BeforeEach(func() { - provisioner = test.Provisioner(test.ProvisionerOptions{Requirements: []v1.NodeSelectorRequirement{{ - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeSpot, v1alpha5.CapacityTypeOnDemand}, - }}}) - }) - - Describe("Custom Constraints", func() { - Context("Provisioner with Labels", func() { - It("should schedule unconstrained pods that don't have matching node selectors", func() { - provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have conflicting node selectors", func() { - provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{"test-key": "different-value"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule pods that have node selectors with undefined key", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{"test-key": "test-value"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that have matching requirements", func() { - provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have conflicting requirements", func() { - provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - }) - Context("Well Known Labels", func() { - It("should use provisioner constraints", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should use node selectors", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should not schedule nodes with a hostname selector", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelHostname: "red-node"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule the pod if nodeselector unknown", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if node selector outside of provisioner constraints", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should schedule compatible requirements with Operator=Gt", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpGt, Values: []string{"8"}, - }} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "16")) - }) - It("should schedule compatible requirements with Operator=Lt", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpLt, Values: []string{"8"}, - }} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "2")) - }) - It("should not schedule incompatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should not schedule incompatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "unknown"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should schedule incompatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should schedule incompatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible node selectors, preferences and requirements", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}, - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should combine multidimensional node selectors, preferences and requirements", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeSelector: map[string]string{ - v1.LabelTopologyZone: "test-zone-3", - v1.LabelInstanceTypeStable: "arm-instance-type", - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type", "arm-instance-type"}}, - }, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, - }, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "arm-instance-type")) - }) - }) - Context("Constraints Validation", func() { - It("should not schedule pods that have node selectors with restricted labels", func() { - ExpectApplied(ctx, env.Client, provisioner) - for label := range v1alpha5.RestrictedLabels { - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should not schedule pods that have node selectors with restricted domains", func() { - ExpectApplied(ctx, env.Client, provisioner) - for domain := range v1alpha5.RestrictedLabelDomains { - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should schedule pods that have node selectors with label in restricted domains exceptions list", func() { - var requirements []v1.NodeSelectorRequirement - for domain := range v1alpha5.LabelDomainExceptions { - requirements = append(requirements, v1.NodeSelectorRequirement{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}) - } - provisioner.Spec.Requirements = requirements - ExpectApplied(ctx, env.Client, provisioner) - for domain := range v1alpha5.LabelDomainExceptions { - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) - } - }) - It("should schedule pods that have node selectors with label in wellknown label list", func() { - schedulable := []*v1.Pod{ - // Constrained by zone - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), - // Constrained by instanceType - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), - // Constrained by architecture - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), - // Constrained by operatingSystem - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), - // Constrained by capacity type - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.LabelCapacityType: "spot"}}), - } - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) - for _, pod := range schedulable { - ExpectScheduled(ctx, env.Client, pod) - } - }) - }) - Context("Scheduling Logic", func() { - It("should not schedule pods that have node selectors with In operator and undefined key", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that have node selectors with NotIn operator and undefined key", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).ToNot(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have node selectors with Exists operator and undefined key", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpExists}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that with DoesNotExists operator and undefined key", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).ToNot(HaveKey("test-key")) - }) - It("should schedule unconstrained pods that don't have matching node selectors", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should schedule pods that have node selectors with matching value and In operator", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have node selectors with matching value and NotIn operator", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule the pod with Exists operator and defined key", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpExists}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should not schedule the pod with DoesNotExists operator and defined key", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule pods that have node selectors with different value and In operator", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that have node selectors with different value and NotIn operator", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should schedule compatible pods to the same node", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}), - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - node1 := ExpectScheduled(ctx, env.Client, pods[0]) - node2 := ExpectScheduled(ctx, env.Client, pods[1]) - Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node2.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should schedule incompatible pods to the different node", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}), - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - node1 := ExpectScheduled(ctx, env.Client, pods[0]) - node2 := ExpectScheduled(ctx, env.Client, pods[1]) - Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node2.Labels).To(HaveKeyWithValue("test-key", "another-value")) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("Exists operator should not overwrite the existing value", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"non-existent-zone"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - }) - Context("Well Known Labels", func() { - It("should use provisioner constraints", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should use node selectors", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should not schedule nodes with a hostname selector", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelHostname: "red-node"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule the pod if nodeselector unknown", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule if node selector outside of provisioner constraints", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should schedule compatible requirements with Operator=Gt", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpGt, Values: []string{"8"}, - }} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "16")) - }) - It("should schedule compatible requirements with Operator=Lt", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpLt, Values: []string{"8"}, - }} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "2")) - }) - It("should not schedule incompatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should not schedule incompatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "unknown"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should schedule incompatible preferences and requirements with Operator=In", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should schedule incompatible preferences and requirements with Operator=NotIn", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should schedule compatible node selectors, preferences and requirements", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}, - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should combine multidimensional node selectors, preferences and requirements", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeSelector: map[string]string{ - v1.LabelTopologyZone: "test-zone-3", - v1.LabelInstanceTypeStable: "arm-instance-type", - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type", "arm-instance-type"}}, - }, - NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, - }, - }, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "arm-instance-type")) - }) - }) - Context("Constraints Validation", func() { - It("should not schedule pods that have node selectors with restricted labels", func() { - ExpectApplied(ctx, env.Client, provisioner) - for label := range v1alpha5.RestrictedLabels { - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should not schedule pods that have node selectors with restricted domains", func() { - ExpectApplied(ctx, env.Client, provisioner) - for domain := range v1alpha5.RestrictedLabelDomains { - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should schedule pods that have node selectors with label in restricted domains exceptions list", func() { - var requirements []v1.NodeSelectorRequirement - for domain := range v1alpha5.LabelDomainExceptions { - requirements = append(requirements, v1.NodeSelectorRequirement{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}) - } - provisioner.Spec.Requirements = requirements - ExpectApplied(ctx, env.Client, provisioner) - for domain := range v1alpha5.LabelDomainExceptions { - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) - } - }) - It("should schedule pods that have node selectors with label in wellknown label list", func() { - schedulable := []*v1.Pod{ - // Constrained by zone - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), - // Constrained by instanceType - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), - // Constrained by architecture - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), - // Constrained by operatingSystem - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), - // Constrained by capacity type - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.LabelCapacityType: "spot"}}), - } - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) - for _, pod := range schedulable { - ExpectScheduled(ctx, env.Client, pod) - } - }) - }) - Context("Scheduling Logic", func() { - It("should not schedule pods that have node selectors with In operator and undefined key", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that have node selectors with NotIn operator and undefined key", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).ToNot(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have node selectors with Exists operator and undefined key", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpExists}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that with DoesNotExists operator and undefined key", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).ToNot(HaveKey("test-key")) - }) - It("should schedule unconstrained pods that don't have matching node selectors", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should schedule pods that have node selectors with matching value and In operator", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should not schedule pods that have node selectors with matching value and NotIn operator", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule the pod with Exists operator and defined key", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpExists}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should not schedule the pod with DoesNotExists operator and defined key", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should not schedule pods that have node selectors with different value and In operator", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should schedule pods that have node selectors with different value and NotIn operator", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - }) - It("should schedule compatible pods to the same node", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}), - test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - node1 := ExpectScheduled(ctx, env.Client, pods[0]) - node2 := ExpectScheduled(ctx, env.Client, pods[1]) - Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node2.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should schedule incompatible pods to the different node", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - }}), - test.UnschedulablePod( - test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - node1 := ExpectScheduled(ctx, env.Client, pods[0]) - node2 := ExpectScheduled(ctx, env.Client, pods[1]) - Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) - Expect(node2.Labels).To(HaveKeyWithValue("test-key", "another-value")) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("Exists operator should not overwrite the existing value", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"non-existent-zone"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - }) - }) - - Describe("Preferential Fallback", func() { - Context("Required", func() { - It("should not relax the final term", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type"}}, - } - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, // Should not be relaxed - }}, - }}}} - // Don't relax - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should relax multiple terms", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - }}, - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, // OR operator, never get to this one - }}, - }}}} - // Success - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1")) - }) - }) - Context("Preferred", func() { - It("should relax all terms", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - }}, - }, - }}} - // Success - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should relax to use lighter weights", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 100, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, - }}, - }, - { - Weight: 50, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ // OR operator, never get to this one - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - }}, - }, - }}} - // Success - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - }) - It("should schedule even if preference is conflicting with requirement", func() { - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-3"}}, - }}, - }, - }, - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ - {MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, // Should not be relaxed - }}, - }}, - }} - // Success - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - }) - It("should schedule even if preference requirements are conflicting", func() { - pod := test.UnschedulablePod(test.PodOptions{NodePreferences: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"invalid"}}, - }}) - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - }) - }) - - Describe("Instance Type Compatibility", func() { - It("should not schedule if requesting more resources than any instance type has", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("512"), - }}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should launch pods with different archs on different instances", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, - }} - nodeNames := sets.NewString() - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: v1alpha5.ArchitectureAmd64}, - }), - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: v1alpha5.ArchitectureArm64}, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodeNames.Insert(node.Name) - } - Expect(nodeNames.Len()).To(Equal(2)) - }) - It("should exclude instance types that are not supported by the pod constraints (node affinity/instance type)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"arm-instance-type"}, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // arm instance type conflicts with the provisioner limitation of AMD only - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should exclude instance types that are not supported by the pod constraints (node affinity/operating system)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"ios"}, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // there's an instance with an OS of ios, but it has an arm processor so the provider requirements will - // exclude it - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should exclude instance types that are not supported by the provider constraints (arch)", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: resource.MustParse("14")}}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // only the ARM instance has enough CPU, but it's not allowed per the provisioner - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should launch pods with different operating systems on different instances", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, - }} - nodeNames := sets.NewString() - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}, - }), - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Windows)}, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodeNames.Insert(node.Name) - } - Expect(nodeNames.Len()).To(Equal(2)) - }) - It("should launch pods with different instance type node selectors on different instances", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, - }} - nodeNames := sets.NewString() - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelInstanceType: "small-instance-type"}, - }), - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodeNames.Insert(node.Name) - } - Expect(nodeNames.Len()).To(Equal(2)) - }) - It("should launch pods with different zone selectors on different instances", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, - }} - nodeNames := sets.NewString() - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}, - }), - test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodeNames.Insert(node.Name) - } - Expect(nodeNames.Len()).To(Equal(2)) - }) - It("should launch pods with resources that aren't on any single instance type on different instances", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - const fakeGPU1 = "karpenter.sh/super-great-gpu" - const fakeGPU2 = "karpenter.sh/even-better-gpu" - cloudProvider.InstanceTypes[0].Capacity[fakeGPU1] = resource.MustParse("25") - cloudProvider.InstanceTypes[1].Capacity[fakeGPU2] = resource.MustParse("25") - - nodeNames := sets.NewString() - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{fakeGPU1: resource.MustParse("1")}, - }, - }), - // Should pack onto a different instance since no instance type has both GPUs - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{fakeGPU2: resource.MustParse("1")}, - }, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodeNames.Insert(node.Name) - } - Expect(nodeNames.Len()).To(Equal(2)) - }) - It("should fail to schedule a pod with resources requests that aren't on a single instance type", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - const fakeGPU1 = "karpenter.sh/super-great-gpu" - const fakeGPU2 = "karpenter.sh/even-better-gpu" - cloudProvider.InstanceTypes[0].Capacity[fakeGPU1] = resource.MustParse("25") - cloudProvider.InstanceTypes[1].Capacity[fakeGPU2] = resource.MustParse("25") - - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - fakeGPU1: resource.MustParse("1"), - fakeGPU2: resource.MustParse("1")}, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - Context("Provider Specific Labels", func() { - It("should filter instance types that match labels", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{fake.LabelInstanceSize: "large"}}), - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{fake.LabelInstanceSize: "small"}}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - node := ExpectScheduled(ctx, env.Client, pods[0]) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "fake-it-4")) - node = ExpectScheduled(ctx, env.Client, pods[1]) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "fake-it-0")) - }) - It("should not schedule with incompatible labels", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{ - fake.LabelInstanceSize: "large", - v1.LabelInstanceTypeStable: cloudProvider.InstanceTypes[0].Name, - }}), - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{ - fake.LabelInstanceSize: "small", - v1.LabelInstanceTypeStable: cloudProvider.InstanceTypes[4].Name, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - ExpectNotScheduled(ctx, env.Client, pods[0]) - ExpectNotScheduled(ctx, env.Client, pods[1]) - }) - It("should schedule optional labels", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // Only some instance types have this key - {Key: fake.ExoticInstanceLabelKey, Operator: v1.NodeSelectorOpExists}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKey(fake.ExoticInstanceLabelKey)) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, cloudProvider.InstanceTypes[4].Name)) - }) - It("should schedule without optional labels if disallowed", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - ExpectApplied(ctx, env.Client, test.Provisioner()) - pod := test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // Only some instance types have this key - {Key: fake.ExoticInstanceLabelKey, Operator: v1.NodeSelectorOpDoesNotExist}, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).ToNot(HaveKey(fake.ExoticInstanceLabelKey)) - }) - }) - }) - - Describe("Binpacking", func() { - It("should schedule a small pod on the smallest instance", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("100M"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) - }) - It("should schedule a small pod on the smallest possible instance type", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("2000M"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) - }) - It("should take pod runtime class into consideration", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1"), - }, - }}) - // the pod has overhead of 2 CPUs - runtimeClass := &nodev1.RuntimeClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-runtime-class", - }, - Handler: "default", - Overhead: &nodev1.Overhead{ - PodFixed: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - }, - }, - } - pod.Spec.RuntimeClassName = &runtimeClass.Name - ExpectApplied(ctx, env.Client, runtimeClass) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - // overhead of 2 + request of 1 = at least 3 CPUs, so it won't fit on small-instance-type which it otherwise - // would - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) - }) - It("should schedule multiple small pods on the smallest possible instance type", func() { - opts := test.PodOptions{ - Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("10M"), - }, - }} - pods := test.Pods(5, opts) - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := sets.NewString() - for _, p := range pods { - node := ExpectScheduled(ctx, env.Client, p) - nodeNames.Insert(node.Name) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) - } - Expect(nodeNames).To(HaveLen(1)) - }) - It("should create new nodes when a node is at capacity", func() { - opts := test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, - Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1.8G"), - }, - }} - ExpectApplied(ctx, env.Client, provisioner) - pods := test.Pods(40, opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := sets.NewString() - for _, p := range pods { - node := ExpectScheduled(ctx, env.Client, p) - nodeNames.Insert(node.Name) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) - } - Expect(nodeNames).To(HaveLen(20)) - }) - It("should pack small and large pods together", func() { - largeOpts := test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, - Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1.8G"), - }, - }} - smallOpts := test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, - Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("400M"), - }, - }} - - // Two large pods are all that will fit on the default-instance type (the largest instance type) which will create - // twenty nodes. This leaves just enough room on each of those nodes for one additional small pod per node, so we - // should only end up with 20 nodes total. - provPods := append(test.Pods(40, largeOpts), test.Pods(20, smallOpts)...) - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, provPods...) - nodeNames := sets.NewString() - for _, p := range provPods { - node := ExpectScheduled(ctx, env.Client, p) - nodeNames.Insert(node.Name) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) - } - Expect(nodeNames).To(HaveLen(20)) - }) - It("should pack nodes tightly", func() { - cloudProvider.InstanceTypes = fake.InstanceTypes(5) - var nodes []*v1.Node - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.5")}, - }, - }), - test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - }), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - nodes = append(nodes, node) - } - Expect(nodes).To(HaveLen(2)) - // the first pod consumes nearly all CPU of the largest instance type with no room for the second pod, the - // second pod is much smaller in terms of resources and should get a smaller node - Expect(nodes[0].Labels[v1.LabelInstanceTypeStable]).ToNot(Equal(nodes[1].Labels[v1.LabelInstanceTypeStable])) - }) - It("should handle zero-quantity resource requests", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")}, - Limits: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")}, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - // requesting a resource of quantity zero of a type unsupported by any instance is fine - ExpectScheduled(ctx, env.Client, pod) - }) - It("should not schedule pods that exceed every instance type's capacity", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("2Ti"), - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should create new nodes when a node is at capacity due to pod limits per node", func() { - opts := test.PodOptions{ - NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, - Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1m"), - v1.ResourceCPU: resource.MustParse("1m"), - }, - }} - ExpectApplied(ctx, env.Client, provisioner) - pods := test.Pods(25, opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := sets.NewString() - // all of the test instance types support 5 pods each, so we use the 5 instances of the smallest one for our 25 pods - for _, p := range pods { - node := ExpectScheduled(ctx, env.Client, p) - nodeNames.Insert(node.Name) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) - } - Expect(nodeNames).To(HaveLen(5)) - }) - It("should take into account initContainer resource requests when binpacking", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourceCPU: resource.MustParse("1"), - }, - }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourceCPU: resource.MustParse("2"), - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) - }) - It("should not schedule pods when initContainer resource requests are greater than available instance types", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourceCPU: resource.MustParse("1"), - }, - }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1Ti"), - v1.ResourceCPU: resource.MustParse("2"), - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should select for valid instance types, regardless of price", func() { - // capacity sizes and prices don't correlate here, regardless we should filter and see that all three instance types - // are valid before preferring the cheapest one 'large' - cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "medium", - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("2Gi"), - }, - Offerings: []cloudprovider.Offering{ - { - CapacityType: v1alpha5.CapacityTypeOnDemand, - Zone: "test-zone-1a", - Price: 3.00, - Available: true, - }, - }, - }), - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "small", - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi"), - }, - Offerings: []cloudprovider.Offering{ - { - CapacityType: v1alpha5.CapacityTypeOnDemand, - Zone: "test-zone-1a", - Price: 2.00, - Available: true, - }, - }, - }), - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "large", - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("4"), - v1.ResourceMemory: resource.MustParse("4Gi"), - }, - Offerings: []cloudprovider.Offering{ - { - CapacityType: v1alpha5.CapacityTypeOnDemand, - Zone: "test-zone-1a", - Price: 1.00, - Available: true, - }, - }, - }), - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1m"), - v1.ResourceMemory: resource.MustParse("1Mi"), - }, - }}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - // large is the cheapest, so we should pick it, but the other two types are also valid options - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("large")) - // all three options should be passed to the cloud provider - possibleInstanceType := sets.NewString(pscheduling.NewNodeSelectorRequirements(cloudProvider.CreateCalls[0].Spec.Requirements...).Get(v1.LabelInstanceTypeStable).Values()...) - Expect(possibleInstanceType).To(Equal(sets.NewString("small", "medium", "large"))) - }) - }) - - Describe("In-Flight Nodes", func() { - It("should not launch a second node if there is an in-flight node that can support the pod", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - secondPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should not launch a second node if there is an in-flight node that can support the pod (node selectors)", func() { - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2"}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - // the node gets created in test-zone-2 - secondPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1", "test-zone-2"}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - // test-zone-2 is in the intersection of their node selectors and the node has capacity, so we shouldn't create a new node - node2 := ExpectScheduled(ctx, env.Client, secondPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - Expect(node1.Name).To(Equal(node2.Name)) - - // the node gets created in test-zone-2 - thirdPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1", "test-zone-3"}, - }}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, thirdPod) - // node is in test-zone-2, so this pod needs a new node - node3 := ExpectScheduled(ctx, env.Client, thirdPod) - Expect(node1.Name).ToNot(Equal(node3.Name)) - }) - It("should launch a second node if a pod won't fit on the existingNodes node", func() { - ExpectApplied(ctx, env.Client, provisioner) - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1001m"), - }, - }} - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - // the node will have 2000m CPU, so these two pods can't both fit on it - opts.ResourceRequirements.Limits[v1.ResourceCPU] = resource.MustParse("1") - secondPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("should launch a second node if a pod isn't compatible with the existingNodes node (node selector)", func() { - ExpectApplied(ctx, env.Client, provisioner) - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - secondPod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("should launch a second node if an in-flight node is terminating", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod(opts) - bindings := ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - ExpectScheduled(ctx, env.Client, initialPod) - - // delete the node/machine - machine1 := bindings.Get(initialPod).Machine - node1 := bindings.Get(initialPod).Node - machine1.Finalizers = nil - node1.Finalizers = nil - ExpectApplied(ctx, env.Client, machine1, node1) - ExpectDeleted(ctx, env.Client, machine1, node1) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - ExpectReconcileSucceeded(ctx, machineStateController, client.ObjectKeyFromObject(machine1)) - - secondPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - Context("Topology", func() { - It("should balance pods across zones with in-flight nodes", func() { - labels := map[string]string{"foo": "bar"} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) - - // reconcile our nodes with the cluster state so they'll show up as in-flight - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - for _, node := range nodeList.Items { - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKey{Name: node.Name}) - } - - firstRoundNumNodes := len(nodeList.Items) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 5)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 3)) - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - - // shouldn't create any new nodes as the in-flight ones can support the pods - Expect(nodeList.Items).To(HaveLen(firstRoundNumNodes)) - }) - It("should balance pods across hostnames with in-flight nodes", func() { - labels := map[string]string{"foo": "bar"} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1)) - - // reconcile our nodes with the cluster state so they'll show up as in-flight - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - for _, node := range nodeList.Items { - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKey{Name: node.Name}) - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 5)..., - ) - // we prefer to launch new nodes to satisfy the topology spread even though we could technically schedule against existingNodes - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1, 1, 1, 1, 1, 1)) - }) - }) - Context("Taints", func() { - It("should assume pod will schedule to a tainted node with no taints", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("8"), - }, - }} - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - node1.Spec.Taints = nil - ExpectApplied(ctx, env.Client, node1) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - secondPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should not assume pod will schedule to a tainted node", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("8"), - }, - }} - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod(opts) - bindings := ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - ExpectScheduled(ctx, env.Client, initialPod) - - machine1 := bindings.Get(initialPod).Machine - node1 := bindings.Get(initialPod).Node - machine1.StatusConditions().MarkTrue(v1alpha5.MachineInitialized) - node1.Labels = lo.Assign(node1.Labels, map[string]string{v1alpha5.LabelNodeInitialized: "true"}) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - // and taint it - node1.Spec.Taints = append(node1.Spec.Taints, v1.Taint{ - Key: "foo.com/taint", - Value: "tainted", - Effect: v1.TaintEffectNoSchedule, - }) - ExpectApplied(ctx, env.Client, machine1, node1) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - secondPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("should assume pod will schedule to a tainted node with a custom startup taint", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("8"), - }, - }} - provisioner.Spec.StartupTaints = append(provisioner.Spec.StartupTaints, v1.Taint{ - Key: "foo.com/taint", - Value: "tainted", - Effect: v1.TaintEffectNoSchedule, - }) - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - // startup taint + node not ready taint = 2 - Expect(node1.Spec.Taints).To(HaveLen(2)) - Expect(node1.Spec.Taints).To(ContainElement(v1.Taint{ - Key: "foo.com/taint", - Value: "tainted", - Effect: v1.TaintEffectNoSchedule, - })) - ExpectApplied(ctx, env.Client, node1) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - secondPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should not assume pod will schedule to a node with startup taints after initialization", func() { - startupTaint := v1.Taint{Key: "ignore-me", Value: "nothing-to-see-here", Effect: v1.TaintEffectNoSchedule} - provisioner.Spec.StartupTaints = []v1.Taint{startupTaint} - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod() - bindings := ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - ExpectScheduled(ctx, env.Client, initialPod) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - - // Mark it initialized which only occurs once the startup taint was removed and re-apply only the startup taint. - // We also need to add resource capacity as after initialization we assume that kubelet has recorded them. - - machine1 := bindings.Get(initialPod).Machine - node1 := bindings.Get(initialPod).Node - machine1.StatusConditions().MarkTrue(v1alpha5.MachineInitialized) - node1.Labels = lo.Assign(node1.Labels, map[string]string{v1alpha5.LabelNodeInitialized: "true"}) - - node1.Spec.Taints = []v1.Taint{startupTaint} - node1.Status.Capacity = v1.ResourceList{v1.ResourcePods: resource.MustParse("10")} - ExpectApplied(ctx, env.Client, machine1, node1) - - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - // we should launch a new node since the startup taint is there, but was gone at some point - secondPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - It("should consider a tainted NotReady node as in-flight even if initialized", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: resource.MustParse("10m")}, - }} - ExpectApplied(ctx, env.Client, provisioner) - - // Schedule to New Machine - pod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node1 := ExpectScheduled(ctx, env.Client, pod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - // Mark Initialized - node1.Labels[v1alpha5.LabelNodeInitialized] = "true" - node1.Spec.Taints = []v1.Taint{ - {Key: v1.TaintNodeNotReady, Effect: v1.TaintEffectNoSchedule}, - {Key: v1.TaintNodeUnreachable, Effect: v1.TaintEffectNoSchedule}, - {Key: cloudproviderapi.TaintExternalCloudProvider, Effect: v1.TaintEffectNoSchedule, Value: "true"}, - } - ExpectApplied(ctx, env.Client, node1) - // Schedule to In Flight Machine - pod = test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) - - Expect(node1.Name).To(Equal(node2.Name)) - }) - }) - Context("Daemonsets", func() { - It("should track daemonset usage separately so we know how many DS resources are remaining to be scheduled", func() { - ds := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - ) - ExpectApplied(ctx, env.Client, provisioner, ds) - Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(ds), ds)).To(Succeed()) - - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("8"), - }, - }} - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - - // create our daemonset pod and manually bind it to the node - dsPod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("2Gi"), - }}, - }) - dsPod.OwnerReferences = append(dsPod.OwnerReferences, metav1.OwnerReference{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - Name: ds.Name, - UID: ds.UID, - Controller: ptr.Bool(true), - BlockOwnerDeletion: ptr.Bool(true), - }) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - ExpectApplied(ctx, env.Client, provisioner, dsPod) - cluster.ForEachNode(func(f *state.StateNode) bool { - dsRequests := f.DaemonSetRequests() - available := f.Available() - Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 0)) - // no pods so we have the full (16 cpu - 100m overhead) - Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 15.9)) - return true - }) - ExpectManualBinding(ctx, env.Client, dsPod, node1) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(dsPod)) - - cluster.ForEachNode(func(f *state.StateNode) bool { - dsRequests := f.DaemonSetRequests() - available := f.Available() - Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 1)) - // only the DS pod is bound, so available is reduced by one and the DS requested is incremented by one - Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 14.9)) - return true - }) - - opts = test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("14.9"), - }, - }} - // this pod should schedule on the existingNodes node as the daemonset pod has already bound, meaning that the - // remaining daemonset resources should be zero leaving 14.9 CPUs for the pod - secondPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - Expect(node1.Name).To(Equal(node2.Name)) - }) - It("should handle unexpected daemonset pods binding to the node", func() { - ds1 := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - NodeSelector: map[string]string{ - "my-node-label": "value", - }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi")}}, - }}, - ) - ds2 := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1m"), - }}}}) - ExpectApplied(ctx, env.Client, provisioner, ds1, ds2) - Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(ds1), ds1)).To(Succeed()) - - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("8"), - }, - }} - initialPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node1 := ExpectScheduled(ctx, env.Client, initialPod) - // this label appears on the node for some reason that Karpenter can't track - node1.Labels["my-node-label"] = "value" - ExpectApplied(ctx, env.Client, node1) - - // create our daemonset pod and manually bind it to the node - dsPod := test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{ - "my-node-label": "value", - }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("2Gi"), - }}, - }) - dsPod.OwnerReferences = append(dsPod.OwnerReferences, metav1.OwnerReference{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - Name: ds1.Name, - UID: ds1.UID, - Controller: ptr.Bool(true), - BlockOwnerDeletion: ptr.Bool(true), - }) - - // delete the pod so that the node is empty - ExpectDeleted(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - - ExpectApplied(ctx, env.Client, provisioner, dsPod) - cluster.ForEachNode(func(f *state.StateNode) bool { - dsRequests := f.DaemonSetRequests() - available := f.Available() - Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 0)) - // no pods, so we have the full (16 CPU - 100m overhead) - Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 15.9)) - return true - }) - ExpectManualBinding(ctx, env.Client, dsPod, node1) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(dsPod)) - - cluster.ForEachNode(func(f *state.StateNode) bool { - dsRequests := f.DaemonSetRequests() - available := f.Available() - Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 1)) - // only the DS pod is bound, so available is reduced by one and the DS requested is incremented by one - Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 14.9)) - return true - }) - - opts = test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("15.5"), - }, - }} - // This pod should not schedule on the inflight node as it requires more CPU than we have. This verifies - // we don't reintroduce a bug where more daemonsets scheduled than anticipated due to unexepected labels - // appearing on the node which caused us to compute a negative amount of resources remaining for daemonsets - // which in turn caused us to mis-calculate the amount of resources that were free on the node. - secondPod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - node2 := ExpectScheduled(ctx, env.Client, secondPod) - // must create a new node - Expect(node1.Name).ToNot(Equal(node2.Name)) - }) - }) - // nolint:gosec - It("should pack in-flight nodes before launching new nodes", func() { - cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "medium", - Resources: v1.ResourceList{ - // enough CPU for four pods + a bit of overhead - v1.ResourceCPU: resource.MustParse("4.25"), - v1.ResourcePods: resource.MustParse("4"), - }, - }), - } - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1"), - }, - }} - - ExpectApplied(ctx, env.Client, provisioner) - - // scheduling in multiple batches random sets of pods - for i := 0; i < 10; i++ { - initialPods := test.UnschedulablePods(opts, rand.Intn(10)) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPods...) - for _, pod := range initialPods { - node := ExpectScheduled(ctx, env.Client, pod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - } - } - - // due to the in-flight node support, we should pack existing nodes before launching new node. The end result - // is that we should only have some spare capacity on our final node - nodesWithCPUFree := 0 - cluster.ForEachNode(func(n *state.StateNode) bool { - available := n.Available() - if available.Cpu().AsApproximateFloat64() >= 1 { - nodesWithCPUFree++ - } - return true - }) - Expect(nodesWithCPUFree).To(BeNumerically("<=", 1)) - }) - It("should not launch a second node if there is an in-flight node that can support the pod (#2011)", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - - // there was a bug in cluster state where we failed to identify the instance type resources when using a - // MachineTemplateRef so modify our provisioner to use the MachineTemplateRef and ensure that the second pod schedules - // to the existingNodes node - provisioner.Spec.Provider = nil - provisioner.Spec.ProviderRef = &v1alpha5.MachineTemplateRef{} - - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(opts) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pod) - var nodes v1.NodeList - Expect(env.Client.List(ctx, &nodes)).To(Succeed()) - Expect(nodes.Items).To(HaveLen(1)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodes.Items[0])) - - pod.Status.Conditions = []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}} - ExpectApplied(ctx, env.Client, pod) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pod) - Expect(env.Client.List(ctx, &nodes)).To(Succeed()) - // shouldn't create a second node - Expect(nodes.Items).To(HaveLen(1)) - }) - It("should order initialized nodes for scheduling un-initialized nodes when all other nodes are inflight", func() { - ExpectApplied(ctx, env.Client, provisioner) - - var machines []*v1alpha5.Machine - var node *v1.Node - //nolint:gosec - elem := rand.Intn(100) // The machine/node that will be marked as initialized - for i := 0; i < 100; i++ { - m := test.Machine(v1alpha5.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1alpha5.ProvisionerNameLabelKey: provisioner.Name, - }, - }, - }) - ExpectApplied(ctx, env.Client, m) - if i == elem { - m, node = ExpectMachineDeployed(ctx, env.Client, cluster, cloudProvider, m) - } else { - var err error - m, err = ExpectMachineDeployedNoNode(ctx, env.Client, cluster, cloudProvider, m) - Expect(err).ToNot(HaveOccurred()) - } - machines = append(machines, m) - } - - // Make one of the nodes and machines initialized - ExpectMakeMachinesInitialized(ctx, env.Client, machines[elem]) - ExpectMakeNodesInitialized(ctx, env.Client, node) - ExpectReconcileSucceeded(ctx, machineStateController, client.ObjectKeyFromObject(machines[elem])) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - scheduledNode := ExpectScheduled(ctx, env.Client, pod) - - // Expect that the scheduled node is equal to node3 since it's initialized - Expect(scheduledNode.Name).To(Equal(node.Name)) - }) - }) - - Describe("Existing Nodes", func() { - It("should schedule a pod to an existing node unowned by Karpenter", func() { - node := test.Node(test.NodeOptions{ - Allocatable: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10"), - v1.ResourceMemory: resource.MustParse("10Gi"), - v1.ResourcePods: resource.MustParse("110"), - }, - }) - ExpectApplied(ctx, env.Client, node) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(opts) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - scheduledNode := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).To(Equal(scheduledNode.Name)) - }) - It("should schedule multiple pods to an existing node unowned by Karpenter", func() { - node := test.Node(test.NodeOptions{ - Allocatable: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10"), - v1.ResourceMemory: resource.MustParse("100Gi"), - v1.ResourcePods: resource.MustParse("110"), - }, - }) - ExpectApplied(ctx, env.Client, node) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - ExpectApplied(ctx, env.Client, provisioner) - pods := test.UnschedulablePods(opts, 100) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - - for _, pod := range pods { - scheduledNode := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).To(Equal(scheduledNode.Name)) - } - }) - It("should order initialized nodes for scheduling un-initialized nodes", func() { - ExpectApplied(ctx, env.Client, provisioner) - - var machines []*v1alpha5.Machine - var nodes []*v1.Node - for i := 0; i < 100; i++ { - m := test.Machine(v1alpha5.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1alpha5.ProvisionerNameLabelKey: provisioner.Name, - }, - }, - }) - ExpectApplied(ctx, env.Client, m) - m, n := ExpectMachineDeployed(ctx, env.Client, cluster, cloudProvider, m) - machines = append(machines, m) - nodes = append(nodes, n) - } - - // Make one of the nodes and machines initialized - elem := rand.Intn(100) //nolint:gosec - ExpectMakeMachinesInitialized(ctx, env.Client, machines[elem]) - ExpectMakeNodesInitialized(ctx, env.Client, nodes[elem]) - ExpectReconcileSucceeded(ctx, machineStateController, client.ObjectKeyFromObject(machines[elem])) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(nodes[elem])) - - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - scheduledNode := ExpectScheduled(ctx, env.Client, pod) - - // Expect that the scheduled node is equal to the ready node since it's initialized - Expect(scheduledNode.Name).To(Equal(nodes[elem].Name)) - }) - It("should consider a pod incompatible with an existing node but compatible with Provisioner", func() { - machine, node := test.MachineAndNode(v1alpha5.Machine{ - Status: v1alpha5.MachineStatus{ - Allocatable: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10"), - v1.ResourceMemory: resource.MustParse("10Gi"), - v1.ResourcePods: resource.MustParse("110"), - }, - }, - }) - ExpectApplied(ctx, env.Client, machine, node) - ExpectMakeMachinesInitialized(ctx, env.Client, machine) - ExpectMakeNodesInitialized(ctx, env.Client, node) - - ExpectReconcileSucceeded(ctx, machineStateController, client.ObjectKeyFromObject(machine)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1"}, - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - Context("Daemonsets", func() { - It("should not subtract daemonset overhead that is not strictly compatible with an existing node", func() { - machine, node := test.MachineAndNode(v1alpha5.Machine{ - Status: v1alpha5.MachineStatus{ - Allocatable: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourcePods: resource.MustParse("110"), - }, - }, - }) - // This DaemonSet is not compatible with the existing Machine/Node - ds := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100"), - v1.ResourceMemory: resource.MustParse("100Gi")}, - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1"}, - }, - }, - }}, - ) - ExpectApplied(ctx, env.Client, provisioner, machine, node, ds) - ExpectMakeMachinesInitialized(ctx, env.Client, machine) - ExpectMakeNodesInitialized(ctx, env.Client, node) - - ExpectReconcileSucceeded(ctx, machineStateController, client.ObjectKeyFromObject(machine)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi")}, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - scheduledNode := ExpectScheduled(ctx, env.Client, pod) - Expect(scheduledNode.Name).To(Equal(node.Name)) - - // Add another pod and expect that pod not to schedule against a Provisioner since we will model the DS against the Provisioner - // In this case, the DS overhead will take over the entire capacity for every "theoretical node" so we can't schedule a new pod to any new Node - pod2 := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi")}, - }, - }) - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod2) - ExpectNotScheduled(ctx, env.Client, pod2) - }) - }) - }) - - Describe("No Pre-Binding", func() { - It("should not bind pods to nodes", func() { - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - }, - }} - - var nodeList v1.NodeList - // shouldn't have any nodes - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(0)) - - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod(opts) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - ExpectNotScheduled(ctx, env.Client, initialPod) - - // should launch a single node - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(1)) - node1 := &nodeList.Items[0] - - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - secondPod := test.UnschedulablePod(opts) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - ExpectNotScheduled(ctx, env.Client, secondPod) - // shouldn't create a second node as it can bind to the existingNodes node - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(1)) - }) - It("should handle resource zeroing of extended resources by kubelet", func() { - // Issue #1459 - opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("10m"), - fake.ResourceGPUVendorA: resource.MustParse("1"), - }, - }} - - var nodeList v1.NodeList - // shouldn't have any nodes - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(0)) - - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod(opts) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - ExpectNotScheduled(ctx, env.Client, initialPod) - - // should launch a single node - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(1)) - node1 := &nodeList.Items[0] - - // simulate kubelet zeroing out the extended resources on the node at startup - node1.Status.Capacity = map[v1.ResourceName]resource.Quantity{ - fake.ResourceGPUVendorA: resource.MustParse("0"), - } - node1.Status.Allocatable = map[v1.ResourceName]resource.Quantity{ - fake.ResourceGPUVendorB: resource.MustParse("0"), - } - - ExpectApplied(ctx, env.Client, node1) - - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - secondPod := test.UnschedulablePod(opts) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, secondPod) - ExpectNotScheduled(ctx, env.Client, secondPod) - // shouldn't create a second node as it can bind to the existingNodes node - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(1)) - }) - It("should respect self pod affinity without pod binding (zone)", func() { - // Issue #1975 - affLabels := map[string]string{"security": "s2"} - - pods := test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: affLabels, - }, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }}, - }, 2) - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pods[0]) - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - for i := range nodeList.Items { - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodeList.Items[i])) - } - // the second pod can schedule against the in-flight node, but for that to work we need to be careful - // in how we fulfill the self-affinity by taking the existing node's domain as a preference over any - // random viable domain - ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pods[1]) - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(1)) - }) - }) - - Describe("VolumeUsage", func() { - BeforeEach(func() { - cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ - fake.NewInstanceType( - fake.InstanceTypeOptions{ - Name: "instance-type", - Resources: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1024"), - v1.ResourcePods: resource.MustParse("1024"), - }, - }), - } - provisioner.Spec.Limits = nil - }) - It("should launch multiple nodes if required due to volume limits", func() { - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: csiProvider, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(10), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, - Provisioner: ptr.String(csiProvider), - Zones: []string{"test-zone-1"}}) - ExpectApplied(ctx, env.Client, sc) - - var pods []*v1.Pod - for i := 0; i < 6; i++ { - pvcA := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - StorageClassName: ptr.String("my-storage-class"), - ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("my-claim-a-%d", i)}, - }) - pvcB := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - StorageClassName: ptr.String("my-storage-class"), - ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("my-claim-b-%d", i)}, - }) - ExpectApplied(ctx, env.Client, pvcA, pvcB) - pods = append(pods, test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{pvcA.Name, pvcB.Name}, - })) - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - // we need to create a new node as the in-flight one can only contain 5 pods due to the CSINode volume limit - Expect(nodeList.Items).To(HaveLen(2)) - }) - It("should launch a single node if all pods use the same PVC", func() { - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: csiProvider, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(10), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, - Provisioner: ptr.String(csiProvider), - Zones: []string{"test-zone-1"}}) - ExpectApplied(ctx, env.Client, sc) - - pv := test.PersistentVolume(test.PersistentVolumeOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, - Zones: []string{"test-zone-1"}}) - - pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, - StorageClassName: ptr.String("my-storage-class"), - VolumeName: pv.Name, - }) - ExpectApplied(ctx, env.Client, pv, pvc) - - var pods []*v1.Pod - for i := 0; i < 100; i++ { - pods = append(pods, test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{pvc.Name}, - })) - } - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - // 100 of the same PVC should all be schedulable on the same node - Expect(nodeList.Items).To(HaveLen(1)) - }) - It("should not fail for NFS volumes", func() { - ExpectApplied(ctx, env.Client, provisioner) - initialPod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pv := test.PersistentVolume(test.PersistentVolumeOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, - StorageClassName: "nfs", - Zones: []string{"test-zone-1"}}) - pv.Spec.NFS = &v1.NFSVolumeSource{ - Server: "fake.server", - Path: "/some/path", - } - pv.Spec.CSI = nil - - pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, - VolumeName: pv.Name, - StorageClassName: ptr.String(""), - }) - ExpectApplied(ctx, env.Client, pv, pvc) - - var pods []*v1.Pod - for i := 0; i < 5; i++ { - pods = append(pods, test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{pvc.Name, pvc.Name}, - })) - } - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - // 5 of the same PVC should all be schedulable on the same node - Expect(nodeList.Items).To(HaveLen(1)) - }) - It("should launch nodes for pods with ephemeral volume using the specified storage class name", func() { - // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-storage-class", - }, - Provisioner: ptr.String(csiProvider), - Zones: []string{"test-zone-1"}}) - // Create another default storage class that shouldn't be used and has no associated limits - sc2 := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String("other-provider"), - Zones: []string{"test-zone-1"}}) - - initialPod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has a specified storage class, so it should use the one specified - initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - StorageClassName: lo.ToPtr(sc.Name), - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, provisioner, sc, sc2, initialPod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: csiProvider, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(1), - }, - }, - { - Name: "other-provider", - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(10), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has a specified storage class, so it should use the one specified - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - StorageClassName: lo.ToPtr(sc.Name), - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, provisioner, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).ToNot(Equal(node2.Name)) - }) - It("should launch nodes for pods with ephemeral volume using a default storage class", func() { - // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String(csiProvider), - Zones: []string{"test-zone-1"}}) - - initialPod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one - initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, provisioner, sc, initialPod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: csiProvider, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(1), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - - ExpectApplied(ctx, env.Client, sc, provisioner, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).ToNot(Equal(node2.Name)) - }) - It("should launch nodes for pods with ephemeral volume using the newest storage class", func() { - if env.Version.Minor() < 26 { - Skip("Multiple default storage classes is only available in K8s >= 1.26.x") - } - // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String("other-provider"), - Zones: []string{"test-zone-1"}}) - sc2 := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "newer-default-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String(csiProvider), - Zones: []string{"test-zone-1"}}) - - ExpectApplied(ctx, env.Client, sc) - // Wait a few seconds to apply the second storage class to get a newer creationTimestamp - time.Sleep(time.Second * 2) - ExpectApplied(ctx, env.Client, sc2) - - initialPod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one - initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, provisioner, sc, initialPod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: csiProvider, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(1), - }, - }, - { - Name: "other-provider", - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(10), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, sc, provisioner, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).ToNot(Equal(node2.Name)) - }) - It("should not launch nodes for pods with ephemeral volume using a non-existent storage classes", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{}) - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - StorageClassName: ptr.String("non-existent"), - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - var nodeList v1.NodeList - Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) - // no nodes should be created as the storage class doesn't eixst - Expect(nodeList.Items).To(HaveLen(0)) - }) - Context("CSIMigration", func() { - It("should launch nodes for pods with non-dynamic PVC using a migrated PVC/PV", func() { - // We should assume that this PVC/PV is using CSI driver implicitly to limit pod scheduling - // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "in-tree-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String(plugins.AWSEBSInTreePluginName), - Zones: []string{"test-zone-1"}}) - pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - StorageClassName: ptr.String(sc.Name), - }) - ExpectApplied(ctx, env.Client, provisioner, sc, pvc) - initialPod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{pvc.Name}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: plugins.AWSEBSDriverName, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(1), - }, - }, - }, - }, - } - pv := test.PersistentVolume(test.PersistentVolumeOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-volume", - }, - Zones: []string{"test-zone-1"}, - UseAWSInTreeDriver: true, - }) - ExpectApplied(ctx, env.Client, csiNode, pvc, pv) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pvc2 := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - StorageClassName: ptr.String(sc.Name), - }) - pod := test.UnschedulablePod(test.PodOptions{ - PersistentVolumeClaims: []string{pvc2.Name}, - }) - ExpectApplied(ctx, env.Client, pvc2, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).ToNot(Equal(node2.Name)) - }) - It("should launch nodes for pods with ephemeral volume using a migrated PVC/PV", func() { - // We should assume that this PVC/PV is using CSI driver implicitly to limit pod scheduling - // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "in-tree-storage-class", - Annotations: map[string]string{ - pscheduling.IsDefaultStorageClassAnnotation: "true", - }, - }, - Provisioner: ptr.String(plugins.AWSEBSInTreePluginName), - Zones: []string{"test-zone-1"}}) - - initialPod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that references the in-tree storage provider - initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - StorageClassName: ptr.String(sc.Name), - }, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, provisioner, sc, initialPod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) - node := ExpectScheduled(ctx, env.Client, initialPod) - csiNode := &storagev1.CSINode{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - }, - Spec: storagev1.CSINodeSpec{ - Drivers: []storagev1.CSINodeDriver{ - { - Name: plugins.AWSEBSDriverName, - NodeID: "fake-node-id", - Allocatable: &storagev1.VolumeNodeResources{ - Count: ptr.Int32(1), - }, - }, - }, - }, - } - ExpectApplied(ctx, env.Client, csiNode) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) - - pod := test.UnschedulablePod(test.PodOptions{}) - // Pod has an ephemeral volume claim that reference the in-tree storage provider - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "tmp-ephemeral", - VolumeSource: v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - StorageClassName: ptr.String(sc.Name), - }, - }, - }, - }, - }) - // Pod should not schedule to the first node since we should realize that we have hit our volume limits - ExpectApplied(ctx, env.Client, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node2 := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Name).ToNot(Equal(node2.Name)) - }) - }) - }) -}) diff --git a/pkg/controllers/provisioning/scheduling/provisioner_topology_test.go b/pkg/controllers/provisioning/scheduling/provisioner_topology_test.go deleted file mode 100644 index 4466ac732f..0000000000 --- a/pkg/controllers/provisioning/scheduling/provisioner_topology_test.go +++ /dev/null @@ -1,2430 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheduling_test - -import ( - "time" - - . "github.com/onsi/gomega" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" - - . "github.com/onsi/ginkgo/v2" - - "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter-core/pkg/cloudprovider/fake" - "github.com/aws/karpenter-core/pkg/test" - . "github.com/aws/karpenter-core/pkg/test/expectations" -) - -var _ = Describe("Topology", func() { - var provisioner *v1alpha5.Provisioner - labels := map[string]string{"test": "test"} - BeforeEach(func() { - provisioner = test.Provisioner(test.ProvisionerOptions{Requirements: []v1.NodeSelectorRequirement{{ - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeSpot, v1alpha5.CapacityTypeOnDemand}, - }}}) - }) - - It("should ignore unknown topology keys", func() { - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod( - test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: []v1.TopologySpreadConstraint{{ - TopologyKey: "unknown", - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }}}, - ), - test.UnschedulablePod(), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - ExpectNotScheduled(ctx, env.Client, pods[0]) - ExpectScheduled(ctx, env.Client, pods[1]) - }) - - It("should not spread an invalid label selector", func() { - if env.Version.Minor() >= 24 { - Skip("Invalid label selector now is denied by admission in K8s >= 1.27.x") - } - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "{{ zqfmgb }}"}}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 2)...) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2)) - }) - - It("should ignore pods if node does not exist", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - podAwaitingGC := test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology, NodeName: "does-not-exist"}) - ExpectApplied(ctx, env.Client, provisioner, podAwaitingGC) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) - }) - - Context("Zonal", func() { - It("should balance pods across zones (match labels)", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) - }) - It("should balance pods across zones (match expressions)", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "test", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"test"}, - }, - }, - }, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) - }) - It("should respect provisioner zonal constraints", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) - }) - It("should respect provisioner zonal constraints (subset) with requirements", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - // should spread the two pods evenly across the only valid zones in our universe (the two zones from our single provisioner) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) - }) - It("should respect provisioner zonal constraints (subset) with labels", func() { - provisioner.Spec.Labels = lo.Assign(provisioner.Spec.Labels, map[string]string{v1.LabelTopologyZone: "test-zone-1"}) - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - // should spread the two pods evenly across the only valid zones in our universe (the two zones from our single nodePool) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4)) - }) - It("should respect provisioner zonal constraints (subset) with labels and requirements", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - provisioner.Spec.Labels = lo.Assign(provisioner.Spec.Labels, map[string]string{v1.LabelTopologyZone: "test-zone-1"}) - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - // should spread the two pods evenly across the only valid zones in our universe (the two zones from our single nodePool) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4)) - }) - It("should respect provisioner zonal constraints (subset) with labels across NodePools", func() { - provisioner.Spec.Labels = lo.Assign(provisioner.Spec.Labels, map[string]string{v1.LabelTopologyZone: "test-zone-1"}) - provisioner2 := test.Provisioner(test.ProvisionerOptions{ - Labels: map[string]string{ - v1.LabelTopologyZone: "test-zone-2", - }, - }) - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner, provisioner2) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - // should spread the two pods evenly across the only valid zones in our universe (the two zones from our single nodePool) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) - }) - It("should respect provisioner zonal constraints (existing pod)", func() { - ExpectApplied(ctx, env.Client, provisioner) - // need enough resource requests that the first node we create fills a node and can't act as an in-flight - // node for the other pods - rr := v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1.1"), - }, - } - pod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, - NodeSelector: map[string]string{ - v1.LabelTopologyZone: "test-zone-3", - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}, 6)..., - ) - // we should have unschedulable pods now, the provisioner can only schedule to zone-1/zone-2, but because of the existing - // pod in zone-3 it can put a max of two per zone before it would violate max skew - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2, 2)) - }) - It("should schedule to the non-minimum domain if its all that's available", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 5, - }} - rr := v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1.1"), - }, - } - // force this pod onto zone-1 - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, TopologySpreadConstraints: topology})) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) - - // force this pod onto zone-2 - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, TopologySpreadConstraints: topology})) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) - - // now only allow scheduling pods on zone-3 - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, TopologySpreadConstraints: topology}, 10)..., - ) - - // max skew of 5, so test-zone-1/2 will have 1 pod each, test-zone-3 will have 6, and the rest will fail to schedule - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 6)) - }) - It("should only schedule to minimum domains if already violating max skew", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - rr := v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1.1"), - }, - } - createPods := func(count int) []*v1.Pod { - var pods []*v1.Pod - for i := 0; i < count; i++ { - pods = append(pods, test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, TopologySpreadConstraints: topology})) - } - return pods - } - // Spread 9 pods - ExpectApplied(ctx, env.Client, provisioner) - pods := createPods(9) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 3)) - - // Delete pods to create a skew - for _, pod := range pods { - node := ExpectScheduled(ctx, env.Client, pod) - if node.Labels[v1.LabelTopologyZone] != "test-zone-1" { - ExpectDeleted(ctx, env.Client, pod) - } - } - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3)) - - // Create 3 more pods, skew should recover - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, createPods(3)...) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 1, 2)) - }) - It("should not violate max-skew when unsat = do not schedule", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - rr := v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1.1"), - }, - } - // force this pod onto zone-1 - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, TopologySpreadConstraints: topology})) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) - - // now only allow scheduling pods on zone-2 and zone-3 - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "test-zone-3"}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, TopologySpreadConstraints: topology}, 10)..., - ) - - // max skew of 1, so test-zone-2/3 will have 2 nodes each and the rest of the pods will fail to schedule - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2, 2)) - }) - It("should not violate max-skew when unsat = do not schedule (discover domains)", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - rr := v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1.1"), - }, - } - // force this pod onto zone-1 - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr})) - - // now only allow scheduling pods on zone-2 and zone-3 - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "test-zone-3"}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, ResourceRequirements: rr}, 10)..., - ) - - // max skew of 1, so test-zone-2/3 will have 2 nodes each and the rest of the pods will fail to schedule since - // test-zone-1 has 1 pods in it. - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2, 2)) - }) - It("should only count running/scheduled pods with matching labels scheduled to nodes with a corresponding domain", func() { - wrongNamespace := test.RandomName() - firstNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}}) - secondNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}}) - thirdNode := test.Node(test.NodeOptions{}) // missing topology domain - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner, firstNode, secondNode, thirdNode, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: wrongNamespace}}) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(firstNode)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(secondNode)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(thirdNode)) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.Pod(test.PodOptions{NodeName: firstNode.Name}), // ignored, missing labels - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}}), // ignored, pending - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: thirdNode.Name}), // ignored, no domain on node - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, Namespace: wrongNamespace}, NodeName: firstNode.Name}), // ignored, wrong namespace - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, DeletionTimestamp: &metav1.Time{Time: time.Now().Add(10 * time.Second)}}}), // ignored, terminating - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodFailed}), // ignored, phase=Failed - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodSucceeded}), // ignored, phase=Succeeded - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: secondNode.Name}), - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), - ) - nodes := v1.NodeList{} - Expect(env.Client.List(ctx, &nodes)).To(Succeed()) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2, 1)) - }) - It("should match all pods when labelSelector is not specified", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePod(), - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) - }) - It("should handle interdependent selectors", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - pods := test.UnschedulablePods(test.PodOptions{TopologySpreadConstraints: topology}, 5) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - pods..., - ) - // This is weird, but the topology label selector is used for determining domain counts. The pod that - // owns the topology is what the spread actually applies to. In this test case, there are no pods matching - // the label selector, so the max skew is zero. This means we can pack all the pods onto the same node since - // it doesn't violate the topology spread constraint (i.e. adding new pods doesn't increase skew since the - // pods we are adding don't count toward skew). This behavior is called out at - // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ , though it's not - // recommended for users. - nodeNames := sets.NewString() - for _, p := range pods { - nodeNames.Insert(p.Spec.NodeName) - } - Expect(nodeNames).To(HaveLen(1)) - }) - It("should respect minDomains constraints", func() { - if env.Version.Minor() < 24 { - Skip("MinDomains TopologySpreadConstraint is only available starting in K8s >= 1.24.x") - } - var minDomains int32 = 3 - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - MinDomains: &minDomains, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 3)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) - }) - It("satisfied minDomains constraints (equal) should allow expected pod scheduling", func() { - if env.Version.Minor() < 24 { - Skip("MinDomains TopologySpreadConstraint is only available starting in K8s >= 1.24.x") - } - var minDomains int32 = 3 - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - MinDomains: &minDomains, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 11)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 4, 3)) - }) - It("satisfied minDomains constraints (greater than minimum) should allow expected pod scheduling", func() { - if env.Version.Minor() < 24 { - Skip("MinDomains TopologySpreadConstraint is only available starting in K8s >= 1.24.x") - } - var minDomains int32 = 2 - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - MinDomains: &minDomains, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 11)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 4, 3)) - }) - }) - - Context("Hostname", func() { - It("should balance pods across nodes", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1)) - }) - It("should balance pods on the same hostname up to maxskew", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 4, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4)) - }) - It("balance multiple deployments with hostname topology spread", func() { - // Issue #1425 - spreadPod := func(appName string) test.PodOptions { - return test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": appName, - }, - }, - TopologySpreadConstraints: []v1.TopologySpreadConstraint{ - { - MaxSkew: 1, - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": appName}, - }, - }, - }, - } - } - - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod(spreadPod("app1")), test.UnschedulablePod(spreadPod("app1")), - test.UnschedulablePod(spreadPod("app2")), test.UnschedulablePod(spreadPod("app2")), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, p := range pods { - ExpectScheduled(ctx, env.Client, p) - } - nodes := v1.NodeList{} - Expect(env.Client.List(ctx, &nodes)).To(Succeed()) - // this wasn't part of #1425, but ensures that we launch the minimum number of nodes - Expect(nodes.Items).To(HaveLen(2)) - }) - It("balance multiple deployments with hostname topology spread & varying arch", func() { - // Issue #1425 - spreadPod := func(appName, arch string) test.PodOptions { - return test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": appName, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{arch}, - }, - }, - TopologySpreadConstraints: []v1.TopologySpreadConstraint{ - { - MaxSkew: 1, - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": appName}, - }, - }, - }, - } - } - - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - test.UnschedulablePod(spreadPod("app1", v1alpha5.ArchitectureAmd64)), test.UnschedulablePod(spreadPod("app1", v1alpha5.ArchitectureAmd64)), - test.UnschedulablePod(spreadPod("app2", v1alpha5.ArchitectureArm64)), test.UnschedulablePod(spreadPod("app2", v1alpha5.ArchitectureArm64)), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, p := range pods { - ExpectScheduled(ctx, env.Client, p) - } - nodes := v1.NodeList{} - Expect(env.Client.List(ctx, &nodes)).To(Succeed()) - // same test as the previous one, but now the architectures are different so we need four nodes in total - Expect(nodes.Items).To(HaveLen(4)) - }) - }) - - Context("CapacityType", func() { - It("should balance pods across capacity types", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) - }) - It("should respect provisioner capacity type constraints", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot, v1alpha5.CapacityTypeOnDemand}}} - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) - }) - It("should not violate max-skew when unsat = do not schedule (capacity type)", func() { - // this test can pass in a flaky manner if we don't restrict our min domain selection to valid choices - // per the provisioner spec - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - rr := v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1.1"), - }, - } - // force this pod onto spot - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, TopologySpreadConstraints: topology})) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) - - // now only allow scheduling pods on on-demand - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, TopologySpreadConstraints: topology}, 5)..., - ) - - // max skew of 1, so on-demand will have 2 pods and the rest of the pods will fail to schedule - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2)) - }) - It("should violate max-skew when unsat = schedule anyway (capacity type)", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.ScheduleAnyway, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - rr := v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1.1"), - }, - } - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, TopologySpreadConstraints: topology})) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) - - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand}}} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, TopologySpreadConstraints: topology}, 5)..., - ) - - // max skew of 1, on-demand will end up with 5 pods even though spot has a single pod - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 5)) - }) - It("should only count running/scheduled pods with matching labels scheduled to nodes with a corresponding domain", func() { - wrongNamespace := test.RandomName() - firstNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeSpot}}}) - secondNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeOnDemand}}}) - thirdNode := test.Node(test.NodeOptions{}) // missing topology capacity type - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner, firstNode, secondNode, thirdNode, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: wrongNamespace}}) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(firstNode)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(secondNode)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(thirdNode)) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.Pod(test.PodOptions{NodeName: firstNode.Name}), // ignored, missing labels - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}}), // ignored, pending - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: thirdNode.Name}), // ignored, no domain on node - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, Namespace: wrongNamespace}, NodeName: firstNode.Name}), // ignored, wrong namespace - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, DeletionTimestamp: &metav1.Time{Time: time.Now().Add(10 * time.Second)}}}), // ignored, terminating - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodFailed}), // ignored, phase=Failed - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodSucceeded}), // ignored, phase=Succeeded - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), - test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: secondNode.Name}), - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), - ) - nodes := v1.NodeList{} - Expect(env.Client.List(ctx, &nodes)).To(Succeed()) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 3)) - }) - It("should match all pods when labelSelector is not specified", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePod(), - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) - }) - It("should handle interdependent selectors", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - pods := test.UnschedulablePods(test.PodOptions{TopologySpreadConstraints: topology}, 5) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - // This is weird, but the topology label selector is used for determining domain counts. The pod that - // owns the topology is what the spread actually applies to. In this test case, there are no pods matching - // the label selector, so the max skew is zero. This means we can pack all the pods onto the same node since - // it doesn't violate the topology spread constraint (i.e. adding new pods doesn't increase skew since the - // pods we are adding don't count toward skew). This behavior is called out at - // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ , though it's not - // recommended for users. - nodeNames := sets.NewString() - for _, p := range pods { - nodeNames.Insert(p.Spec.NodeName) - } - Expect(nodeNames).To(HaveLen(1)) - }) - It("should balance pods across capacity-types (node required affinity constrained)", func() { - ExpectApplied(ctx, env.Client, provisioner) - pods := test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - NodeRequirements: []v1.NodeSelectorRequirement{ - // launch this on-demand pod in zone-1 - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, - {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{"on-demand"}}, - }, - }, 1) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - ExpectScheduled(ctx, env.Client, pods[0]) - - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - // Try to run 5 pods, with a node selector restricted to test-zone-2, they should all schedule on the same - // spot node. This doesn't violate the max-skew of 1 as the node selector requirement here excludes the - // existing on-demand pod from counting within this topology. - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - // limit our provisioner to only creating spot nodes - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, - {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{"spot"}}, - }, - TopologySpreadConstraints: topology, - }, 5)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 5)) - }) - It("should balance pods across capacity-types (no constraints)", func() { - rr := v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "single-pod-instance-type"}, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{"on-demand"}, - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - // limit our provisioner to only creating spot nodes - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{"spot"}}, - } - - // since there is no node selector on this pod, the topology can see the single on-demand node that already - // exists and that limits us to scheduling 2 more spot pods before we would violate max-skew - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, - TopologySpreadConstraints: topology, - }, 5)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2)) - }) - It("should balance pods across arch (no constraints)", func() { - rr := v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - } - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "single-pod-instance-type"}, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"amd64"}, - }, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - - ExpectScheduled(ctx, env.Client, pod) - - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelArchStable, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - // limit our provisioner to only creating arm64 nodes - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{"arm64"}}} - - // since there is no node selector on this pod, the topology can see the single arm64 node that already - // exists and that limits us to scheduling 2 more spot pods before we would violate max-skew - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - ResourceRequirements: rr, - TopologySpreadConstraints: topology, - }, 5)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2)) - }) - }) - - Context("Combined Hostname and Zonal Topology", func() { - It("should spread pods while respecting both constraints (hostname and zonal)", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }, { - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 3, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 2)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 3)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2, 1)) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 5)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 3, 3)) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 11)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(7, 7, 7)) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) - }) - It("should balance pods across provisioner requirements", func() { - spotProv := test.Provisioner(test.ProvisionerOptions{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{"spot"}, - }, - { - Key: "capacity.spread.4-1", - Operator: v1.NodeSelectorOpIn, - Values: []string{"2", "3", "4", "5"}, - }, - }, - }) - onDemandProv := test.Provisioner(test.ProvisionerOptions{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{"on-demand"}, - }, - { - Key: "capacity.spread.4-1", - Operator: v1.NodeSelectorOpIn, - Values: []string{"1"}, - }, - }, - }) - - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: "capacity.spread.4-1", - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, spotProv, onDemandProv) - pods := test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - }, 20) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, p := range pods { - ExpectScheduled(ctx, env.Client, p) - } - - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 4, 4, 4, 4)) - // due to the spread across provisioners, we've forced a 4:1 spot to on-demand spread - ExpectSkew(ctx, env.Client, "default", &v1.TopologySpreadConstraint{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }).To(ConsistOf(4, 16)) - }) - - It("should spread pods while respecting both constraints", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }, { - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.ScheduleAnyway, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - - // create a second provisioner that can't provision at all - provisionerB := test.Provisioner() - provisionerB.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}} - provisionerB.Spec.Limits = &v1alpha5.Limits{ - Resources: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("0"), - }, - } - - ExpectApplied(ctx, env.Client, provisioner, provisionerB) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 10)..., - ) - - // should get one pod per zone, can't schedule to test-zone-3 since that provisioner is effectively disabled - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) - // and one pod per node - ExpectSkew(ctx, env.Client, "default", &topology[1]).To(ConsistOf(1, 1)) - }) - - It("should spread pods while respecting both constraints", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }, { - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 3, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 2)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 3)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 2)) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 5)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 5)) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 11)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(11, 10)) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) - }) - }) - - Context("Combined Zonal and Capacity Type Topology", func() { - It("should spread pods while respecting both constraints", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }, { - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 2)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 1))) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 1))) - - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 3)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 3))) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 2))) - - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 3)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 5))) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 4))) - - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 11)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 11))) - ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 7))) - }) - }) - - Context("Combined Hostname, Zonal, and Capacity Type Topology", func() { - It("should spread pods while respecting all constraints", func() { - // ensure we've got an instance type for every zone/capacity-type pair - cloudProvider.InstanceTypes = fake.InstanceTypesAssorted() - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }, { - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 2, - }, { - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 3, - }} - - // add varying numbers of pods, checking after each scheduling to ensure that our max required max skew - // has not been violated for each constraint - for i := 1; i < 15; i++ { - pods := test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, i) - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - ExpectMaxSkew(ctx, env.Client, "default", &topology[0]).To(BeNumerically("<=", 1)) - ExpectMaxSkew(ctx, env.Client, "default", &topology[1]).To(BeNumerically("<=", 2)) - ExpectMaxSkew(ctx, env.Client, "default", &topology[2]).To(BeNumerically("<=", 3)) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } - } - }) - }) - - // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#interaction-with-node-affinity-and-node-selectors - Context("Combined Zonal Topology and Node Affinity", func() { - It("should limit spread options by nodeSelector", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - append( - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}, - }, 5), - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}, - }, 10)..., - )..., - ) - // we limit the zones of each pod via node selectors, which causes the topology spreads to only consider - // the single zone as the only valid domain for the topology spread allowing us to schedule multiple pods per domain - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 10)) - }) - It("should limit spread options by node requirements", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1", "test-zone-2"}, - }, - }, - }, 10)...) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 5)) - }) - It("should limit spread options by required node affinity", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{ - "test-zone-1", "test-zone-2", - }}}, - }, 6)...) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3)) - - // open the provisioner back to up so it can see all zones again - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}} - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{ - "test-zone-2", "test-zone-3", - }}}, - })) - - // it will schedule on the currently empty zone-3 even though max-skew is violated as it improves max-skew - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 1)) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - }, 5)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 4, 4)) - }) - It("should not limit spread options by preferred node affinity", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - NodePreferences: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{ - "test-zone-1", "test-zone-2", - }}}, - }, 6)...) - - // scheduling shouldn't be affected since it's a preferred affinity - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2, 2)) - }) - }) - - // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#interaction-with-node-affinity-and-node-selectors - Context("Combined Capacity Type Topology and Node Affinity", func() { - It("should limit spread options by nodeSelector", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.ScheduleAnyway, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - append( - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - NodeSelector: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeSpot}, - }, 5), - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - NodeSelector: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeOnDemand}, - }, 5)..., - )..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 5)) - }) - It("should limit spread options by node affinity (capacity type)", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1alpha5.LabelCapacityType, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - // need to limit the rules to spot or else it will know that on-demand has 0 pods and won't violate the max-skew - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}, - }, - }, 3)...) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3)) - - // open the rules back to up so it can see all capacity types - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand, v1alpha5.CapacityTypeSpot}}, - }, - })) - - // it will schedule on the currently empty on-demand even though max-skew is violated as it improves max-skew - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 1)) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - }, 5)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 4)) - }) - }) - - Context("Pod Affinity/Anti-Affinity", func() { - It("should schedule a pod with empty pod affinity and anti-affinity", func() { - ExpectApplied(ctx, env.Client) - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{ - PodRequirements: []v1.PodAffinityTerm{}, - PodAntiRequirements: []v1.PodAffinityTerm{}, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should respect pod affinity (hostname)", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - affLabels := map[string]string{"security": "s2"} - - affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) - // affPod2 will try to get scheduled with affPod1 - affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelHostname, - }}}) - - var pods []*v1.Pod - pods = append(pods, test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - }, 10)...) - pods = append(pods, affPod1) - pods = append(pods, affPod2) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - n1 := ExpectScheduled(ctx, env.Client, affPod1) - n2 := ExpectScheduled(ctx, env.Client, affPod2) - // should be scheduled on the same node - Expect(n1.Name).To(Equal(n2.Name)) - }) - It("should respect pod affinity (arch)", func() { - affLabels := map[string]string{"security": "s2"} - tsc := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: affLabels}, - MaxSkew: 1, - }} - - affPod1 := test.UnschedulablePod(test.PodOptions{ - TopologySpreadConstraints: tsc, - ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - }, - NodeSelector: map[string]string{ - v1.LabelArchStable: "arm64", - }}) - // affPod2 will try to get scheduled with affPod1 - affPod2 := test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, - TopologySpreadConstraints: tsc, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelArchStable, - }}}) - - pods := []*v1.Pod{affPod1, affPod2} - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - n1 := ExpectScheduled(ctx, env.Client, affPod1) - n2 := ExpectScheduled(ctx, env.Client, affPod2) - // should be scheduled on a node with the same arch - Expect(n1.Labels[v1.LabelArchStable]).To(Equal(n2.Labels[v1.LabelArchStable])) - // but due to TSC, not on the same node - Expect(n1.Name).ToNot(Equal(n2.Name)) - }) - It("should respect self pod affinity (hostname)", func() { - affLabels := map[string]string{"security": "s2"} - - pods := test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: affLabels, - }, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelHostname, - }}, - }, 3) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := map[string]struct{}{} - for _, p := range pods { - n := ExpectScheduled(ctx, env.Client, p) - nodeNames[n.Name] = struct{}{} - } - Expect(len(nodeNames)).To(Equal(1)) - }) - It("should respect self pod affinity for first empty topology domain only (hostname)", func() { - affLabels := map[string]string{"security": "s2"} - createPods := func() []*v1.Pod { - return test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: affLabels, - }, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelHostname, - }}, - }, 10) - } - ExpectApplied(ctx, env.Client, provisioner) - pods := createPods() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := map[string]struct{}{} - unscheduledCount := 0 - scheduledCount := 0 - for _, p := range pods { - p = ExpectPodExists(ctx, env.Client, p.Name, p.Namespace) - if p.Spec.NodeName == "" { - unscheduledCount++ - } else { - nodeNames[p.Spec.NodeName] = struct{}{} - scheduledCount++ - } - } - // the node can only hold 5 pods, so we should get a single node with 5 pods and 5 unschedulable pods from that batch - Expect(len(nodeNames)).To(Equal(1)) - Expect(scheduledCount).To(BeNumerically("==", 5)) - Expect(unscheduledCount).To(BeNumerically("==", 5)) - - // and pods in a different batch should not schedule as well even if the node is not ready yet - pods = createPods() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, p := range pods { - ExpectNotScheduled(ctx, env.Client, p) - } - }) - It("should respect self pod affinity for first empty topology domain only (hostname/constrained zones)", func() { - affLabels := map[string]string{"security": "s2"} - // put one pod in test-zone-1, this does affect pod affinity even though we have different node selectors. - // The node selector and required node affinity restrictions to topology counting only apply to topology spread. - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: affLabels, - }, - NodeSelector: map[string]string{ - v1.LabelTopologyZone: "test-zone-1", - }, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelHostname, - }}, - })) - - pods := test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: affLabels, - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2", "test-zone-3"}, - }, - }, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelHostname, - }}, - }, 10) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, p := range pods { - // none of this should schedule - ExpectNotScheduled(ctx, env.Client, p) - } - }) - It("should respect self pod affinity (zone)", func() { - affLabels := map[string]string{"security": "s2"} - - pods := test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: affLabels, - }, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }}, - }, 3) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := map[string]struct{}{} - for _, p := range pods { - n := ExpectScheduled(ctx, env.Client, p) - nodeNames[n.Name] = struct{}{} - } - Expect(len(nodeNames)).To(Equal(1)) - }) - It("should respect self pod affinity (zone w/ constraint)", func() { - affLabels := map[string]string{"security": "s2"} - // the pod needs to provide it's own zonal affinity, but we further limit it to only being on test-zone-3 - pods := test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: affLabels, - }, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }}, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-3"}, - }, - }, - }, 3) - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - nodeNames := map[string]struct{}{} - for _, p := range pods { - n := ExpectScheduled(ctx, env.Client, p) - nodeNames[n.Name] = struct{}{} - Expect(n.Labels[v1.LabelTopologyZone]).To(Equal("test-zone-3")) - } - Expect(len(nodeNames)).To(Equal(1)) - }) - It("should allow violation of preferred pod affinity", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - affPod2 := test.UnschedulablePod(test.PodOptions{PodPreferences: []v1.WeightedPodAffinityTerm{{ - Weight: 50, - PodAffinityTerm: v1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"security": "s2"}, - }, - TopologyKey: v1.LabelHostname, - }, - }}}) - - var pods []*v1.Pod - pods = append(pods, test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - }, 10)...) - - pods = append(pods, affPod2) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - // should be scheduled as the pod it has affinity to doesn't exist, but it's only a preference and not a - // hard constraints - ExpectScheduled(ctx, env.Client, affPod2) - - }) - It("should allow violation of preferred pod anti-affinity", func() { - affPods := test.UnschedulablePods(test.PodOptions{PodAntiPreferences: []v1.WeightedPodAffinityTerm{ - { - Weight: 50, - PodAffinityTerm: v1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - TopologyKey: v1.LabelTopologyZone, - }, - }, - }}, 10) - - var pods []*v1.Pod - pods = append(pods, test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }}, - }, 3)...) - - pods = append(pods, affPods...) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, aff := range affPods { - ExpectScheduled(ctx, env.Client, aff) - } - - }) - It("should separate nodes using simple pod anti-affinity on hostname", func() { - affLabels := map[string]string{"security": "s2"} - // pod affinity/anti-affinity are bidirectional, so run this a few times to ensure we handle it regardless - // of pod scheduling order - ExpectApplied(ctx, env.Client, provisioner) - for i := 0; i < 10; i++ { - affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) - // affPod2 will avoid affPod1 - affPod2 := test.UnschedulablePod(test.PodOptions{PodAntiRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelHostname, - }}}) - - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, affPod2, affPod1) - n1 := ExpectScheduled(ctx, env.Client, affPod1) - n2 := ExpectScheduled(ctx, env.Client, affPod2) - // should not be scheduled on the same node - Expect(n1.Name).ToNot(Equal(n2.Name)) - } - }) - It("should not violate pod anti-affinity on zone", func() { - affLabels := map[string]string{"security": "s2"} - zone1Pod := test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - }, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) - zone2Pod := test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - }, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) - zone3Pod := test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - }, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) - - affPod := test.UnschedulablePod(test.PodOptions{ - PodAntiRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }}}) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, zone1Pod, zone2Pod, zone3Pod, affPod) - // the three larger zone specific pods should get scheduled first due to first fit descending onto one - // node per zone. - ExpectScheduled(ctx, env.Client, zone1Pod) - ExpectScheduled(ctx, env.Client, zone2Pod) - ExpectScheduled(ctx, env.Client, zone3Pod) - // the pod with anti-affinity - ExpectNotScheduled(ctx, env.Client, affPod) - }) - It("should not violate pod anti-affinity on zone (other schedules first)", func() { - affLabels := map[string]string{"security": "s2"} - pod := test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - }}) - affPod := test.UnschedulablePod(test.PodOptions{ - PodAntiRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }}}) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod, affPod) - // the pod we need to avoid schedules first, but we don't know where. - ExpectScheduled(ctx, env.Client, pod) - // the pod with anti-affinity - ExpectNotScheduled(ctx, env.Client, affPod) - }) - It("should not violate pod anti-affinity (arch)", func() { - affLabels := map[string]string{"security": "s2"} - tsc := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: affLabels}, - MaxSkew: 1, - }} - - affPod1 := test.UnschedulablePod(test.PodOptions{ - TopologySpreadConstraints: tsc, - ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - }, - NodeSelector: map[string]string{ - v1.LabelArchStable: "arm64", - }}) - - // affPod2 will try to get scheduled on a node with a different archi from affPod1. Due to resource - // requests we try to schedule it last - affPod2 := test.UnschedulablePod(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, - TopologySpreadConstraints: tsc, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - }, - PodAntiRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelArchStable, - }}}) - - pods := []*v1.Pod{affPod1, affPod2} - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - n1 := ExpectScheduled(ctx, env.Client, affPod1) - n2 := ExpectScheduled(ctx, env.Client, affPod2) - // should not be scheduled on nodes with the same arch - Expect(n1.Labels[v1.LabelArchStable]).ToNot(Equal(n2.Labels[v1.LabelArchStable])) - }) - It("should violate preferred pod anti-affinity on zone (inverse)", func() { - affLabels := map[string]string{"security": "s2"} - anti := []v1.WeightedPodAffinityTerm{ - { - Weight: 10, - PodAffinityTerm: v1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }, - }, - } - rr := v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - } - zone1Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiPreferences: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) - zone2Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiPreferences: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) - zone3Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiPreferences: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) - - affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, zone1Pod, zone2Pod, zone3Pod, affPod) - // three pods with anti-affinity will schedule first due to first fit-descending - ExpectScheduled(ctx, env.Client, zone1Pod) - ExpectScheduled(ctx, env.Client, zone2Pod) - ExpectScheduled(ctx, env.Client, zone3Pod) - // the anti-affinity was a preference, so this can schedule - ExpectScheduled(ctx, env.Client, affPod) - }) - It("should not violate pod anti-affinity on zone (inverse)", func() { - affLabels := map[string]string{"security": "s2"} - anti := []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }} - rr := v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - } - zone1Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiRequirements: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) - zone2Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiRequirements: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) - zone3Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiRequirements: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) - - affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, zone1Pod, zone2Pod, zone3Pod, affPod) - // three pods with anti-affinity will schedule first due to first fit-descending - ExpectScheduled(ctx, env.Client, zone1Pod) - ExpectScheduled(ctx, env.Client, zone2Pod) - ExpectScheduled(ctx, env.Client, zone3Pod) - // this pod with no anti-affinity rules can't schedule. It has no anti-affinity rules, but every zone has a - // pod with anti-affinity rules that prevent it from scheduling - ExpectNotScheduled(ctx, env.Client, affPod) - }) - It("should not violate pod anti-affinity on zone (Schrödinger)", func() { - affLabels := map[string]string{"security": "s2"} - anti := []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }} - zoneAnywherePod := test.UnschedulablePod(test.PodOptions{ - PodAntiRequirements: anti, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - }, - }) - - affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, zoneAnywherePod, affPod) - // the pod with anti-affinity will schedule first due to first fit-descending, but we don't know which zone it landed in - node1 := ExpectScheduled(ctx, env.Client, zoneAnywherePod) - - // this pod cannot schedule since the pod with anti-affinity could potentially be in any zone - affPod = ExpectNotScheduled(ctx, env.Client, affPod) - - // a second batching will now allow the pod to schedule as the zoneAnywherePod has been committed to a zone - // by the actual node creation - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, affPod) - node2 := ExpectScheduled(ctx, env.Client, affPod) - Expect(node1.Labels[v1.LabelTopologyZone]).ToNot(Equal(node2.Labels[v1.LabelTopologyZone])) - - }) - It("should not violate pod anti-affinity on zone (inverse w/existing nodes)", func() { - affLabels := map[string]string{"security": "s2"} - anti := []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }} - rr := v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - } - zone1Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiRequirements: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) - zone2Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiRequirements: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) - zone3Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiRequirements: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) - - affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) - - // provision these so we get three nodes that exist in the cluster with anti-affinity to a pod that we will - // then try to schedule - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, zone1Pod, zone2Pod, zone3Pod) - node1 := ExpectScheduled(ctx, env.Client, zone1Pod) - node2 := ExpectScheduled(ctx, env.Client, zone2Pod) - node3 := ExpectScheduled(ctx, env.Client, zone3Pod) - - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node3)) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone1Pod)) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone2Pod)) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone3Pod)) - - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone1Pod)) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone2Pod)) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone3Pod)) - - // this pod with no anti-affinity rules can't schedule. It has no anti-affinity rules, but every zone has an - // existing pod (not from this batch) with anti-affinity rules that prevent it from scheduling - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, affPod) - ExpectNotScheduled(ctx, env.Client, affPod) - }) - It("should violate preferred pod anti-affinity on zone (inverse w/existing nodes)", func() { - affLabels := map[string]string{"security": "s2"} - anti := []v1.WeightedPodAffinityTerm{ - { - Weight: 10, - PodAffinityTerm: v1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }, - }, - } - rr := v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, - } - zone1Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiPreferences: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) - zone2Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiPreferences: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) - zone3Pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: rr, - PodAntiPreferences: anti, - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) - - affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) - - // provision these so we get three nodes that exist in the cluster with anti-affinity to a pod that we will - // then try to schedule - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, zone1Pod, zone2Pod, zone3Pod) - node1 := ExpectScheduled(ctx, env.Client, zone1Pod) - node2 := ExpectScheduled(ctx, env.Client, zone2Pod) - node3 := ExpectScheduled(ctx, env.Client, zone3Pod) - - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) - ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node3)) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone1Pod)) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone2Pod)) - ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone3Pod)) - - // this pod with no anti-affinity rules can schedule, though it couldn't if the anti-affinity were required - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, affPod) - ExpectScheduled(ctx, env.Client, affPod) - }) - It("should allow violation of a pod affinity preference with a conflicting required constraint", func() { - affLabels := map[string]string{"security": "s2"} - constraint := v1.TopologySpreadConstraint{ - MaxSkew: 1, - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - } - affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) - affPods := test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - // limit these pods to one per host - TopologySpreadConstraints: []v1.TopologySpreadConstraint{constraint}, - // with a preference to the other pod - PodPreferences: []v1.WeightedPodAffinityTerm{{ - Weight: 50, - PodAffinityTerm: v1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelHostname, - }, - }}}, 3) - ExpectApplied(ctx, env.Client, provisioner) - pods := append(affPods, affPod1) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - // all pods should be scheduled since the affinity term is just a preference - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } - // and we'll get three nodes due to the topology spread - ExpectSkew(ctx, env.Client, "", &constraint).To(ConsistOf(1, 1, 1)) - }) - It("should support pod anti-affinity with a zone topology", func() { - affLabels := map[string]string{"security": "s2"} - - // affPods will avoid being scheduled in the same zone - createPods := func() []*v1.Pod { - return test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, - PodAntiRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }}}, 3) - } - - top := &v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone} - - // One of the downsides of late committal is that absent other constraints, it takes multiple batches of - // scheduling for zonal anti-affinities to work themselves out. The first schedule, we know that the pod - // will land in test-zone-1, test-zone-2, or test-zone-3, but don't know which it collapses to until the - // node is actually created. - - // one pod pod will schedule - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, createPods()...) - ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1)) - // delete all of the unscheduled ones as provisioning will only bind pods passed into the provisioning call - // the scheduler looks at all pods though, so it may assume a pod from this batch schedules and no others do - ExpectDeleteAllUnscheduledPods(ctx, env.Client) - - // second pod in a second zone - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, createPods()...) - ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1, 1)) - ExpectDeleteAllUnscheduledPods(ctx, env.Client) - - // third pod in the last zone - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, createPods()...) - ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1, 1, 1)) - ExpectDeleteAllUnscheduledPods(ctx, env.Client) - - // and nothing else can schedule - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, createPods()...) - ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1, 1, 1)) - ExpectDeleteAllUnscheduledPods(ctx, env.Client) - }) - It("should not schedule pods with affinity to a non-existent pod", func() { - affLabels := map[string]string{"security": "s2"} - affPods := test.UnschedulablePods(test.PodOptions{ - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }}}, 10) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, affPods...) - // the pod we have affinity to is not in the cluster, so all of these pods are unschedulable - for _, p := range affPods { - ExpectNotScheduled(ctx, env.Client, p) - } - }) - It("should support pod affinity with zone topology (unconstrained target)", func() { - affLabels := map[string]string{"security": "s2"} - - // the pod that the others have an affinity to - targetPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) - - // affPods all want to schedule in the same zone as targetPod, but can't as it's zone is undetermined - affPods := test.UnschedulablePods(test.PodOptions{ - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }}}, 10) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, append(affPods, targetPod)...) - top := &v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone} - // these pods can't schedule as the pod they have affinity to isn't limited to any particular zone - for i := range affPods { - ExpectNotScheduled(ctx, env.Client, affPods[i]) - affPods[i] = ExpectPodExists(ctx, env.Client, affPods[i].Name, affPods[i].Namespace) - } - ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1)) - - // now that targetPod has been scheduled to a node, it's zone is committed and the pods with affinity to it - // should schedule in the same zone - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, affPods...) - for _, pod := range affPods { - ExpectScheduled(ctx, env.Client, pod) - } - ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(11)) - }) - It("should support pod affinity with zone topology (constrained target)", func() { - affLabels := map[string]string{"security": "s2"} - - // the pod that the others have an affinity to - affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1"}, - }, - }}) - - // affPods will all be scheduled in the same zone as affPod1 - affPods := test.UnschedulablePods(test.PodOptions{ - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelTopologyZone, - }}}, 10) - - affPods = append(affPods, affPod1) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, affPods...) - top := &v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone} - ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(11)) - }) - It("should handle multiple dependent affinities", func() { - dbLabels := map[string]string{"type": "db", "spread": "spread"} - webLabels := map[string]string{"type": "web", "spread": "spread"} - cacheLabels := map[string]string{"type": "cache", "spread": "spread"} - uiLabels := map[string]string{"type": "ui", "spread": "spread"} - for i := 0; i < 50; i++ { - ExpectApplied(ctx, env.Client, provisioner.DeepCopy()) - // we have to schedule DB -> Web -> Cache -> UI in that order or else there are pod affinity violations - pods := []*v1.Pod{ - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: dbLabels}}), - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: webLabels}, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{MatchLabels: dbLabels}, - TopologyKey: v1.LabelHostname}, - }}), - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: cacheLabels}, - PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{MatchLabels: webLabels}, - TopologyKey: v1.LabelHostname}, - }}), - test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: uiLabels}, - PodRequirements: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{MatchLabels: cacheLabels}, - TopologyKey: v1.LabelHostname}, - }}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for i := range pods { - ExpectScheduled(ctx, env.Client, pods[i]) - } - ExpectCleanedUp(ctx, env.Client) - cluster.Reset() - } - }) - It("should fail to schedule pods with unsatisfiable dependencies", func() { - dbLabels := map[string]string{"type": "db", "spread": "spread"} - webLabels := map[string]string{"type": "web", "spread": "spread"} - ExpectApplied(ctx, env.Client, provisioner) - // this pods wants to schedule with a non-existent pod, this test just ensures that the scheduling loop - // doesn't infinite loop - pod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: dbLabels}, - PodRequirements: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{MatchLabels: webLabels}, - TopologyKey: v1.LabelHostname, - }, - }}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) - It("should filter pod affinity topologies by namespace, no matching pods", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - ExpectApplied(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other-ns-no-match"}}) - affLabels := map[string]string{"security": "s2"} - - affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels, Namespace: "other-ns-no-match"}}) - // affPod2 will try to get scheduled with affPod1 - affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - TopologyKey: v1.LabelHostname, - }}}) - - var pods []*v1.Pod - // creates 10 nodes due to topo spread - pods = append(pods, test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - }, 10)...) - pods = append(pods, affPod1) - pods = append(pods, affPod2) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - - // the target pod gets scheduled - ExpectScheduled(ctx, env.Client, affPod1) - // but the one with affinity does not since the target pod is not in the same namespace and doesn't - // match the namespace list or namespace selector - ExpectNotScheduled(ctx, env.Client, affPod2) - }) - It("should filter pod affinity topologies by namespace, matching pods namespace list", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - ExpectApplied(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other-ns-list"}}) - affLabels := map[string]string{"security": "s2"} - - affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels, Namespace: "other-ns-list"}}) - // affPod2 will try to get scheduled with affPod1 - affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - Namespaces: []string{"other-ns-list"}, - TopologyKey: v1.LabelHostname, - }}}) - - var pods []*v1.Pod - // create 10 nodes - pods = append(pods, test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - }, 10)...) - // put our target pod on one of them - pods = append(pods, affPod1) - // and our pod with affinity should schedule on the same node - pods = append(pods, affPod2) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - n1 := ExpectScheduled(ctx, env.Client, affPod1) - n2 := ExpectScheduled(ctx, env.Client, affPod2) - // should be scheduled on the same node - Expect(n1.Name).To(Equal(n2.Name)) - }) - It("should filter pod affinity topologies by namespace, empty namespace selector", func() { - if env.Version.Minor() < 21 { - Skip("namespace selector is only supported on K8s >= 1.21.x") - } - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - ExpectApplied(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "empty-ns-selector", Labels: map[string]string{"foo": "bar"}}}) - affLabels := map[string]string{"security": "s2"} - - affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels, Namespace: "empty-ns-selector"}}) - // affPod2 will try to get scheduled with affPod1 - affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: affLabels, - }, - // select all pods in all namespaces since the selector is empty - NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{}}, - TopologyKey: v1.LabelHostname, - }}}) - - var pods []*v1.Pod - // create 10 nodes - pods = append(pods, test.UnschedulablePods(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: topology, - }, 10)...) - // put our target pod on one of them - pods = append(pods, affPod1) - // and our pod with affinity should schedule on the same node - pods = append(pods, affPod2) - - ExpectApplied(ctx, env.Client, provisioner) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - n1 := ExpectScheduled(ctx, env.Client, affPod1) - n2 := ExpectScheduled(ctx, env.Client, affPod2) - // should be scheduled on the same node due to the empty namespace selector - Expect(n1.Name).To(Equal(n2.Name)) - }) - It("should count topology across multiple provisioners", func() { - ExpectApplied(ctx, env.Client, - test.Provisioner(test.ProvisionerOptions{ - Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}}, - }), - test.Provisioner(test.ProvisionerOptions{ - Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "test-zone-3"}}}, - }), - ) - labels := map[string]string{"foo": "bar"} - topology := v1.TopologySpreadConstraint{ - TopologyKey: v1.LabelTopologyZone, - MaxSkew: 1, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - WhenUnsatisfiable: v1.DoNotSchedule, - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, test.Pods(10, test.UnscheduleablePodOptions(test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, - TopologySpreadConstraints: []v1.TopologySpreadConstraint{topology}, - }))...) - ExpectSkew(ctx, env.Client, "default", &topology).To(ConsistOf(3, 3, 4)) - }) - }) -}) - -var _ = Describe("Taints", func() { - var provisioner *v1alpha5.Provisioner - BeforeEach(func() { - provisioner = test.Provisioner(test.ProvisionerOptions{Requirements: []v1.NodeSelectorRequirement{{ - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeSpot, v1alpha5.CapacityTypeOnDemand}, - }}}) - }) - It("should taint nodes with provisioner taints", func() { - provisioner.Spec.Taints = []v1.Taint{{Key: "test", Value: "bar", Effect: v1.TaintEffectNoSchedule}} - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod( - test.PodOptions{Tolerations: []v1.Toleration{{Effect: v1.TaintEffectNoSchedule, Operator: v1.TolerationOpExists}}}, - ) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Spec.Taints).To(ContainElement(provisioner.Spec.Taints[0])) - }) - It("should schedule pods that tolerate provisioner constraints", func() { - provisioner.Spec.Taints = []v1.Taint{{Key: "test-key", Value: "test-value", Effect: v1.TaintEffectNoSchedule}} - ExpectApplied(ctx, env.Client, provisioner) - pods := []*v1.Pod{ - // Tolerates with OpExists - test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpExists, Effect: v1.TaintEffectNoSchedule}}}), - // Tolerates with OpEqual - test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Value: "test-value", Operator: v1.TolerationOpEqual, Effect: v1.TaintEffectNoSchedule}}}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } - ExpectApplied(ctx, env.Client, provisioner) - otherPods := []*v1.Pod{ - // Missing toleration - test.UnschedulablePod(), - // key mismatch with OpExists - test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "invalid", Operator: v1.TolerationOpExists}}}), - // value mismatch - test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpEqual, Effect: v1.TaintEffectNoSchedule}}}), - } - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, otherPods...) - for _, pod := range otherPods { - ExpectNotScheduled(ctx, env.Client, pod) - } - }) - It("should provision nodes with taints and schedule pods if the taint is only a startup taint", func() { - provisioner.Spec.StartupTaints = []v1.Taint{{Key: "ignore-me", Value: "nothing-to-see-here", Effect: v1.TaintEffectNoSchedule}} - - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - }) - It("should not generate taints for OpExists", func() { - ExpectApplied(ctx, env.Client, provisioner) - pod := test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpExists, Effect: v1.TaintEffectNoExecute}}}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Spec.Taints).To(HaveLen(1)) // Expect no taints generated beyond the default - }) -}) diff --git a/pkg/controllers/provisioning/scheduling/suite_test.go b/pkg/controllers/provisioning/scheduling/suite_test.go index 8e070fa4b8..ed310f9b50 100644 --- a/pkg/controllers/provisioning/scheduling/suite_test.go +++ b/pkg/controllers/provisioning/scheduling/suite_test.go @@ -19,12 +19,20 @@ import ( "context" "fmt" "math" + "math/rand" "testing" "time" "github.com/samber/lo" v1 "k8s.io/api/core/v1" + nodev1 "k8s.io/api/node/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" + cloudproviderapi "k8s.io/cloud-provider/api" + "k8s.io/csi-translation-lib/plugins" clock "k8s.io/utils/clock/testing" "sigs.k8s.io/controller-runtime/pkg/client" @@ -47,6 +55,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "knative.dev/pkg/logging/testing" + "knative.dev/pkg/ptr" . "github.com/aws/karpenter-core/pkg/test/expectations" ) @@ -105,6 +114,3140 @@ var _ = AfterEach(func() { cluster.Reset() }) +var _ = Context("NodePool", func() { + var nodePool *v1beta1.NodePool + BeforeEach(func() { + nodePool = test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1beta1.CapacityTypeSpot, v1beta1.CapacityTypeOnDemand}, + }, + }, + }, + }, + }, + }) + }) + + Describe("Custom Constraints", func() { + Context("NodePool with Labels", func() { + It("should schedule unconstrained pods that don't have matching node selectors", func() { + nodePool.Spec.Template.Labels = map[string]string{"test-key": "test-value"} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should not schedule pods that have conflicting node selectors", func() { + nodePool.Spec.Template.Labels = map[string]string{"test-key": "test-value"} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{"test-key": "different-value"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule pods that have node selectors with undefined key", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{"test-key": "test-value"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule pods that have matching requirements", func() { + nodePool.Spec.Template.Labels = map[string]string{"test-key": "test-value"} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should not schedule pods that have conflicting requirements", func() { + nodePool.Spec.Template.Labels = map[string]string{"test-key": "test-value"} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + }) + Context("Well Known Labels", func() { + It("should use NodePool constraints", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should use node selectors", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should not schedule nodes with a hostname selector", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelHostname: "red-node"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule the pod if nodeselector unknown", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule if node selector outside of NodePool constraints", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should schedule compatible requirements with Operator=Gt", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpGt, Values: []string{"8"}, + }} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "16")) + }) + It("should schedule compatible requirements with Operator=Lt", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpLt, Values: []string{"8"}, + }} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "2")) + }) + It("should not schedule incompatible preferences and requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "unknown"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should not schedule incompatible preferences and requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible preferences and requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "unknown"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should schedule incompatible preferences and requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible preferences and requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-3"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should schedule incompatible preferences and requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible node selectors, preferences and requirements", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}, + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should combine multidimensional node selectors, preferences and requirements", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeSelector: map[string]string{ + v1.LabelTopologyZone: "test-zone-3", + v1.LabelInstanceTypeStable: "arm-instance-type", + }, + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}}, + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type", "arm-instance-type"}}, + }, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, + }, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "arm-instance-type")) + }) + }) + Context("Constraints Validation", func() { + It("should not schedule pods that have node selectors with restricted labels", func() { + ExpectApplied(ctx, env.Client, nodePool) + for label := range v1beta1.RestrictedLabels { + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + } + }) + It("should not schedule pods that have node selectors with restricted domains", func() { + ExpectApplied(ctx, env.Client, nodePool) + for domain := range v1beta1.RestrictedLabelDomains { + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + } + }) + It("should schedule pods that have node selectors with label in restricted domains exceptions list", func() { + var requirements []v1.NodeSelectorRequirement + for domain := range v1beta1.LabelDomainExceptions { + requirements = append(requirements, v1.NodeSelectorRequirement{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}) + } + nodePool.Spec.Template.Spec.Requirements = requirements + ExpectApplied(ctx, env.Client, nodePool) + for domain := range v1beta1.LabelDomainExceptions { + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) + } + }) + It("should schedule pods that have node selectors with label in wellknown label list", func() { + schedulable := []*v1.Pod{ + // Constrained by zone + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), + // Constrained by instanceType + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), + // Constrained by architecture + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), + // Constrained by operatingSystem + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), + // Constrained by capacity type + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.CapacityTypeLabelKey: "spot"}}), + } + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) + for _, pod := range schedulable { + ExpectScheduled(ctx, env.Client, pod) + } + }) + }) + Context("Scheduling Logic", func() { + It("should not schedule pods that have node selectors with In operator and undefined key", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule pods that have node selectors with NotIn operator and undefined key", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).ToNot(HaveKeyWithValue("test-key", "test-value")) + }) + It("should not schedule pods that have node selectors with Exists operator and undefined key", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpExists}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule pods that with DoesNotExists operator and undefined key", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).ToNot(HaveKey("test-key")) + }) + It("should schedule unconstrained pods that don't have matching node selectors", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should schedule pods that have node selectors with matching value and In operator", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should not schedule pods that have node selectors with matching value and NotIn operator", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule the pod with Exists operator and defined key", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpExists}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should not schedule the pod with DoesNotExists operator and defined key", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule pods that have node selectors with different value and In operator", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule pods that have node selectors with different value and NotIn operator", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should schedule compatible pods to the same node", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}), + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, + }}), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + node1 := ExpectScheduled(ctx, env.Client, pods[0]) + node2 := ExpectScheduled(ctx, env.Client, pods[1]) + Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) + Expect(node2.Labels).To(HaveKeyWithValue("test-key", "test-value")) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should schedule incompatible pods to the different node", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}), + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, + }}), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + node1 := ExpectScheduled(ctx, env.Client, pods[0]) + node2 := ExpectScheduled(ctx, env.Client, pods[1]) + Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) + Expect(node2.Labels).To(HaveKeyWithValue("test-key", "another-value")) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("Exists operator should not overwrite the existing value", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"non-existent-zone"}}, + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + }) + Context("Well Known Labels", func() { + It("should use NodePool constraints", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should use node selectors", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should not schedule nodes with a hostname selector", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelHostname: "red-node"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule the pod if nodeselector unknown", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule if node selector outside of NodePool constraints", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should schedule compatible requirements with Operator=Gt", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpGt, Values: []string{"8"}, + }} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "16")) + }) + It("should schedule compatible requirements with Operator=Lt", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpLt, Values: []string{"8"}, + }} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "2")) + }) + It("should not schedule incompatible preferences and requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "unknown"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should not schedule incompatible preferences and requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible preferences and requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "unknown"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should schedule incompatible preferences and requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible preferences and requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-3"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should schedule incompatible preferences and requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible node selectors, preferences and requirements", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}, + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should combine multidimensional node selectors, preferences and requirements", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeSelector: map[string]string{ + v1.LabelTopologyZone: "test-zone-3", + v1.LabelInstanceTypeStable: "arm-instance-type", + }, + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}}, + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type", "arm-instance-type"}}, + }, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, + }, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "arm-instance-type")) + }) + }) + Context("Constraints Validation", func() { + It("should not schedule pods that have node selectors with restricted labels", func() { + ExpectApplied(ctx, env.Client, nodePool) + for label := range v1beta1.RestrictedLabels { + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + } + }) + It("should not schedule pods that have node selectors with restricted domains", func() { + ExpectApplied(ctx, env.Client, nodePool) + for domain := range v1beta1.RestrictedLabelDomains { + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + } + }) + It("should schedule pods that have node selectors with label in restricted domains exceptions list", func() { + var requirements []v1.NodeSelectorRequirement + for domain := range v1beta1.LabelDomainExceptions { + requirements = append(requirements, v1.NodeSelectorRequirement{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}) + } + nodePool.Spec.Template.Spec.Requirements = requirements + ExpectApplied(ctx, env.Client, nodePool) + for domain := range v1beta1.LabelDomainExceptions { + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) + } + }) + It("should schedule pods that have node selectors with label in wellknown label list", func() { + schedulable := []*v1.Pod{ + // Constrained by zone + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), + // Constrained by instanceType + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), + // Constrained by architecture + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), + // Constrained by operatingSystem + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), + // Constrained by capacity type + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.CapacityTypeLabelKey: "spot"}}), + } + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) + for _, pod := range schedulable { + ExpectScheduled(ctx, env.Client, pod) + } + }) + }) + Context("Scheduling Logic", func() { + It("should not schedule pods that have node selectors with In operator and undefined key", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule pods that have node selectors with NotIn operator and undefined key", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).ToNot(HaveKeyWithValue("test-key", "test-value")) + }) + It("should not schedule pods that have node selectors with Exists operator and undefined key", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpExists}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule pods that with DoesNotExists operator and undefined key", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).ToNot(HaveKey("test-key")) + }) + It("should schedule unconstrained pods that don't have matching node selectors", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should schedule pods that have node selectors with matching value and In operator", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should not schedule pods that have node selectors with matching value and NotIn operator", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule the pod with Exists operator and defined key", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpExists}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should not schedule the pod with DoesNotExists operator and defined key", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule pods that have node selectors with different value and In operator", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule pods that have node selectors with different value and NotIn operator", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should schedule compatible pods to the same node", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}), + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, + }}), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + node1 := ExpectScheduled(ctx, env.Client, pods[0]) + node2 := ExpectScheduled(ctx, env.Client, pods[1]) + Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) + Expect(node2.Labels).To(HaveKeyWithValue("test-key", "test-value")) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should schedule incompatible pods to the different node", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}), + test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, + }}), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + node1 := ExpectScheduled(ctx, env.Client, pods[0]) + node2 := ExpectScheduled(ctx, env.Client, pods[1]) + Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) + Expect(node2.Labels).To(HaveKeyWithValue("test-key", "another-value")) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("Exists operator should not overwrite the existing value", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"non-existent-zone"}}, + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + }) + }) + + Describe("Preferential Fallback", func() { + Context("Required", func() { + It("should not relax the final term", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type"}}, + } + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, // Should not be relaxed + }}, + }}}} + // Don't relax + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should relax multiple terms", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, + }}, + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, // OR operator, never get to this one + }}, + }}}} + // Success + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1")) + }) + }) + Context("Preferred", func() { + It("should relax all terms", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + }, + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + }, + }}} + // Success + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should relax to use lighter weights", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 100, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, + }}, + }, + { + Weight: 50, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, + }}, + }, + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ // OR operator, never get to this one + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, + }}, + }, + }}} + // Success + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should schedule even if preference is conflicting with requirement", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-3"}}, + }}, + }, + }, + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, // Should not be relaxed + }}, + }}, + }} + // Success + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should schedule even if preference requirements are conflicting", func() { + pod := test.UnschedulablePod(test.PodOptions{NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"invalid"}}, + }}) + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + }) + }) + + Describe("Instance Type Compatibility", func() { + It("should not schedule if requesting more resources than any instance type has", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("512"), + }}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should launch pods with different archs on different instances", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1beta1.ArchitectureArm64, v1beta1.ArchitectureAmd64}, + }} + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: v1beta1.ArchitectureAmd64}, + }), + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: v1beta1.ArchitectureArm64}, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should exclude instance types that are not supported by the pod constraints (node affinity/instance type)", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1beta1.ArchitectureAmd64}, + }} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"arm-instance-type"}, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + // arm instance type conflicts with the nodePool limitation of AMD only + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should exclude instance types that are not supported by the pod constraints (node affinity/operating system)", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1beta1.ArchitectureAmd64}, + }} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"ios"}, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + // there's an instance with an OS of ios, but it has an arm processor so the provider requirements will + // exclude it + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should exclude instance types that are not supported by the provider constraints (arch)", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1beta1.ArchitectureAmd64}, + }} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: resource.MustParse("14")}}}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + // only the ARM instance has enough CPU, but it's not allowed per the nodePool + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should launch pods with different operating systems on different instances", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1beta1.ArchitectureArm64, v1beta1.ArchitectureAmd64}, + }} + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}, + }), + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Windows)}, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should launch pods with different instance type node selectors on different instances", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1beta1.ArchitectureArm64, v1beta1.ArchitectureAmd64}, + }} + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelInstanceType: "small-instance-type"}, + }), + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should launch pods with different zone selectors on different instances", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1beta1.ArchitectureArm64, v1beta1.ArchitectureAmd64}, + }} + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}, + }), + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should launch pods with resources that aren't on any single instance type on different instances", func() { + cloudProvider.InstanceTypes = fake.InstanceTypes(5) + const fakeGPU1 = "karpenter.sh/super-great-gpu" + const fakeGPU2 = "karpenter.sh/even-better-gpu" + cloudProvider.InstanceTypes[0].Capacity[fakeGPU1] = resource.MustParse("25") + cloudProvider.InstanceTypes[1].Capacity[fakeGPU2] = resource.MustParse("25") + + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{fakeGPU1: resource.MustParse("1")}, + }, + }), + // Should pack onto a different instance since no instance type has both GPUs + test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{fakeGPU2: resource.MustParse("1")}, + }, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should fail to schedule a pod with resources requests that aren't on a single instance type", func() { + cloudProvider.InstanceTypes = fake.InstanceTypes(5) + const fakeGPU1 = "karpenter.sh/super-great-gpu" + const fakeGPU2 = "karpenter.sh/even-better-gpu" + cloudProvider.InstanceTypes[0].Capacity[fakeGPU1] = resource.MustParse("25") + cloudProvider.InstanceTypes[1].Capacity[fakeGPU2] = resource.MustParse("25") + + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + fakeGPU1: resource.MustParse("1"), + fakeGPU2: resource.MustParse("1")}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + Context("Provider Specific Labels", func() { + It("should filter instance types that match labels", func() { + cloudProvider.InstanceTypes = fake.InstanceTypes(5) + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{fake.LabelInstanceSize: "large"}}), + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{fake.LabelInstanceSize: "small"}}), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + node := ExpectScheduled(ctx, env.Client, pods[0]) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "fake-it-4")) + node = ExpectScheduled(ctx, env.Client, pods[1]) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "fake-it-0")) + }) + It("should not schedule with incompatible labels", func() { + cloudProvider.InstanceTypes = fake.InstanceTypes(5) + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{ + fake.LabelInstanceSize: "large", + v1.LabelInstanceTypeStable: cloudProvider.InstanceTypes[0].Name, + }}), + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{ + fake.LabelInstanceSize: "small", + v1.LabelInstanceTypeStable: cloudProvider.InstanceTypes[4].Name, + }}), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + ExpectNotScheduled(ctx, env.Client, pods[0]) + ExpectNotScheduled(ctx, env.Client, pods[1]) + }) + It("should schedule optional labels", func() { + cloudProvider.InstanceTypes = fake.InstanceTypes(5) + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + // Only some instance types have this key + {Key: fake.ExoticInstanceLabelKey, Operator: v1.NodeSelectorOpExists}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKey(fake.ExoticInstanceLabelKey)) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, cloudProvider.InstanceTypes[4].Name)) + }) + It("should schedule without optional labels if disallowed", func() { + cloudProvider.InstanceTypes = fake.InstanceTypes(5) + ExpectApplied(ctx, env.Client, test.NodePool()) + pod := test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + // Only some instance types have this key + {Key: fake.ExoticInstanceLabelKey, Operator: v1.NodeSelectorOpDoesNotExist}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).ToNot(HaveKey(fake.ExoticInstanceLabelKey)) + }) + }) + }) + + Describe("Binpacking", func() { + It("should schedule a small pod on the smallest instance", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("100M"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) + }) + It("should schedule a small pod on the smallest possible instance type", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("2000M"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) + }) + It("should take pod runtime class into consideration", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1"), + }, + }}) + // the pod has overhead of 2 CPUs + runtimeClass := &nodev1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-runtime-class", + }, + Handler: "default", + Overhead: &nodev1.Overhead{ + PodFixed: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }, + }, + } + pod.Spec.RuntimeClassName = &runtimeClass.Name + ExpectApplied(ctx, env.Client, runtimeClass) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + // overhead of 2 + request of 1 = at least 3 CPUs, so it won't fit on small-instance-type which it otherwise + // would + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) + }) + It("should schedule multiple small pods on the smallest possible instance type", func() { + opts := test.PodOptions{ + Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("10M"), + }, + }} + pods := test.Pods(5, opts) + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + nodeNames := sets.NewString() + for _, p := range pods { + node := ExpectScheduled(ctx, env.Client, p) + nodeNames.Insert(node.Name) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) + } + Expect(nodeNames).To(HaveLen(1)) + }) + It("should create new nodes when a node is at capacity", func() { + opts := test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, + Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1.8G"), + }, + }} + ExpectApplied(ctx, env.Client, nodePool) + pods := test.Pods(40, opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + nodeNames := sets.NewString() + for _, p := range pods { + node := ExpectScheduled(ctx, env.Client, p) + nodeNames.Insert(node.Name) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) + } + Expect(nodeNames).To(HaveLen(20)) + }) + It("should pack small and large pods together", func() { + largeOpts := test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, + Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1.8G"), + }, + }} + smallOpts := test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, + Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("400M"), + }, + }} + + // Two large pods are all that will fit on the default-instance type (the largest instance type) which will create + // twenty nodes. This leaves just enough room on each of those nodes for one additional small pod per node, so we + // should only end up with 20 nodes total. + provPods := append(test.Pods(40, largeOpts), test.Pods(20, smallOpts)...) + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, provPods...) + nodeNames := sets.NewString() + for _, p := range provPods { + node := ExpectScheduled(ctx, env.Client, p) + nodeNames.Insert(node.Name) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) + } + Expect(nodeNames).To(HaveLen(20)) + }) + It("should pack nodes tightly", func() { + cloudProvider.InstanceTypes = fake.InstanceTypes(5) + var nodes []*v1.Node + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.5")}, + }, + }), + test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodes = append(nodes, node) + } + Expect(nodes).To(HaveLen(2)) + // the first pod consumes nearly all CPU of the largest instance type with no room for the second pod, the + // second pod is much smaller in terms of resources and should get a smaller node + Expect(nodes[0].Labels[v1.LabelInstanceTypeStable]).ToNot(Equal(nodes[1].Labels[v1.LabelInstanceTypeStable])) + }) + It("should handle zero-quantity resource requests", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")}, + Limits: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + // requesting a resource of quantity zero of a type unsupported by any instance is fine + ExpectScheduled(ctx, env.Client, pod) + }) + It("should not schedule pods that exceed every instance type's capacity", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("2Ti"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should create new nodes when a node is at capacity due to pod limits per node", func() { + opts := test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, + Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1m"), + v1.ResourceCPU: resource.MustParse("1m"), + }, + }} + ExpectApplied(ctx, env.Client, nodePool) + pods := test.Pods(25, opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + nodeNames := sets.NewString() + // all of the test instance types support 5 pods each, so we use the 5 instances of the smallest one for our 25 pods + for _, p := range pods { + node := ExpectScheduled(ctx, env.Client, p) + nodeNames.Insert(node.Name) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) + } + Expect(nodeNames).To(HaveLen(5)) + }) + It("should take into account initContainer resource requests when binpacking", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1Gi"), + v1.ResourceCPU: resource.MustParse("1"), + }, + }, + InitImage: "pause", + InitResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1Gi"), + v1.ResourceCPU: resource.MustParse("2"), + }, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) + }) + It("should not schedule pods when initContainer resource requests are greater than available instance types", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1Gi"), + v1.ResourceCPU: resource.MustParse("1"), + }, + }, + InitImage: "pause", + InitResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1Ti"), + v1.ResourceCPU: resource.MustParse("2"), + }, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should select for valid instance types, regardless of price", func() { + // capacity sizes and prices don't correlate here, regardless we should filter and see that all three instance types + // are valid before preferring the cheapest one 'large' + cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ + fake.NewInstanceType(fake.InstanceTypeOptions{ + Name: "medium", + Resources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + Offerings: []cloudprovider.Offering{ + { + CapacityType: v1beta1.CapacityTypeOnDemand, + Zone: "test-zone-1a", + Price: 3.00, + Available: true, + }, + }, + }), + fake.NewInstanceType(fake.InstanceTypeOptions{ + Name: "small", + Resources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + Offerings: []cloudprovider.Offering{ + { + CapacityType: v1beta1.CapacityTypeOnDemand, + Zone: "test-zone-1a", + Price: 2.00, + Available: true, + }, + }, + }), + fake.NewInstanceType(fake.InstanceTypeOptions{ + Name: "large", + Resources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + Offerings: []cloudprovider.Offering{ + { + CapacityType: v1beta1.CapacityTypeOnDemand, + Zone: "test-zone-1a", + Price: 1.00, + Available: true, + }, + }, + }), + } + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1m"), + v1.ResourceMemory: resource.MustParse("1Mi"), + }, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + // large is the cheapest, so we should pick it, but the other two types are also valid options + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("large")) + // all three options should be passed to the cloud provider + possibleInstanceType := sets.NewString(pscheduling.NewNodeSelectorRequirements(cloudProvider.CreateCalls[0].Spec.Requirements...).Get(v1.LabelInstanceTypeStable).Values()...) + Expect(possibleInstanceType).To(Equal(sets.NewString("small", "medium", "large"))) + }) + }) + + Describe("In-Flight Nodes", func() { + It("should not launch a second node if there is an in-flight node that can support the pod", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should not launch a second node if there is an in-flight node that can support the pod (node selectors)", func() { + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-2"}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + // the node gets created in test-zone-2 + secondPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1", "test-zone-2"}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + // test-zone-2 is in the intersection of their node selectors and the node has capacity, so we shouldn't create a new node + node2 := ExpectScheduled(ctx, env.Client, secondPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + Expect(node1.Name).To(Equal(node2.Name)) + + // the node gets created in test-zone-2 + thirdPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1", "test-zone-3"}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, thirdPod) + // node is in test-zone-2, so this pod needs a new node + node3 := ExpectScheduled(ctx, env.Client, thirdPod) + Expect(node1.Name).ToNot(Equal(node3.Name)) + }) + It("should launch a second node if a pod won't fit on the existingNodes node", func() { + ExpectApplied(ctx, env.Client, nodePool) + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1001m"), + }, + }} + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + // the node will have 2000m CPU, so these two pods can't both fit on it + opts.ResourceRequirements.Limits[v1.ResourceCPU] = resource.MustParse("1") + secondPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("should launch a second node if a pod isn't compatible with the existingNodes node (node selector)", func() { + ExpectApplied(ctx, env.Client, nodePool) + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("should launch a second node if an in-flight node is terminating", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod(opts) + bindings := ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + ExpectScheduled(ctx, env.Client, initialPod) + + // delete the node/nodeclaim + nodeClaim1 := bindings.Get(initialPod).NodeClaim + node1 := bindings.Get(initialPod).Node + nodeClaim1.Finalizers = nil + node1.Finalizers = nil + ExpectApplied(ctx, env.Client, nodeClaim1, node1) + ExpectDeleted(ctx, env.Client, nodeClaim1, node1) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + ExpectReconcileSucceeded(ctx, nodeClaimStateController, client.ObjectKeyFromObject(nodeClaim1)) + + secondPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + Context("Topology", func() { + It("should balance pods across zones with in-flight nodes", func() { + labels := map[string]string{"foo": "bar"} + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, + test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) + + // reconcile our nodes with the cluster state so they'll show up as in-flight + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + for _, node := range nodeList.Items { + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKey{Name: node.Name}) + } + + firstRoundNumNodes := len(nodeList.Items) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, + test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 5)..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 3)) + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + + // shouldn't create any new nodes as the in-flight ones can support the pods + Expect(nodeList.Items).To(HaveLen(firstRoundNumNodes)) + }) + It("should balance pods across hostnames with in-flight nodes", func() { + labels := map[string]string{"foo": "bar"} + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, + test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1)) + + // reconcile our nodes with the cluster state so they'll show up as in-flight + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + for _, node := range nodeList.Items { + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKey{Name: node.Name}) + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, + test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 5)..., + ) + // we prefer to launch new nodes to satisfy the topology spread even though we could technically schedule against existingNodes + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1, 1, 1, 1, 1, 1)) + }) + }) + Context("Taints", func() { + It("should assume pod will schedule to a tainted node with no taints", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + }, + }} + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + node1.Spec.Taints = nil + ExpectApplied(ctx, env.Client, node1) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should not assume pod will schedule to a tainted node", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + }, + }} + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod(opts) + bindings := ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + ExpectScheduled(ctx, env.Client, initialPod) + + nodeClaim1 := bindings.Get(initialPod).NodeClaim + node1 := bindings.Get(initialPod).Node + nodeClaim1.StatusConditions().MarkTrue(v1beta1.Initialized) + node1.Labels = lo.Assign(node1.Labels, map[string]string{v1beta1.NodeInitializedLabelKey: "true"}) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + // and taint it + node1.Spec.Taints = append(node1.Spec.Taints, v1.Taint{ + Key: "foo.com/taint", + Value: "tainted", + Effect: v1.TaintEffectNoSchedule, + }) + ExpectApplied(ctx, env.Client, nodeClaim1, node1) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("should assume pod will schedule to a tainted node with a custom startup taint", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + }, + }} + nodePool.Spec.Template.Spec.StartupTaints = append(nodePool.Spec.Template.Spec.StartupTaints, v1.Taint{ + Key: "foo.com/taint", + Value: "tainted", + Effect: v1.TaintEffectNoSchedule, + }) + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + // startup taint + node not ready taint = 2 + Expect(node1.Spec.Taints).To(HaveLen(2)) + Expect(node1.Spec.Taints).To(ContainElement(v1.Taint{ + Key: "foo.com/taint", + Value: "tainted", + Effect: v1.TaintEffectNoSchedule, + })) + ExpectApplied(ctx, env.Client, node1) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should not assume pod will schedule to a node with startup taints after initialization", func() { + startupTaint := v1.Taint{Key: "ignore-me", Value: "nothing-to-see-here", Effect: v1.TaintEffectNoSchedule} + nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{startupTaint} + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod() + bindings := ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + ExpectScheduled(ctx, env.Client, initialPod) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + + // Mark it initialized which only occurs once the startup taint was removed and re-apply only the startup taint. + // We also need to add resource capacity as after initialization we assume that kubelet has recorded them. + + nodeClaim1 := bindings.Get(initialPod).NodeClaim + node1 := bindings.Get(initialPod).Node + nodeClaim1.StatusConditions().MarkTrue(v1beta1.Initialized) + node1.Labels = lo.Assign(node1.Labels, map[string]string{v1beta1.NodeInitializedLabelKey: "true"}) + + node1.Spec.Taints = []v1.Taint{startupTaint} + node1.Status.Capacity = v1.ResourceList{v1.ResourcePods: resource.MustParse("10")} + ExpectApplied(ctx, env.Client, nodeClaim1, node1) + + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + // we should launch a new node since the startup taint is there, but was gone at some point + secondPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("should consider a tainted NotReady node as in-flight even if initialized", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: resource.MustParse("10m")}, + }} + ExpectApplied(ctx, env.Client, nodePool) + + // Schedule to New NodeClaim + pod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node1 := ExpectScheduled(ctx, env.Client, pod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + // Mark Initialized + node1.Labels[v1beta1.NodeInitializedLabelKey] = "true" + node1.Spec.Taints = []v1.Taint{ + {Key: v1.TaintNodeNotReady, Effect: v1.TaintEffectNoSchedule}, + {Key: v1.TaintNodeUnreachable, Effect: v1.TaintEffectNoSchedule}, + {Key: cloudproviderapi.TaintExternalCloudProvider, Effect: v1.TaintEffectNoSchedule, Value: "true"}, + } + ExpectApplied(ctx, env.Client, node1) + // Schedule to In Flight NodeClaim + pod = test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node2 := ExpectScheduled(ctx, env.Client, pod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) + + Expect(node1.Name).To(Equal(node2.Name)) + }) + }) + Context("Daemonsets", func() { + It("should track daemonset usage separately so we know how many DS resources are remaining to be scheduled", func() { + ds := test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi")}}, + }}, + ) + ExpectApplied(ctx, env.Client, nodePool, ds) + Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(ds), ds)).To(Succeed()) + + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + }, + }} + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + + // create our daemonset pod and manually bind it to the node + dsPod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }}, + }) + dsPod.OwnerReferences = append(dsPod.OwnerReferences, metav1.OwnerReference{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + Name: ds.Name, + UID: ds.UID, + Controller: ptr.Bool(true), + BlockOwnerDeletion: ptr.Bool(true), + }) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + ExpectApplied(ctx, env.Client, nodePool, dsPod) + cluster.ForEachNode(func(f *state.StateNode) bool { + dsRequests := f.DaemonSetRequests() + available := f.Available() + Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 0)) + // no pods so we have the full (16 cpu - 100m overhead) + Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 15.9)) + return true + }) + ExpectManualBinding(ctx, env.Client, dsPod, node1) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(dsPod)) + + cluster.ForEachNode(func(f *state.StateNode) bool { + dsRequests := f.DaemonSetRequests() + available := f.Available() + Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 1)) + // only the DS pod is bound, so available is reduced by one and the DS requested is incremented by one + Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 14.9)) + return true + }) + + opts = test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("14.9"), + }, + }} + // this pod should schedule on the existingNodes node as the daemonset pod has already bound, meaning that the + // remaining daemonset resources should be zero leaving 14.9 CPUs for the pod + secondPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should handle unexpected daemonset pods binding to the node", func() { + ds1 := test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + NodeSelector: map[string]string{ + "my-node-label": "value", + }, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi")}}, + }}, + ) + ds2 := test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1m"), + }}}}) + ExpectApplied(ctx, env.Client, nodePool, ds1, ds2) + Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(ds1), ds1)).To(Succeed()) + + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + }, + }} + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + // this label appears on the node for some reason that Karpenter can't track + node1.Labels["my-node-label"] = "value" + ExpectApplied(ctx, env.Client, node1) + + // create our daemonset pod and manually bind it to the node + dsPod := test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{ + "my-node-label": "value", + }, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }}, + }) + dsPod.OwnerReferences = append(dsPod.OwnerReferences, metav1.OwnerReference{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + Name: ds1.Name, + UID: ds1.UID, + Controller: ptr.Bool(true), + BlockOwnerDeletion: ptr.Bool(true), + }) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + ExpectApplied(ctx, env.Client, nodePool, dsPod) + cluster.ForEachNode(func(f *state.StateNode) bool { + dsRequests := f.DaemonSetRequests() + available := f.Available() + Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 0)) + // no pods, so we have the full (16 CPU - 100m overhead) + Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 15.9)) + return true + }) + ExpectManualBinding(ctx, env.Client, dsPod, node1) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(dsPod)) + + cluster.ForEachNode(func(f *state.StateNode) bool { + dsRequests := f.DaemonSetRequests() + available := f.Available() + Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 1)) + // only the DS pod is bound, so available is reduced by one and the DS requested is incremented by one + Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 14.9)) + return true + }) + + opts = test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("15.5"), + }, + }} + // This pod should not schedule on the inflight node as it requires more CPU than we have. This verifies + // we don't reintroduce a bug where more daemonsets scheduled than anticipated due to unexepected labels + // appearing on the node which caused us to compute a negative amount of resources remaining for daemonsets + // which in turn caused us to mis-calculate the amount of resources that were free on the node. + secondPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + // must create a new node + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + + }) + // nolint:gosec + It("should pack in-flight nodes before launching new nodes", func() { + cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ + fake.NewInstanceType(fake.InstanceTypeOptions{ + Name: "medium", + Resources: v1.ResourceList{ + // enough CPU for four pods + a bit of overhead + v1.ResourceCPU: resource.MustParse("4.25"), + v1.ResourcePods: resource.MustParse("4"), + }, + }), + } + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1"), + }, + }} + + ExpectApplied(ctx, env.Client, nodePool) + + // scheduling in multiple batches random sets of pods + for i := 0; i < 10; i++ { + initialPods := test.UnschedulablePods(opts, rand.Intn(10)) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPods...) + for _, pod := range initialPods { + node := ExpectScheduled(ctx, env.Client, pod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + } + } + + // due to the in-flight node support, we should pack existing nodes before launching new node. The end result + // is that we should only have some spare capacity on our final node + nodesWithCPUFree := 0 + cluster.ForEachNode(func(n *state.StateNode) bool { + available := n.Available() + if available.Cpu().AsApproximateFloat64() >= 1 { + nodesWithCPUFree++ + } + return true + }) + Expect(nodesWithCPUFree).To(BeNumerically("<=", 1)) + }) + It("should not launch a second node if there is an in-flight node that can support the pod (#2011)", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(opts) + ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pod) + var nodes v1.NodeList + Expect(env.Client.List(ctx, &nodes)).To(Succeed()) + Expect(nodes.Items).To(HaveLen(1)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodes.Items[0])) + + pod.Status.Conditions = []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}} + ExpectApplied(ctx, env.Client, pod) + ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pod) + Expect(env.Client.List(ctx, &nodes)).To(Succeed()) + // shouldn't create a second node + Expect(nodes.Items).To(HaveLen(1)) + }) + It("should order initialized nodes for scheduling un-initialized nodes when all other nodes are inflight", func() { + ExpectApplied(ctx, env.Client, nodePool) + + var nodeClaims []*v1beta1.NodeClaim + var node *v1.Node + //nolint:gosec + elem := rand.Intn(100) // The nodeclaim/node that will be marked as initialized + for i := 0; i < 100; i++ { + nc := test.NodeClaim(v1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + v1beta1.NodePoolLabelKey: nodePool.Name, + }, + }, + }) + ExpectApplied(ctx, env.Client, nc) + if i == elem { + nc, node = ExpectNodeClaimDeployed(ctx, env.Client, cluster, cloudProvider, nc) + } else { + var err error + nc, err = ExpectNodeClaimDeployedNoNode(ctx, env.Client, cluster, cloudProvider, nc) + Expect(err).ToNot(HaveOccurred()) + } + nodeClaims = append(nodeClaims, nc) + } + + // Make one of the nodes and nodeClaims initialized + ExpectMakeNodeClaimsInitialized(ctx, env.Client, nodeClaims[elem]) + ExpectMakeNodesInitialized(ctx, env.Client, node) + ExpectReconcileSucceeded(ctx, nodeClaimStateController, client.ObjectKeyFromObject(nodeClaims[elem])) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + scheduledNode := ExpectScheduled(ctx, env.Client, pod) + + // Expect that the scheduled node is equal to node3 since it's initialized + Expect(scheduledNode.Name).To(Equal(node.Name)) + }) + }) + + Describe("Existing Nodes", func() { + It("should schedule a pod to an existing node unowned by Karpenter", func() { + node := test.Node(test.NodeOptions{ + Allocatable: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("10Gi"), + v1.ResourcePods: resource.MustParse("110"), + }, + }) + ExpectApplied(ctx, env.Client, node) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + scheduledNode := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Name).To(Equal(scheduledNode.Name)) + }) + It("should schedule multiple pods to an existing node unowned by Karpenter", func() { + node := test.Node(test.NodeOptions{ + Allocatable: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("100Gi"), + v1.ResourcePods: resource.MustParse("110"), + }, + }) + ExpectApplied(ctx, env.Client, node) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + ExpectApplied(ctx, env.Client, nodePool) + pods := test.UnschedulablePods(opts, 100) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + + for _, pod := range pods { + scheduledNode := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Name).To(Equal(scheduledNode.Name)) + } + }) + It("should order initialized nodes for scheduling un-initialized nodes", func() { + ExpectApplied(ctx, env.Client, nodePool) + + var nodeClaims []*v1beta1.NodeClaim + var nodes []*v1.Node + for i := 0; i < 100; i++ { + nc := test.NodeClaim(v1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + v1beta1.NodePoolLabelKey: nodePool.Name, + }, + }, + }) + ExpectApplied(ctx, env.Client, nc) + nc, n := ExpectNodeClaimDeployed(ctx, env.Client, cluster, cloudProvider, nc) + nodeClaims = append(nodeClaims, nc) + nodes = append(nodes, n) + } + + // Make one of the nodes and nodeClaims initialized + elem := rand.Intn(100) //nolint:gosec + ExpectMakeNodeClaimsInitialized(ctx, env.Client, nodeClaims[elem]) + ExpectMakeNodesInitialized(ctx, env.Client, nodes[elem]) + ExpectReconcileSucceeded(ctx, nodeClaimStateController, client.ObjectKeyFromObject(nodeClaims[elem])) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(nodes[elem])) + + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + scheduledNode := ExpectScheduled(ctx, env.Client, pod) + + // Expect that the scheduled node is equal to the ready node since it's initialized + Expect(scheduledNode.Name).To(Equal(nodes[elem].Name)) + }) + It("should consider a pod incompatible with an existing node but compatible with NodePool", func() { + nodeClaim, node := test.NodeClaimAndNode(v1beta1.NodeClaim{ + Status: v1beta1.NodeClaimStatus{ + Allocatable: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("10Gi"), + v1.ResourcePods: resource.MustParse("110"), + }, + }, + }) + ExpectApplied(ctx, env.Client, nodeClaim, node) + ExpectMakeNodeClaimsInitialized(ctx, env.Client, nodeClaim) + ExpectMakeNodesInitialized(ctx, env.Client, node) + + ExpectReconcileSucceeded(ctx, nodeClaimStateController, client.ObjectKeyFromObject(nodeClaim)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + pod := test.UnschedulablePod(test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1"}, + }, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + Context("Daemonsets", func() { + It("should not subtract daemonset overhead that is not strictly compatible with an existing node", func() { + nodeClaim, node := test.NodeClaimAndNode(v1beta1.NodeClaim{ + Status: v1beta1.NodeClaimStatus{ + Allocatable: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi"), + v1.ResourcePods: resource.MustParse("110"), + }, + }, + }) + // This DaemonSet is not compatible with the existing NodeClaim/Node + ds := test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100"), + v1.ResourceMemory: resource.MustParse("100Gi")}, + }, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1"}, + }, + }, + }}, + ) + ExpectApplied(ctx, env.Client, nodePool, nodeClaim, node, ds) + ExpectMakeNodeClaimsInitialized(ctx, env.Client, nodeClaim) + ExpectMakeNodesInitialized(ctx, env.Client, node) + + ExpectReconcileSucceeded(ctx, nodeClaimStateController, client.ObjectKeyFromObject(nodeClaim)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + scheduledNode := ExpectScheduled(ctx, env.Client, pod) + Expect(scheduledNode.Name).To(Equal(node.Name)) + + // Add another pod and expect that pod not to schedule against a nodePool since we will model the DS against the nodePool + // In this case, the DS overhead will take over the entire capacity for every "theoretical node" so we can't schedule a new pod to any new Node + pod2 := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }) + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod2) + ExpectNotScheduled(ctx, env.Client, pod2) + }) + }) + }) + + Describe("No Pre-Binding", func() { + It("should not bind pods to nodes", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + + var nodeList v1.NodeList + // shouldn't have any nodes + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(0)) + + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod(opts) + ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + ExpectNotScheduled(ctx, env.Client, initialPod) + + // should launch a single node + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(1)) + node1 := &nodeList.Items[0] + + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + secondPod := test.UnschedulablePod(opts) + ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + ExpectNotScheduled(ctx, env.Client, secondPod) + // shouldn't create a second node as it can bind to the existingNodes node + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(1)) + }) + It("should handle resource zeroing of extended resources by kubelet", func() { + // Issue #1459 + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + fake.ResourceGPUVendorA: resource.MustParse("1"), + }, + }} + + var nodeList v1.NodeList + // shouldn't have any nodes + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(0)) + + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod(opts) + ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + ExpectNotScheduled(ctx, env.Client, initialPod) + + // should launch a single node + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(1)) + node1 := &nodeList.Items[0] + + // simulate kubelet zeroing out the extended resources on the node at startup + node1.Status.Capacity = map[v1.ResourceName]resource.Quantity{ + fake.ResourceGPUVendorA: resource.MustParse("0"), + } + node1.Status.Allocatable = map[v1.ResourceName]resource.Quantity{ + fake.ResourceGPUVendorB: resource.MustParse("0"), + } + + ExpectApplied(ctx, env.Client, node1) + + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + secondPod := test.UnschedulablePod(opts) + ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, secondPod) + ExpectNotScheduled(ctx, env.Client, secondPod) + // shouldn't create a second node as it can bind to the existingNodes node + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(1)) + }) + It("should respect self pod affinity without pod binding (zone)", func() { + // Issue #1975 + affLabels := map[string]string{"security": "s2"} + + pods := test.UnschedulablePods(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: affLabels, + }, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }}, + }, 2) + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pods[0]) + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + for i := range nodeList.Items { + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodeList.Items[i])) + } + // the second pod can schedule against the in-flight node, but for that to work we need to be careful + // in how we fulfill the self-affinity by taking the existing node's domain as a preference over any + // random viable domain + ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, pods[1]) + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(1)) + }) + }) + + Describe("VolumeUsage", func() { + BeforeEach(func() { + cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ + fake.NewInstanceType( + fake.InstanceTypeOptions{ + Name: "instance-type", + Resources: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1024"), + v1.ResourcePods: resource.MustParse("1024"), + }, + }), + } + nodePool.Spec.Limits = nil + }) + It("should launch multiple nodes if required due to volume limits", func() { + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + csiNode := &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: csiProvider, + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(10), + }, + }, + }, + }, + } + ExpectApplied(ctx, env.Client, csiNode) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, + Provisioner: ptr.String(csiProvider), + Zones: []string{"test-zone-1"}}) + ExpectApplied(ctx, env.Client, sc) + + var pods []*v1.Pod + for i := 0; i < 6; i++ { + pvcA := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + StorageClassName: ptr.String("my-storage-class"), + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("my-claim-a-%d", i)}, + }) + pvcB := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + StorageClassName: ptr.String("my-storage-class"), + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("my-claim-b-%d", i)}, + }) + ExpectApplied(ctx, env.Client, pvcA, pvcB) + pods = append(pods, test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{pvcA.Name, pvcB.Name}, + })) + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + // we need to create a new node as the in-flight one can only contain 5 pods due to the CSINode volume limit + Expect(nodeList.Items).To(HaveLen(2)) + }) + It("should launch a single node if all pods use the same PVC", func() { + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + csiNode := &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: csiProvider, + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(10), + }, + }, + }, + }, + } + ExpectApplied(ctx, env.Client, csiNode) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, + Provisioner: ptr.String(csiProvider), + Zones: []string{"test-zone-1"}}) + ExpectApplied(ctx, env.Client, sc) + + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, + Zones: []string{"test-zone-1"}}) + + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, + StorageClassName: ptr.String("my-storage-class"), + VolumeName: pv.Name, + }) + ExpectApplied(ctx, env.Client, pv, pvc) + + var pods []*v1.Pod + for i := 0; i < 100; i++ { + pods = append(pods, test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name}, + })) + } + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + // 100 of the same PVC should all be schedulable on the same node + Expect(nodeList.Items).To(HaveLen(1)) + }) + It("should not fail for NFS volumes", func() { + ExpectApplied(ctx, env.Client, nodePool) + initialPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, + StorageClassName: "nfs", + Zones: []string{"test-zone-1"}}) + pv.Spec.NFS = &v1.NFSVolumeSource{ + Server: "fake.server", + Path: "/some/path", + } + pv.Spec.CSI = nil + + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, + VolumeName: pv.Name, + StorageClassName: ptr.String(""), + }) + ExpectApplied(ctx, env.Client, pv, pvc) + + var pods []*v1.Pod + for i := 0; i < 5; i++ { + pods = append(pods, test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name, pvc.Name}, + })) + } + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + // 5 of the same PVC should all be schedulable on the same node + Expect(nodeList.Items).To(HaveLen(1)) + }) + It("should launch nodes for pods with ephemeral volume using the specified storage class name", func() { + // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-storage-class", + }, + Provisioner: ptr.String(csiProvider), + Zones: []string{"test-zone-1"}}) + // Create another default storage class that shouldn't be used and has no associated limits + sc2 := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-storage-class", + Annotations: map[string]string{ + pscheduling.IsDefaultStorageClassAnnotation: "true", + }, + }, + Provisioner: ptr.String("other-provider"), + Zones: []string{"test-zone-1"}}) + + initialPod := test.UnschedulablePod(test.PodOptions{}) + // Pod has an ephemeral volume claim that has a specified storage class, so it should use the one specified + initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ + Name: "tmp-ephemeral", + VolumeSource: v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: v1.PersistentVolumeClaimSpec{ + StorageClassName: lo.ToPtr(sc.Name), + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool, sc, sc2, initialPod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + csiNode := &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: csiProvider, + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(1), + }, + }, + { + Name: "other-provider", + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(10), + }, + }, + }, + }, + } + ExpectApplied(ctx, env.Client, csiNode) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + pod := test.UnschedulablePod(test.PodOptions{}) + // Pod has an ephemeral volume claim that has a specified storage class, so it should use the one specified + pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ + Name: "tmp-ephemeral", + VolumeSource: v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: v1.PersistentVolumeClaimSpec{ + StorageClassName: lo.ToPtr(sc.Name), + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool, pod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node2 := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Name).ToNot(Equal(node2.Name)) + }) + It("should launch nodes for pods with ephemeral volume using a default storage class", func() { + // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-storage-class", + Annotations: map[string]string{ + pscheduling.IsDefaultStorageClassAnnotation: "true", + }, + }, + Provisioner: ptr.String(csiProvider), + Zones: []string{"test-zone-1"}}) + + initialPod := test.UnschedulablePod(test.PodOptions{}) + // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one + initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ + Name: "tmp-ephemeral", + VolumeSource: v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool, sc, initialPod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + csiNode := &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: csiProvider, + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(1), + }, + }, + }, + }, + } + ExpectApplied(ctx, env.Client, csiNode) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + pod := test.UnschedulablePod(test.PodOptions{}) + // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one + pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ + Name: "tmp-ephemeral", + VolumeSource: v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }) + + ExpectApplied(ctx, env.Client, sc, nodePool, pod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node2 := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Name).ToNot(Equal(node2.Name)) + }) + It("should launch nodes for pods with ephemeral volume using the newest storage class", func() { + if env.Version.Minor() < 26 { + Skip("Multiple default storage classes is only available in K8s >= 1.26.x") + } + // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-storage-class", + Annotations: map[string]string{ + pscheduling.IsDefaultStorageClassAnnotation: "true", + }, + }, + Provisioner: ptr.String("other-provider"), + Zones: []string{"test-zone-1"}}) + sc2 := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "newer-default-storage-class", + Annotations: map[string]string{ + pscheduling.IsDefaultStorageClassAnnotation: "true", + }, + }, + Provisioner: ptr.String(csiProvider), + Zones: []string{"test-zone-1"}}) + + ExpectApplied(ctx, env.Client, sc) + // Wait a few seconds to apply the second storage class to get a newer creationTimestamp + time.Sleep(time.Second * 2) + ExpectApplied(ctx, env.Client, sc2) + + initialPod := test.UnschedulablePod(test.PodOptions{}) + // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one + initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ + Name: "tmp-ephemeral", + VolumeSource: v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool, sc, initialPod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + csiNode := &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: csiProvider, + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(1), + }, + }, + { + Name: "other-provider", + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(10), + }, + }, + }, + }, + } + ExpectApplied(ctx, env.Client, csiNode) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + pod := test.UnschedulablePod(test.PodOptions{}) + // Pod has an ephemeral volume claim that has NO storage class, so it should use the default one + pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ + Name: "tmp-ephemeral", + VolumeSource: v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, sc, nodePool, pod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node2 := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Name).ToNot(Equal(node2.Name)) + }) + It("should not launch nodes for pods with ephemeral volume using a non-existent storage classes", func() { + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(test.PodOptions{}) + pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ + Name: "tmp-ephemeral", + VolumeSource: v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: v1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.String("non-existent"), + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + // no nodes should be created as the storage class doesn't eixst + Expect(nodeList.Items).To(HaveLen(0)) + }) + Context("CSIMigration", func() { + It("should launch nodes for pods with non-dynamic PVC using a migrated PVC/PV", func() { + // We should assume that this PVC/PV is using CSI driver implicitly to limit pod scheduling + // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "in-tree-storage-class", + Annotations: map[string]string{ + pscheduling.IsDefaultStorageClassAnnotation: "true", + }, + }, + Provisioner: ptr.String(plugins.AWSEBSInTreePluginName), + Zones: []string{"test-zone-1"}}) + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + StorageClassName: ptr.String(sc.Name), + }) + ExpectApplied(ctx, env.Client, nodePool, sc, pvc) + initialPod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + csiNode := &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: plugins.AWSEBSDriverName, + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(1), + }, + }, + }, + }, + } + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-volume", + }, + Zones: []string{"test-zone-1"}, + UseAWSInTreeDriver: true, + }) + ExpectApplied(ctx, env.Client, csiNode, pvc, pv) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + pvc2 := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + StorageClassName: ptr.String(sc.Name), + }) + pod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc2.Name}, + }) + ExpectApplied(ctx, env.Client, pvc2, pod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node2 := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Name).ToNot(Equal(node2.Name)) + }) + It("should launch nodes for pods with ephemeral volume using a migrated PVC/PV", func() { + // We should assume that this PVC/PV is using CSI driver implicitly to limit pod scheduling + // Launch an initial pod onto a node and register the CSI Node with a volume count limit of 1 + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "in-tree-storage-class", + Annotations: map[string]string{ + pscheduling.IsDefaultStorageClassAnnotation: "true", + }, + }, + Provisioner: ptr.String(plugins.AWSEBSInTreePluginName), + Zones: []string{"test-zone-1"}}) + + initialPod := test.UnschedulablePod(test.PodOptions{}) + // Pod has an ephemeral volume claim that references the in-tree storage provider + initialPod.Spec.Volumes = append(initialPod.Spec.Volumes, v1.Volume{ + Name: "tmp-ephemeral", + VolumeSource: v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: ptr.String(sc.Name), + }, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool, sc, initialPod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + csiNode := &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: plugins.AWSEBSDriverName, + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(1), + }, + }, + }, + }, + } + ExpectApplied(ctx, env.Client, csiNode) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + pod := test.UnschedulablePod(test.PodOptions{}) + // Pod has an ephemeral volume claim that reference the in-tree storage provider + pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ + Name: "tmp-ephemeral", + VolumeSource: v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: ptr.String(sc.Name), + }, + }, + }, + }, + }) + // Pod should not schedule to the first node since we should realize that we have hit our volume limits + ExpectApplied(ctx, env.Client, pod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node2 := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Name).ToNot(Equal(node2.Name)) + }) + }) + }) +}) + // nolint:gocyclo func ExpectMaxSkew(ctx context.Context, c client.Client, namespace string, constraint *v1.TopologySpreadConstraint) Assertion { GinkgoHelper() diff --git a/pkg/controllers/provisioning/scheduling/nodepool_topology_test.go b/pkg/controllers/provisioning/scheduling/topology_test.go similarity index 100% rename from pkg/controllers/provisioning/scheduling/nodepool_topology_test.go rename to pkg/controllers/provisioning/scheduling/topology_test.go diff --git a/pkg/controllers/provisioning/suite_test.go b/pkg/controllers/provisioning/suite_test.go index ce02a78446..7a379d6fec 100644 --- a/pkg/controllers/provisioning/suite_test.go +++ b/pkg/controllers/provisioning/suite_test.go @@ -16,7 +16,7 @@ package provisioning_test import ( "context" - "math/rand" + "fmt" "testing" "time" @@ -24,6 +24,7 @@ import ( . "github.com/onsi/gomega" "github.com/samber/lo" v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -31,9 +32,10 @@ import ( "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" + "knative.dev/pkg/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/karpenter-core/pkg/apis" - "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/cloudprovider/fake" @@ -95,569 +97,1469 @@ var _ = AfterEach(func() { cluster.Reset() }) -var _ = Describe("Combined/Provisioning", func() { - var provisioner *v1alpha5.Provisioner - var nodePool *v1beta1.NodePool - BeforeEach(func() { - provisioner = test.Provisioner() - nodePool = test.NodePool() +var _ = Describe("Provisioning", func() { + It("should provision nodes", func() { + ExpectApplied(ctx, env.Client, test.NodePool()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + nodes := &v1.NodeList{} + Expect(env.Client.List(ctx, nodes)).To(Succeed()) + Expect(len(nodes.Items)).To(Equal(1)) + ExpectScheduled(ctx, env.Client, pod) }) - It("should schedule pods using owner label selectors", func() { - ExpectApplied(ctx, env.Client, nodePool, provisioner) - - provisionerPod := test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{ - v1alpha5.ProvisionerNameLabelKey: provisioner.Name, - }, - }) - nodePoolPod := test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{ - v1beta1.NodePoolLabelKey: nodePool.Name, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, provisionerPod, nodePoolPod) - node := ExpectScheduled(ctx, env.Client, provisionerPod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) - node = ExpectScheduled(ctx, env.Client, nodePoolPod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) + It("should ignore NodePools that are deleting", func() { + nodePool := test.NodePool() + ExpectApplied(ctx, env.Client, nodePool) + ExpectDeletionTimestampSet(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + nodes := &v1.NodeList{} + Expect(env.Client.List(ctx, nodes)).To(Succeed()) + Expect(len(nodes.Items)).To(Equal(0)) + ExpectNotScheduled(ctx, env.Client, pod) }) - It("should schedule pods using custom labels", func() { - provisioner.Spec.Labels = map[string]string{ - "provisioner": "true", + It("should provision nodes for pods with supported node selectors", func() { + nodePool := test.NodePool() + schedulable := []*v1.Pod{ + // Constrained by nodepool + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: nodePool.Name}}), + // Constrained by zone + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), + // Constrained by instanceType + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), + // Constrained by architecture + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), + // Constrained by operatingSystem + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), } - nodePool.Spec.Template.Labels = map[string]string{ - "nodepool": "true", + unschedulable := []*v1.Pod{ + // Ignored, matches another nodepool + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: "unknown"}}), + // Ignored, invalid zone + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}), + // Ignored, invalid instance type + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "unknown"}}), + // Ignored, invalid architecture + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "unknown"}}), + // Ignored, invalid operating system + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: "unknown"}}), + // Ignored, invalid capacity type + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.CapacityTypeLabelKey: "unknown"}}), + // Ignored, label selector does not match + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{"foo": "bar"}}), + } + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) + for _, pod := range schedulable { + ExpectScheduled(ctx, env.Client, pod) + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, unschedulable...) + for _, pod := range unschedulable { + ExpectNotScheduled(ctx, env.Client, pod) } - ExpectApplied(ctx, env.Client, nodePool, provisioner) - - provisionerPod := test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{ - "provisioner": "true", - }, - }) - nodePoolPod := test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{ - "nodepool": "true", - }, - }) - - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, provisionerPod, nodePoolPod) - node := ExpectScheduled(ctx, env.Client, provisionerPod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) - node = ExpectScheduled(ctx, env.Client, nodePoolPod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) }) - It("should schedule pods using custom requirements", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: "provisioner", - Operator: v1.NodeSelectorOpIn, - Values: []string{"true"}, - }, + It("should provision nodes for pods with supported node affinities", func() { + nodePool := test.NodePool() + schedulable := []*v1.Pod{ + // Constrained by nodepool + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1beta1.NodePoolLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{nodePool.Name}}}}), + // Constrained by zone + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}}}), + // Constrained by instanceType + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type"}}}}), + // Constrained by architecture + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{"arm64"}}}}), + // Constrained by operatingSystem + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{string(v1.Linux)}}}}), + } + unschedulable := []*v1.Pod{ + // Ignored, matches another nodepool + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1beta1.NodePoolLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), + // Ignored, invalid zone + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), + // Ignored, invalid instance type + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), + // Ignored, invalid architecture + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), + // Ignored, invalid operating system + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), + // Ignored, invalid capacity type + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}}), + // Ignored, label selector does not match + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{Key: "foo", Operator: v1.NodeSelectorOpIn, Values: []string{"bar"}}}}), + } + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, schedulable...) + for _, pod := range schedulable { + ExpectScheduled(ctx, env.Client, pod) + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, unschedulable...) + for _, pod := range unschedulable { + ExpectNotScheduled(ctx, env.Client, pod) + } + }) + It("should provision nodes for accelerators", func() { + ExpectApplied(ctx, env.Client, test.NodePool()) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{fake.ResourceGPUVendorA: resource.MustParse("1")}}, + }), + test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{fake.ResourceGPUVendorB: resource.MustParse("1")}}, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + for _, pod := range pods { + ExpectScheduled(ctx, env.Client, pod) } - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: "nodepool", - Operator: v1.NodeSelectorOpIn, - Values: []string{"true"}, + }) + It("should provision multiple nodes when maxPods is set", func() { + // Kubelet is actually not observed here, the scheduler is relying on the + // pods resource value which is statically set in the fake cloudprovider + ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + Kubelet: &v1beta1.KubeletConfiguration{MaxPods: ptr.Int32(1)}, + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"single-pod-instance-type"}, + }, + }, + }, + }, }, + })) + pods := []*v1.Pod{ + test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod(), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + nodes := &v1.NodeList{} + Expect(env.Client.List(ctx, nodes)).To(Succeed()) + Expect(len(nodes.Items)).To(Equal(3)) + for _, pod := range pods { + ExpectScheduled(ctx, env.Client, pod) + } + }) + It("should schedule all pods on one inflight node when node is in deleting state", func() { + nodePool := test.NodePool() + its, err := cloudProvider.GetInstanceTypes(ctx, nodePool) + Expect(err).To(BeNil()) + node := test.Node(test.NodeOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + v1beta1.NodePoolLabelKey: nodePool.Name, + v1.LabelInstanceTypeStable: its[0].Name, + }, + Finalizers: []string{v1beta1.TerminationFinalizer}, + }}, + ) + ExpectApplied(ctx, env.Client, node, nodePool) + ExpectReconcileSucceeded(ctx, nodeController, client.ObjectKeyFromObject(node)) + + // Schedule 3 pods to the node that currently exists + for i := 0; i < 3; i++ { + pod := test.UnschedulablePod() + ExpectApplied(ctx, env.Client, pod) + ExpectManualBinding(ctx, env.Client, pod, node) } - ExpectApplied(ctx, env.Client, nodePool, provisioner) - provisionerPod := test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{ - "provisioner": "true", - }, - }) - nodePoolPod := test.UnschedulablePod(test.PodOptions{ - NodeSelector: map[string]string{ - "nodepool": "true", - }, - }) + // Node shouldn't fully delete since it has a finalizer + Expect(env.Client.Delete(ctx, node)).To(Succeed()) + ExpectReconcileSucceeded(ctx, nodeController, client.ObjectKeyFromObject(node)) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, provisionerPod, nodePoolPod) - node := ExpectScheduled(ctx, env.Client, provisionerPod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) - node = ExpectScheduled(ctx, env.Client, nodePoolPod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) + // Provision without a binding since some pods will already be bound + // Should all schedule to the new node, ignoring the old node + bindings := ExpectProvisionedNoBinding(ctx, env.Client, cluster, cloudProvider, prov, test.UnschedulablePod(), test.UnschedulablePod()) + nodes := &v1.NodeList{} + Expect(env.Client.List(ctx, nodes)).To(Succeed()) + Expect(len(nodes.Items)).To(Equal(2)) + + // Scheduler should attempt to schedule all the pods to the new node + for _, n := range bindings { + Expect(n.Node.Name).ToNot(Equal(node.Name)) + } }) - It("should schedule pods using taints", func() { - provisioner.Spec.Taints = append(provisioner.Spec.Taints, v1.Taint{ - Key: "only-provisioner", - Value: "true", - Effect: v1.TaintEffectNoSchedule, + Context("Resource Limits", func() { + It("should not schedule when limits are exceeded", func() { + ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("20")}), + }, + Status: v1beta1.NodePoolStatus{ + Resources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100"), + }, + }, + })) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) }) - nodePool.Spec.Template.Spec.Taints = append(nodePool.Spec.Template.Spec.Taints, v1.Taint{ - Key: "only-nodepool", - Value: "true", - Effect: v1.TaintEffectNoSchedule, + It("should schedule if limits would be met", func() { + ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}), + }, + })) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + // requires a 2 CPU node, but leaves room for overhead + v1.ResourceCPU: resource.MustParse("1.75"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + // A 2 CPU node can be launched + ExpectScheduled(ctx, env.Client, pod) }) - ExpectApplied(ctx, env.Client, provisioner, nodePool) + It("should partially schedule if limits would be exceeded", func() { + ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("3")}), + }, + })) - provisionerPod := test.UnschedulablePod(test.PodOptions{ - Tolerations: []v1.Toleration{ - { - Key: "only-provisioner", - Operator: v1.TolerationOpExists, + // prevent these pods from scheduling on the same node + opts := test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "foo"}, }, - }, - }) - nodePoolPod := test.UnschedulablePod(test.PodOptions{ - Tolerations: []v1.Toleration{ - { - Key: "only-nodepool", - Operator: v1.TolerationOpExists, + PodAntiRequirements: []v1.PodAffinityTerm{ + { + TopologyKey: v1.LabelHostname, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + }, }, - }, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1.5"), + }}} + pods := []*v1.Pod{ + test.UnschedulablePod(opts), + test.UnschedulablePod(opts), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + scheduledPodCount := 0 + unscheduledPodCount := 0 + pod0 := ExpectPodExists(ctx, env.Client, pods[0].Name, pods[0].Namespace) + pod1 := ExpectPodExists(ctx, env.Client, pods[1].Name, pods[1].Namespace) + if pod0.Spec.NodeName == "" { + unscheduledPodCount++ + } else { + scheduledPodCount++ + } + if pod1.Spec.NodeName == "" { + unscheduledPodCount++ + } else { + scheduledPodCount++ + } + Expect(scheduledPodCount).To(Equal(1)) + Expect(unscheduledPodCount).To(Equal(1)) }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, provisionerPod, nodePoolPod) - node := ExpectScheduled(ctx, env.Client, provisionerPod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) - node = ExpectScheduled(ctx, env.Client, nodePoolPod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) - }) - It("should order the NodePools and Provisioner by weight", func() { - var provisioners []*v1alpha5.Provisioner - var nodePools []*v1beta1.NodePool - weights := lo.Reject(rand.Perm(101), func(i int, _ int) bool { return i == 0 }) - for i := 0; i < 10; i++ { - p := test.Provisioner(test.ProvisionerOptions{ - Weight: lo.ToPtr[int32](int32(weights[i])), - }) - provisioners = append(provisioners, p) - ExpectApplied(ctx, env.Client, p) - } - for i := 0; i < 10; i++ { - np := test.NodePool(v1beta1.NodePool{ + It("should not schedule if limits would be exceeded", func() { + ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ Spec: v1beta1.NodePoolSpec{ - Weight: lo.ToPtr[int32](int32(weights[i+10])), + Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}), }, - }) - nodePools = append(nodePools, np) - ExpectApplied(ctx, env.Client, np) - } - highestWeightProvisioner := lo.MaxBy(provisioners, func(a, b *v1alpha5.Provisioner) bool { - return lo.FromPtr(a.Spec.Weight) > lo.FromPtr(b.Spec.Weight) + })) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2.1"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) }) - highestWeightNodePool := lo.MaxBy(nodePools, func(a, b *v1beta1.NodePool) bool { - return lo.FromPtr(a.Spec.Weight) > lo.FromPtr(b.Spec.Weight) + It("should not schedule if limits would be exceeded (GPU)", func() { + ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Limits: v1beta1.Limits(v1.ResourceList{v1.ResourcePods: resource.MustParse("1")}), + }, + })) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + fake.ResourceGPUVendorA: resource.MustParse("1"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + // only available instance type has 2 GPUs which would exceed the limit + ExpectNotScheduled(ctx, env.Client, pod) }) + It("should not schedule to a provisioner after a scheduling round if limits would be exceeded", func() { + ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}), + }, + })) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + // requires a 2 CPU node, but leaves room for overhead + v1.ResourceCPU: resource.MustParse("1.75"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + // A 2 CPU node can be launched + ExpectScheduled(ctx, env.Client, pod) - pod := test.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - - if lo.FromPtr(highestWeightProvisioner.Spec.Weight) > lo.FromPtr(highestWeightNodePool.Spec.Weight) { - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, highestWeightProvisioner.Name)) - } else { - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, highestWeightNodePool.Name)) - } + // This pod requests over the existing limit (would add to 3.5 CPUs) so this should fail + pod = test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + // requires a 2 CPU node, but leaves room for overhead + v1.ResourceCPU: resource.MustParse("1.75"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) }) - Context("Limits", func() { - It("should select a NodePool if a Provisioner is over its limit", func() { - provisioner.Spec.Limits = &v1alpha5.Limits{Resources: v1.ResourceList{v1.ResourceCPU: resource.MustParse("0")}} - ExpectApplied(ctx, env.Client, provisioner, nodePool) - - pod := test.UnschedulablePod() + Context("Daemonsets and Node Overhead", func() { + It("should account for overhead", func() { + ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, + }}, + )) + pod := test.UnschedulablePod( + test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, + }, + ) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) + + allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity + Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) + Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) }) - It("should select a Provisioner if a NodePool is over its limit", func() { - nodePool.Spec.Limits = v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("0")}) - ExpectApplied(ctx, env.Client, provisioner, nodePool) + It("should account for overhead (with startup taint)", func() { + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + StartupTaints: []v1.Taint{{Key: "foo.com/taint", Effect: v1.TaintEffectNoSchedule}}, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool, test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, + }}, + )) + pod := test.UnschedulablePod( + test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity + Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) + Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) + }) + It("should not schedule if overhead is too large", func() { + ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}}, + }}, + )) pod := test.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should account for overhead using daemonset pod spec instead of daemonset spec", func() { + nodePool := test.NodePool() + // Create a daemonset with large resource requests + daemonset := test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}}, + }}, + ) + ExpectApplied(ctx, env.Client, nodePool, daemonset) + // Create the actual daemonSet pod with lower resource requests and expect to use the pod + daemonsetPod := test.UnschedulablePod( + test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "DaemonSet", + Name: daemonset.Name, + UID: daemonset.UID, + Controller: ptr.Bool(true), + BlockOwnerDeletion: ptr.Bool(true), + }, + }, + }, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, + }) + ExpectApplied(ctx, env.Client, nodePool, daemonsetPod) + ExpectReconcileSucceeded(ctx, daemonsetController, client.ObjectKeyFromObject(daemonset)) + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, + NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: nodePool.Name}}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) + + // We expect a smaller instance since the daemonset pod is smaller then daemonset spec + allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity + Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) + Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) }) - }) - Context("Deleting", func() { - It("should select a NodePool if a Provisioner is deleting", func() { - ExpectApplied(ctx, env.Client, nodePool, provisioner) - ExpectDeletionTimestampSet(ctx, env.Client, provisioner) + It("should not schedule if resource requests are not defined and limits (requests) are too large", func() { + ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + }}, + )) pod := test.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) + ExpectNotScheduled(ctx, env.Client, pod) }) - It("should select a Provisioner if a NodePool is deleting", func() { - ExpectApplied(ctx, env.Client, nodePool, provisioner) - ExpectDeletionTimestampSet(ctx, env.Client, nodePool) + It("should schedule based on the max resource requests of containers and initContainers", func() { + ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + }, + InitImage: "pause", + InitResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + }}, + )) pod := test.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) + allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity + Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) + Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) }) - }) - Context("Daemonsets", func() { - It("should select a NodePool if Daemonsets that would schedule to Provisioner would exceed capacity", func() { - cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "provisioner-instance-type", - Resources: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("2Gi"), + It("should not schedule if combined max resources are too large for any node", func() { + ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, }, - }), - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "nodepool-instance-type", - Resources: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("4"), - v1.ResourceMemory: resource.MustParse("4Gi"), + InitImage: "pause", + InitResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, }, - }), - } - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceType, - Operator: v1.NodeSelectorOpIn, - Values: []string{"provisioner-instance-type"}, - }, - } - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceType, - Operator: v1.NodeSelectorOpIn, - Values: []string{"nodepool-instance-type"}, - }, - } - daemonSet := test.DaemonSet( - test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi")}}, }}, - ) - ExpectApplied(ctx, env.Client, nodePool, provisioner, daemonSet) - - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }) + )) + pod := test.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) + ExpectNotScheduled(ctx, env.Client, pod) }) - It("should select a Provisioner if Daemonsets that would schedule to NodePool would exceed capacity", func() { - cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{ - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "provisioner-instance-type", - Resources: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("4"), - v1.ResourceMemory: resource.MustParse("4Gi"), + It("should not schedule if initContainer resources are too large", func() { + ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + InitImage: "pause", + InitResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, }, - }), - fake.NewInstanceType(fake.InstanceTypeOptions{ - Name: "nodepool-instance-type", - Resources: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("2Gi"), + }}, + )) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should be able to schedule pods if resource requests and limits are not defined", func() { + ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{}}, + )) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should ignore daemonsets without matching tolerations", func() { + ExpectApplied(ctx, env.Client, + test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoSchedule}}, + }, + }, }, }), - } - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceType, - Operator: v1.NodeSelectorOpIn, - Values: []string{"provisioner-instance-type"}, - }, - } - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - { - Key: v1.LabelInstanceType, - Operator: v1.NodeSelectorOpIn, - Values: []string{"nodepool-instance-type"}, + test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, + }}, + )) + pod := test.UnschedulablePod( + test.PodOptions{ + Tolerations: []v1.Toleration{{Operator: v1.TolerationOperator(v1.NodeSelectorOpExists)}}, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }, - } - daemonSet := test.DaemonSet( + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity + Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("2"))) + Expect(*allocatable.Memory()).To(Equal(resource.MustParse("2Gi"))) + }) + It("should ignore daemonsets with an invalid selector", func() { + ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( test.DaemonSetOptions{PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi")}}, + NodeSelector: map[string]string{"node": "invalid"}, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, + )) + pod := test.UnschedulablePod( + test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, + }, ) - ExpectApplied(ctx, env.Client, nodePool, provisioner, daemonSet) - - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) + allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity + Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("2"))) + Expect(*allocatable.Memory()).To(Equal(resource.MustParse("2Gi"))) }) - It("should select a NodePool if Daemonset select against a Provisioner and would cause the Provisioner to exceed capacity", func() { - provisioner.Spec.Labels = lo.Assign(provisioner.Spec.Labels, map[string]string{"scheduleme": "true"}) - nodePool.Spec.Template.Labels = lo.Assign(nodePool.Spec.Template.Labels, map[string]string{"scheduleme": "false"}) - - daemonSet := test.DaemonSet( + It("should account daemonsets with NotIn operator and unspecified key", func() { + ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( test.DaemonSetOptions{PodOptions: test.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: "scheduleme", - Operator: v1.NodeSelectorOpIn, - Values: []string{"true"}, - }, - }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}}, + NodeRequirements: []v1.NodeSelectorRequirement{{Key: "foo", Operator: v1.NodeSelectorOpNotIn, Values: []string{"bar"}}}, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, + )) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}}, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, + }, ) - ExpectApplied(ctx, env.Client, nodePool, provisioner, daemonSet) - - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) + allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity + Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) + Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) }) - It("should select a Provisioner if Daemonset select against a NodePool and would cause the NodePool to exceed capacity", func() { - provisioner.Spec.Labels = lo.Assign(provisioner.Spec.Labels, map[string]string{"scheduleme": "false"}) - nodePool.Spec.Template.Labels = lo.Assign(nodePool.Spec.Template.Labels, map[string]string{"scheduleme": "true"}) - - daemonSet := test.DaemonSet( + It("should account for daemonset spec affinity", func() { + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + ObjectMeta: v1beta1.ObjectMeta{ + Labels: map[string]string{ + "foo": "voo", + }, + }, + }, + Limits: v1beta1.Limits(v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }), + }, + }) + nodePoolDaemonset := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + ObjectMeta: v1beta1.ObjectMeta{ + Labels: map[string]string{ + "foo": "bar", + }, + }, + }, + }, + }) + // Create a daemonset with large resource requests + daemonset := test.DaemonSet( test.DaemonSetOptions{PodOptions: test.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{ { - Key: "scheduleme", + Key: "foo", Operator: v1.NodeSelectorOpIn, - Values: []string{"true"}, + Values: []string{"bar"}, }, }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}}, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}}, }}, ) - ExpectApplied(ctx, env.Client, nodePool, provisioner, daemonSet) + ExpectApplied(ctx, env.Client, nodePoolDaemonset, daemonset) + // Create the actual daemonSet pod with lower resource requests and expect to use the pod + daemonsetPod := test.UnschedulablePod( + test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "DaemonSet", + Name: daemonset.Name, + UID: daemonset.UID, + Controller: ptr.Bool(true), + BlockOwnerDeletion: ptr.Bool(true), + }, + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: metav1.ObjectNameField, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node-name"}, + }, + }, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}}, + }) + ExpectApplied(ctx, env.Client, daemonsetPod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, daemonsetPod) + ExpectReconcileSucceeded(ctx, daemonsetController, client.ObjectKeyFromObject(daemonset)) + //Deploy pod pod := test.UnschedulablePod(test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, + NodeSelector: map[string]string{ + "foo": "voo", + }, }) + ExpectApplied(ctx, env.Client, nodePool, pod) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) + ExpectScheduled(ctx, env.Client, pod) }) }) - Context("Preferential Fallback", func() { - It("should fallback from a NodePool to a Provisioner when preferences can't be satisfied against the NodePool", func() { - provisioner.Spec.Labels = lo.Assign(provisioner.Spec.Labels, map[string]string{ - "foo": "true", - }) - ExpectApplied(ctx, env.Client, nodePool, provisioner) - - pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: "foo", Operator: v1.NodeSelectorOpIn, Values: []string{"true"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: "bar", Operator: v1.NodeSelectorOpIn, Values: []string{"true"}}, - }}, + Context("Annotations", func() { + It("should annotate nodes", func() { + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + ObjectMeta: v1beta1.ObjectMeta{ + Annotations: map[string]string{v1beta1.DoNotDisruptAnnotationKey: "true"}, + }, + }, }, - }}} - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) - }) - It("should fallback from a Provisioner to a NodePool when preferences can't be satisfied against the Provisioner", func() { - nodePool.Spec.Template.Labels = lo.Assign(nodePool.Spec.Template.Labels, map[string]string{ - "foo": "true", }) - ExpectApplied(ctx, env.Client, nodePool, provisioner) - + ExpectApplied(ctx, env.Client, nodePool) pod := test.UnschedulablePod() - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: "foo", Operator: v1.NodeSelectorOpIn, Values: []string{"true"}}, - }}, - }, - { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: "bar", Operator: v1.NodeSelectorOpIn, Values: []string{"true"}}, - }}, - }, - }}} ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) + Expect(node.Annotations).To(HaveKeyWithValue(v1beta1.DoNotDisruptAnnotationKey, "true")) }) }) - Context("Topology", func() { - var labels map[string]string - BeforeEach(func() { - labels = map[string]string{"test": "test"} - }) - It("should spread across Provisioners and NodePools when considering zonal topology", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - testZone1Provisioner := test.Provisioner(test.ProvisionerOptions{ - Labels: map[string]string{ - v1.LabelTopologyZone: "test-zone-1", - }, - }) - testZone2NodePool := test.NodePool(v1beta1.NodePool{ + Context("Labels", func() { + It("should label nodes", func() { + nodePool := test.NodePool(v1beta1.NodePool{ Spec: v1beta1.NodePoolSpec{ Template: v1beta1.NodeClaimTemplate{ ObjectMeta: v1beta1.ObjectMeta{ - Labels: map[string]string{ - v1.LabelTopologyZone: "test-zone-2", + Labels: map[string]string{"test-key-1": "test-value-1"}, + }, + Spec: v1beta1.NodeClaimSpec{ + Requirements: []v1.NodeSelectorRequirement{ + {Key: "test-key-2", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value-2"}}, + {Key: "test-key-3", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value-3"}}, + {Key: "test-key-4", Operator: v1.NodeSelectorOpLt, Values: []string{"4"}}, + {Key: "test-key-5", Operator: v1.NodeSelectorOpGt, Values: []string{"5"}}, + {Key: "test-key-6", Operator: v1.NodeSelectorOpExists}, + {Key: "test-key-7", Operator: v1.NodeSelectorOpDoesNotExist}, }, }, }, }, }) - testZone3Provisioner := test.Provisioner(test.ProvisionerOptions{ - Labels: map[string]string{ - v1.LabelTopologyZone: "test-zone-3", - }, - }) - ExpectApplied(ctx, env.Client, testZone1Provisioner, testZone2NodePool, testZone3Provisioner) - - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, nodePool.Name)) + Expect(node.Labels).To(HaveKeyWithValue("test-key-1", "test-value-1")) + Expect(node.Labels).To(HaveKeyWithValue("test-key-2", "test-value-2")) + Expect(node.Labels).To(And(HaveKey("test-key-3"), Not(HaveValue(Equal("test-value-3"))))) + Expect(node.Labels).To(And(HaveKey("test-key-4"), Not(HaveValue(Equal("test-value-4"))))) + Expect(node.Labels).To(And(HaveKey("test-key-5"), Not(HaveValue(Equal("test-value-5"))))) + Expect(node.Labels).To(HaveKey("test-key-6")) + Expect(node.Labels).ToNot(HaveKey("test-key-7")) + }) + It("should label nodes with labels in the LabelDomainExceptions list", func() { + for domain := range v1beta1.LabelDomainExceptions { + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + ObjectMeta: v1beta1.ObjectMeta{ + Labels: map[string]string{domain + "/test": "test-value"}, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) + } + }) + + }) + Context("Taints", func() { + It("should schedule pods that tolerate taints", func() { + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + Taints: []v1.Taint{{Key: "nvidia.com/gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool) + pods := []*v1.Pod{ + test.UnschedulablePod( + test.PodOptions{Tolerations: []v1.Toleration{ + { + Key: "nvidia.com/gpu", + Operator: v1.TolerationOpEqual, + Value: "true", + Effect: v1.TaintEffectNoSchedule, + }, + }}), + test.UnschedulablePod( + test.PodOptions{Tolerations: []v1.Toleration{ + { + Key: "nvidia.com/gpu", + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoSchedule, + }, + }}), + test.UnschedulablePod( + test.PodOptions{Tolerations: []v1.Toleration{ + { + Key: "nvidia.com/gpu", + Operator: v1.TolerationOpExists, + }, + }}), + test.UnschedulablePod( + test.PodOptions{Tolerations: []v1.Toleration{ + { + Operator: v1.TolerationOpExists, + }, + }}), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + for _, pod := range pods { + ExpectScheduled(ctx, env.Client, pod) + } + }) + }) + Context("NodeClaim Creation", func() { + It("should create a nodeclaim request with expected requirements", func() { + nodePool := test.NodePool() + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + + Expect(cloudProvider.CreateCalls).To(HaveLen(1)) + ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], + v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: lo.Keys(instanceTypeMap), + }, + v1.NodeSelectorRequirement{ + Key: v1beta1.NodePoolLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{nodePool.Name}, + }, ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) - - // Expect that among the nodes that have skew that we have deployed nodes across Provisioners and NodePools - nodes := ExpectNodes(ctx, env.Client) - _, ok := lo.Find(nodes, func(n *v1.Node) bool { return n.Labels[v1alpha5.ProvisionerNameLabelKey] == testZone1Provisioner.Name }) - Expect(ok).To(BeTrue()) - _, ok = lo.Find(nodes, func(n *v1.Node) bool { return n.Labels[v1beta1.NodePoolLabelKey] == testZone2NodePool.Name }) - Expect(ok).To(BeTrue()) - _, ok = lo.Find(nodes, func(n *v1.Node) bool { return n.Labels[v1alpha5.ProvisionerNameLabelKey] == testZone3Provisioner.Name }) - Expect(ok).To(BeTrue()) - }) - It("should spread across Provisioners and NodePools and respect zonal constraints (subset) with requirements", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - testZone1Provisioner := test.Provisioner(test.ProvisionerOptions{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1"}, + ExpectScheduled(ctx, env.Client, pod) + }) + It("should create a nodeclaim request with additional expected requirements", func() { + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + Requirements: []v1.NodeSelectorRequirement{ + { + Key: "custom-requirement-key", + Operator: v1.NodeSelectorOpIn, + Values: []string{"value"}, + }, + { + Key: "custom-requirement-key2", + Operator: v1.NodeSelectorOpIn, + Values: []string{"value"}, + }, + }, + }, }, }, }) - testZone2NodePool := test.NodePool(v1beta1.NodePool{ + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + + Expect(cloudProvider.CreateCalls).To(HaveLen(1)) + ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], + v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: lo.Keys(instanceTypeMap), + }, + v1.NodeSelectorRequirement{ + Key: v1beta1.NodePoolLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{nodePool.Name}, + }, + v1.NodeSelectorRequirement{ + Key: "custom-requirement-key", + Operator: v1.NodeSelectorOpIn, + Values: []string{"value"}, + }, + v1.NodeSelectorRequirement{ + Key: "custom-requirement-key2", + Operator: v1.NodeSelectorOpIn, + Values: []string{"value"}, + }, + ) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should create a nodeclaim request restricting instance types on architecture", func() { + nodePool := test.NodePool(v1beta1.NodePool{ Spec: v1beta1.NodePoolSpec{ Template: v1beta1.NodeClaimTemplate{ Spec: v1beta1.NodeClaimSpec{ Requirements: []v1.NodeSelectorRequirement{ { - Key: v1.LabelTopologyZone, + Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-2"}, + Values: []string{"arm64"}, }, }, }, }, }, }) - ExpectApplied(ctx, env.Client, testZone1Provisioner, testZone2NodePool) + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., + Expect(cloudProvider.CreateCalls).To(HaveLen(1)) + + // Expect a more restricted set of instance types + ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], + v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"arm64"}, + }, + v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"arm-instance-type"}, + }, ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) - - // Expect that among the nodes that have skew that we have deployed nodes across Provisioners and NodePools - nodes := ExpectNodes(ctx, env.Client) - _, ok := lo.Find(nodes, func(n *v1.Node) bool { return n.Labels[v1alpha5.ProvisionerNameLabelKey] == testZone1Provisioner.Name }) - Expect(ok).To(BeTrue()) - _, ok = lo.Find(nodes, func(n *v1.Node) bool { return n.Labels[v1beta1.NodePoolLabelKey] == testZone2NodePool.Name }) - Expect(ok).To(BeTrue()) - }) - It("should spread across Provisioners and NodePools and respect zonal constraints (subset) with labels", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - - testZone1Provisioner := test.Provisioner(test.ProvisionerOptions{ - Labels: map[string]string{ - v1.LabelTopologyZone: "test-zone-1", + ExpectScheduled(ctx, env.Client, pod) + }) + It("should create a nodeclaim request restricting instance types on operating system", func() { + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"ios"}, + }, + }, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + + Expect(cloudProvider.CreateCalls).To(HaveLen(1)) + + // Expect a more restricted set of instance types + ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], + v1.NodeSelectorRequirement{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"ios"}, + }, + v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"arm-instance-type"}, + }, + ) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should create a nodeclaim request restricting instance types based on pod resource requests", func() { + nodePool := test.NodePool() + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + fake.ResourceGPUVendorA: resource.MustParse("1"), + }, + Limits: v1.ResourceList{ + fake.ResourceGPUVendorA: resource.MustParse("1"), + }, }, }) - testZone2NodePool := test.NodePool(v1beta1.NodePool{ + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + + Expect(cloudProvider.CreateCalls).To(HaveLen(1)) + + // Expect a more restricted set of instance types + ExpectNodeClaimRequirements(cloudProvider.CreateCalls[0], + v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"gpu-vendor-instance-type"}, + }, + ) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should create a nodeclaim request with the correct owner reference", func() { + nodePool := test.NodePool() + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + + Expect(cloudProvider.CreateCalls).To(HaveLen(1)) + Expect(cloudProvider.CreateCalls[0].OwnerReferences).To(ContainElement( + metav1.OwnerReference{ + APIVersion: "karpenter.sh/v1beta1", + Kind: "NodePool", + Name: nodePool.Name, + UID: nodePool.UID, + BlockOwnerDeletion: lo.ToPtr(true), + }, + )) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should create a nodeclaim request propagating the nodeClass reference", func() { + nodePool := test.NodePool(v1beta1.NodePool{ Spec: v1beta1.NodePoolSpec{ Template: v1beta1.NodeClaimTemplate{ - ObjectMeta: v1beta1.ObjectMeta{ - Labels: map[string]string{ - v1.LabelTopologyZone: "test-zone-2", + Spec: v1beta1.NodeClaimSpec{ + NodeClassRef: &v1beta1.NodeClassReference{ + APIVersion: "cloudprovider.karpenter.sh/v1beta1", + Kind: "CloudProvider", + Name: "default", }, }, }, }, }) - ExpectApplied(ctx, env.Client, testZone1Provisioner, testZone2NodePool) + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., + Expect(cloudProvider.CreateCalls).To(HaveLen(1)) + Expect(cloudProvider.CreateCalls[0].Spec.NodeClassRef).To(Equal( + &v1beta1.NodeClassReference{ + APIVersion: "cloudprovider.karpenter.sh/v1beta1", + Kind: "CloudProvider", + Name: "default", + }, + )) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should create a nodeclaim with resource requests", func() { + ExpectApplied(ctx, env.Client, test.NodePool()) + pod := test.UnschedulablePod( + test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Mi"), + fake.ResourceGPUVendorA: resource.MustParse("1"), + }, + Limits: v1.ResourceList{ + fake.ResourceGPUVendorA: resource.MustParse("1"), + }, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + Expect(cloudProvider.CreateCalls).To(HaveLen(1)) + Expect(cloudProvider.CreateCalls[0].Spec.Resources.Requests).To(HaveLen(4)) + ExpectNodeClaimRequests(cloudProvider.CreateCalls[0], v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Mi"), + fake.ResourceGPUVendorA: resource.MustParse("1"), + v1.ResourcePods: resource.MustParse("1"), + }) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should create a nodeclaim with resource requests with daemon overhead", func() { + ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Mi")}}, + }}, + )) + pod := test.UnschedulablePod( + test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Mi")}}, + }, ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) - - // Expect that among the nodes that have skew that we have deployed nodes across Provisioners and NodePools - nodes := ExpectNodes(ctx, env.Client) - _, ok := lo.Find(nodes, func(n *v1.Node) bool { return n.Labels[v1alpha5.ProvisionerNameLabelKey] == testZone1Provisioner.Name }) - Expect(ok).To(BeTrue()) - _, ok = lo.Find(nodes, func(n *v1.Node) bool { return n.Labels[v1beta1.NodePoolLabelKey] == testZone2NodePool.Name }) - Expect(ok).To(BeTrue()) - }) - It("should spread across Provisioners and NodePools when considering capacity type", func() { - topology := []v1.TopologySpreadConstraint{{ - TopologyKey: v1beta1.CapacityTypeLabelKey, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, - MaxSkew: 1, - }} - onDemandProvisioner := test.Provisioner(test.ProvisionerOptions{ - Requirements: []v1.NodeSelectorRequirement{ + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + Expect(cloudProvider.CreateCalls).To(HaveLen(1)) + ExpectNodeClaimRequests(cloudProvider.CreateCalls[0], v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Mi"), + v1.ResourcePods: resource.MustParse("2"), + }) + ExpectScheduled(ctx, env.Client, pod) + }) + }) + Context("Volume Topology Requirements", func() { + var storageClass *storagev1.StorageClass + BeforeEach(func() { + storageClass = test.StorageClass(test.StorageClassOptions{Zones: []string{"test-zone-2", "test-zone-3"}}) + }) + It("should not schedule if invalid pvc", func() { + ExpectApplied(ctx, env.Client, test.NodePool()) + pod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{"invalid"}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule with an empty storage class", func() { + storageClass := "" + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass}) + ExpectApplied(ctx, env.Client, test.NodePool(), persistentVolumeClaim) + pod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should schedule valid pods when a pod with an invalid pvc is encountered (pvc)", func() { + ExpectApplied(ctx, env.Client, test.NodePool()) + invalidPod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{"invalid"}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, invalidPod) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, invalidPod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should schedule valid pods when a pod with an invalid pvc is encountered (storage class)", func() { + invalidStorageClass := "invalid-storage-class" + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &invalidStorageClass}) + ExpectApplied(ctx, env.Client, test.NodePool(), persistentVolumeClaim) + invalidPod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, invalidPod) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, invalidPod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should schedule valid pods when a pod with an invalid pvc is encountered (volume name)", func() { + invalidVolumeName := "invalid-volume-name" + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: invalidVolumeName}) + ExpectApplied(ctx, env.Client, test.NodePool(), persistentVolumeClaim) + invalidPod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, invalidPod) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, invalidPod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should schedule to storage class zones if volume does not exist", func() { + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass.Name}) + ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim) + pod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}, + }}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should schedule to storage class zones if volume does not exist (ephemeral volume)", func() { + pod := test.UnschedulablePod(test.PodOptions{ + EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ { - Key: v1alpha5.LabelCapacityType, + StorageClassName: &storageClass.Name, + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}, + }}, + }) + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), + }, + StorageClassName: &storageClass.Name, + }) + ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should not schedule if storage class zones are incompatible", func() { + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass.Name}) + ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim) + pod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, + }}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule if storage class zones are incompatible (ephemeral volume)", func() { + pod := test.UnschedulablePod(test.PodOptions{ + EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ + { + StorageClassName: &storageClass.Name, + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, + }}, + }) + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), + }, + StorageClassName: &storageClass.Name, + }) + ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule to volume zones if volume already bound", func() { + persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) + ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim, persistentVolume) + pod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should schedule to volume zones if volume already bound (ephemeral volume)", func() { + pod := test.UnschedulablePod(test.PodOptions{ + EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ + { + StorageClassName: &storageClass.Name, + }, + }, + }) + persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), + }, + VolumeName: persistentVolume.Name, + StorageClassName: &storageClass.Name, + }) + ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, pod, persistentVolumeClaim, persistentVolume) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should not schedule if volume zones are incompatible", func() { + persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) + ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim, persistentVolume) + pod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, + }}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule if volume zones are incompatible (ephemeral volume)", func() { + pod := test.UnschedulablePod(test.PodOptions{ + EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{ + { + StorageClassName: &storageClass.Name, + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, + }}, + }) + persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", pod.Name, pod.Spec.Volumes[0].Name), + }, + VolumeName: persistentVolume.Name, + StorageClassName: &storageClass.Name, + }) + ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, pod, persistentVolumeClaim, persistentVolume) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not relax an added volume topology zone node-selector away", func() { + persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) + persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) + ExpectApplied(ctx, env.Client, test.NodePool(), storageClass, persistentVolumeClaim, persistentVolume) + + pod := test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: "example.com/label", Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeOnDemand}, + Values: []string{"unsupported"}, + }, + }, + }) + + // Add the second capacity type that is OR'd with the first. Previously we only added the volume topology requirement + // to a single node selector term which would sometimes get relaxed away. Now we add it to all of them to AND + // it with each existing term. + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, + v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: v1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1beta1.CapacityTypeOnDemand}, + }, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + }) + Context("Preferential Fallback", func() { + Context("Required", func() { + It("should not relax the final term", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, // Should not be relaxed + }}, + }}}} + // Don't relax + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}}, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should relax multiple terms", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, + }}, + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, // OR operator, never get to this one + }}, + }}}} + // Success + ExpectApplied(ctx, env.Client, test.NodePool()) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1")) + }) + }) + Context("Preferences", func() { + It("should relax all node affinity terms", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + }, + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + }, + }}} + // Success + ExpectApplied(ctx, env.Client, test.NodePool()) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should relax to use lighter weights", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 100, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, + }}, + }, + { + Weight: 50, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, + }}, + }, + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ // OR operator, never get to this one + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, + }}, + }, + }}} + // Success + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}}, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should tolerate PreferNoSchedule taint only after trying to relax Affinity terms", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + }, + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + }, + }}} + // Success + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}}, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Spec.Taints).To(ContainElement(v1.Taint{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule})) + }) + }) + }) + Context("Multiple NodePools", func() { + It("should schedule to an explicitly selected NodePool", func() { + nodePool := test.NodePool() + ExpectApplied(ctx, env.Client, nodePool, test.NodePool()) + pod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: nodePool.Name}}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1beta1.NodePoolLabelKey]).To(Equal(nodePool.Name)) + }) + It("should schedule to a NodePool by labels", func() { + nodePool := test.NodePool(v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + ObjectMeta: v1beta1.ObjectMeta{ + Labels: map[string]string{"foo": "bar"}, + }, }, }, }) - spotNodePool := test.NodePool(v1beta1.NodePool{ + ExpectApplied(ctx, env.Client, nodePool, test.NodePool()) + pod := test.UnschedulablePod(test.PodOptions{NodeSelector: nodePool.Spec.Template.Labels}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1beta1.NodePoolLabelKey]).To(Equal(nodePool.Name)) + }) + It("should not match NodePool with PreferNoSchedule taint when other NodePool match", func() { + nodePool := test.NodePool(v1beta1.NodePool{ Spec: v1beta1.NodePoolSpec{ Template: v1beta1.NodeClaimTemplate{ Spec: v1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1beta1.CapacityTypeSpot}, - }, - }, + Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}}, }, }, }, }) - ExpectApplied(ctx, env.Client, onDemandProvisioner, spotNodePool) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, - test.UnschedulablePods(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}, 4)..., - ) - ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) - - // Expect that among the nodes that have skew that we have deployed nodes across Provisioners and NodePools - nodes := ExpectNodes(ctx, env.Client) - _, ok := lo.Find(nodes, func(n *v1.Node) bool { return n.Labels[v1alpha5.ProvisionerNameLabelKey] == onDemandProvisioner.Name }) - Expect(ok).To(BeTrue()) - _, ok = lo.Find(nodes, func(n *v1.Node) bool { return n.Labels[v1beta1.NodePoolLabelKey] == spotNodePool.Name }) - Expect(ok).To(BeTrue()) + ExpectApplied(ctx, env.Client, nodePool, test.NodePool()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1beta1.NodePoolLabelKey]).ToNot(Equal(nodePool.Name)) + }) + Context("Weighted nodePools", func() { + It("should schedule to the provisioner with the highest priority always", func() { + nodePools := []client.Object{ + test.NodePool(), + test.NodePool(v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Weight: ptr.Int32(20)}}), + test.NodePool(v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Weight: ptr.Int32(100)}}), + } + ExpectApplied(ctx, env.Client, nodePools...) + pods := []*v1.Pod{ + test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod(), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1beta1.NodePoolLabelKey]).To(Equal(nodePools[2].GetName())) + } + }) + It("should schedule to explicitly selected provisioner even if other nodePools are higher priority", func() { + targetedNodePool := test.NodePool() + nodePools := []client.Object{ + targetedNodePool, + test.NodePool(v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Weight: ptr.Int32(20)}}), + test.NodePool(v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Weight: ptr.Int32(100)}}), + } + ExpectApplied(ctx, env.Client, nodePools...) + pod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: targetedNodePool.Name}}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1beta1.NodePoolLabelKey]).To(Equal(targetedNodePool.Name)) + }) }) }) })