test(api): add e2e affinity and toleration test case (#519)

Signed-off-by: Roman Sysoev <[email protected]>

tests/e2e/affinity_toleration_test.go

@@ -0,0 +1,145 @@
+/*
+Copyright 2024 Flant JSC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+
+	virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2"
+	"github.com/deckhouse/virtualization/tests/e2e/ginkgoutil"
+	kc "github.com/deckhouse/virtualization/tests/e2e/kubectl"
+)
+
+var _ = Describe("Virtual machine affinity and toleration", ginkgoutil.CommonE2ETestDecorators(), func() {
+	var (
+		testCaseLabel = map[string]string{"testcase": "affinity-toleration"}
+		vmA           = map[string]string{"vm": "vm-a"}
+		vmB           = map[string]string{"vm": "vm-b"}
+		vmC           = map[string]string{"vm": "vm-c"}
+		vmD           = map[string]string{"vm": "vm-d"}
+	)
+
+	Context("When virtualization resources are applied:", func() {
+		It("result should be succeeded", func() {
+			res := kubectl.Apply(kc.ApplyOptions{
+				Filename:       []string{conf.TestData.AffinityToleration},
+				FilenameOption: kc.Kustomize,
+			})
+			Expect(res.Error()).NotTo(HaveOccurred(), res.StdErr())
+		})
+	})
+
+	Context("When virtual images are applied:", func() {
+		It("checks VIs phases", func() {
+			By(fmt.Sprintf("VIs should be in %s phases", PhaseReady))
+			WaitPhaseByLabel(kc.ResourceVI, PhaseReady, kc.WaitOptions{
+				Labels:    testCaseLabel,
+				Namespace: conf.Namespace,
+				Timeout:   MaxWaitTimeout,
+			})
+		})
+	})
+
+	Context("When virtual machine classes are applied:", func() {
+		It("checks VMClasses phases", func() {
+			By(fmt.Sprintf("VMClasses should be in %s phases", PhaseReady))
+			WaitPhaseByLabel(kc.ResourceVMClass, PhaseReady, kc.WaitOptions{
+				Labels:    testCaseLabel,
+				Namespace: conf.Namespace,
+				Timeout:   MaxWaitTimeout,
+			})
+		})
+	})
+
+	Context("When virtual disks are applied:", func() {
+		It("checks VDs phases", func() {
+			By(fmt.Sprintf("VDs should be in %s phases", PhaseReady))
+			WaitPhaseByLabel(kc.ResourceVD, PhaseReady, kc.WaitOptions{
+				Labels:    testCaseLabel,
+				Namespace: conf.Namespace,
+				Timeout:   MaxWaitTimeout,
+			})
+		})
+	})
+
+	Context("When virtual machines are applied:", func() {
+		It("checks VMs phases", func() {
+			By(fmt.Sprintf("VMs should be in %s phases", PhaseRunning))
+			WaitPhaseByLabel(kc.ResourceVM, PhaseRunning, kc.WaitOptions{
+				Labels:    testCaseLabel,
+				Namespace: conf.Namespace,
+				Timeout:   MaxWaitTimeout,
+			})
+		})
+	})
+
+	Context(fmt.Sprintf("When virtual machines in %s phase", PhaseRunning), func() {
+		It("checks VMs `status.nodeName`", func() {
+			vmObjects := virtv2.VirtualMachineList{}
+			err := GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{
+				Labels:    vmA,
+				Namespace: conf.Namespace,
+			})
+			Expect(err).NotTo(HaveOccurred(), "error: cannot get virtual machines with label %s\nstderr: %s", vmA, err)
+			vmANodeName := vmObjects.Items[0].Status.Node
+			err = GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{
+				Labels:    vmC,
+				Namespace: conf.Namespace,
+			})
+			Expect(err).NotTo(HaveOccurred(), "error: cannot get virtual machines with label %s\nstderr: %s", vmC, err)
+			vmCNodeName := vmObjects.Items[0].Status.Node
+			By("Affinity: `vm-a` and `vm-c` should be running on the same node")
+			Expect(vmANodeName).Should(Equal(vmCNodeName), "error: vm-a and vm-c should be running on the same node")
+			err = GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{
+				Labels:    vmB,
+				Namespace: conf.Namespace,
+			})
+			Expect(err).NotTo(HaveOccurred(), "error: cannot get virtual machines with label %s\nstderr: %s", vmB, err)
+			vmBNodeName := vmObjects.Items[0].Status.Node
+			By("AntiAffinity: `vm-a` and `vm-b` should be running on different nodes")
+			Expect(vmANodeName).ShouldNot(Equal(vmBNodeName), "error: vm-a and vm-b should be running on different nodes")
+			err = GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{
+				Labels:    vmD,
+				Namespace: conf.Namespace,
+			})
+			Expect(err).NotTo(HaveOccurred(), "error: cannot get virtual machines with label %s\nstderr: %s", vmD, err)
+			vmDNodeName := vmObjects.Items[0].Status.Node
+			nodeObj := v1.Node{}
+			err = GetObject(kc.ResourceNode, vmDNodeName, &nodeObj, kc.GetOptions{})
+			Expect(err).NotTo(HaveOccurred(), "error: cannot get node %s:\nstderr: %s", vmDNodeName, err)
+			By("Toleration: `vm-d` should be running on a master node")
+			Expect(nodeObj.Labels).Should(HaveKeyWithValue("node.deckhouse.io/group", "master"))
+		})
+	})
+
+	Context("When test is complited:", func() {
+		It("tries to delete used resources", func() {
+			kustimizationFile := fmt.Sprintf("%s/%s", conf.TestData.AffinityToleration, "kustomization.yaml")
+			err := kustomize.ExcludeResource(kustimizationFile, "ns.yaml")
+			Expect(err).NotTo(HaveOccurred(), "cannot exclude namespace from clean up operation:\n%s", err)
+			res := kubectl.Delete(kc.DeleteOptions{
+				Filename:       []string{conf.TestData.AffinityToleration},
+				FilenameOption: kc.Kustomize,
+			})
+			Expect(res.Error()).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr())
+		})
+	})
+})

tests/e2e/config/config.go

@@ -128,16 +128,17 @@ type Config struct {
 }
 
 type TestData struct {
-	ComplexTest       string `yaml:"complexTest"`
-	Connectivity      string `yaml:"connectivity"`
-	DiskResizing      string `yaml:"diskResizing"`
-	SizingPolicy      string `yaml:"sizingPolicy"`
-	VmConfiguration   string `yaml:"vmConfiguration"`
-	VmLabelAnnotation string `yaml:"vmLabelAnnotation"`
-	VmMigration       string `yaml:"vmMigration"`
-	VmDiskAttachment  string `yaml:"vmDiskAttachment"`
-	Sshkey            string `yaml:"sshKey"`
-	SshUser           string `yaml:"sshUser"`
+	AffinityToleration string `yaml:"affinityToleration"`
+	ComplexTest        string `yaml:"complexTest"`
+	Connectivity       string `yaml:"connectivity"`
+	DiskResizing       string `yaml:"diskResizing"`
+	SizingPolicy       string `yaml:"sizingPolicy"`
+	VmConfiguration    string `yaml:"vmConfiguration"`
+	VmLabelAnnotation  string `yaml:"vmLabelAnnotation"`
+	VmMigration        string `yaml:"vmMigration"`
+	VmDiskAttachment   string `yaml:"vmDiskAttachment"`
+	Sshkey             string `yaml:"sshKey"`
+	SshUser            string `yaml:"sshUser"`
 }
 
 type StorageClass struct {

tests/e2e/default_config.yaml

@@ -19,6 +19,7 @@ ipam:
 helperImages:
   curlImage: "curlimages/curl"
 testData:
+  affinityToleration: "/tmp/testdata/affinity-toleration"
   complexTest: "/tmp/testdata/complex-test"
   connectivity: "/tmp/testdata/connectivity"
   diskResizing: "/tmp/testdata/disk-resizing"

tests/e2e/testdata/affinity-toleration/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: testcases
+namePrefix: pr-number-or-commit-hash-
+resources:
+  - ns.yaml
+  - vmc.yaml
+  - vi
+  - vm
+configurations:
+  - transformer.yaml
+labels:
+  - includeSelectors: true
+    pairs:
+      id: pr-number-or-commit-hash
+      testcase: affinity-toleration

tests/e2e/testdata/affinity-toleration/ns.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: default

tests/e2e/testdata/affinity-toleration/transformer.yaml

@@ -0,0 +1,52 @@
+namespace:
+  - kind: ClusterVirtualImage
+    path: spec/dataSource/objectRef/namespace
+nameReference:
+  - kind: VirtualImage
+    version: v1alpha2 # optional
+    fieldSpecs:
+      - path: spec/dataSource/objectRef/name
+        kind: ClusterVirtualImage
+      - path: spec/dataSource/objectRef/name
+        kind: VirtualImage
+      - path: spec/dataSource/objectRef/name
+        kind: VirtualDisk
+      - path: spec/blockDeviceRefs/name
+        kind: VirtualMachine
+  - kind: ClusterVirtualImage
+    version: v1alpha2 # optional
+    fieldSpecs:
+      - path: spec/dataSource/objectRef/name
+        kind: ClusterVirtualImage
+      - path: spec/dataSource/objectRef/name
+        kind: VirtualImage
+      - path: spec/dataSource/objectRef/name
+        kind: VirtualDisk
+      - path: spec/blockDeviceRefs/name
+        kind: VirtualMachine
+  - kind: VirtualDisk
+    version: v1alpha2 # optional
+    fieldSpecs:
+      - path: spec/blockDeviceRefs/name
+        kind: VirtualMachine
+      - path: spec/blockDeviceRef/name
+        kind: VirtualMachineBlockDeviceAttachment
+  - kind: Secret
+    fieldSpecs:
+      - path: spec/provisioning/userDataRef/name
+        kind: VirtualMachine
+  - kind: VirtualMachineIPAddress
+    version: v1alpha2
+    fieldSpecs:
+      - path: spec/virtualMachineIPAddressName
+        kind: VirtualMachine
+  - kind: VirtualMachine
+    version: v1alpha2
+    fieldSpecs:
+      - path: spec/virtualMachineName
+        kind: VirtualMachineBlockDeviceAttachment
+  - kind: VirtualMachineClass
+    version: v1alpha2
+    fieldSpecs:
+      - path: spec/virtualMachineClassName
+        kind: VirtualMachine

tests/e2e/testdata/affinity-toleration/vi/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - vi-alpine-http.yaml

tests/e2e/testdata/affinity-toleration/vi/vi-alpine-http.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: virtualization.deckhouse.io/v1alpha2
+kind: VirtualImage
+metadata:
+  name: vi-alpine-http
+spec:
+  storage: ContainerRegistry
+  dataSource:
+    type: HTTP
+    http:
+      url: https://0e773854-6b4e-4e76-a65b-d9d81675451a.selstorage.ru/alpine/alpine-v3-20.qcow2

tests/e2e/testdata/affinity-toleration/vm/base/cfg/cloudinit.yaml

@@ -0,0 +1,51 @@
+#cloud-config
+users:
+  - name: cloud
+    # passwd: cloud
+    passwd: $6$rounds=4096$vln/.aPHBOI7BMYR$bBMkqQvuGs5Gyd/1H5DP4m9HjQSy.kgrxpaGEHwkX7KEFV8BS.HZWPitAtZ2Vd8ZqIZRqmlykRCagTgPejt1i.
+    shell: /bin/bash
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    chpasswd: { expire: False }
+    lock_passwd: false
+    ssh_authorized_keys:
+      # testcases
+      - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxcXHmwaGnJ8scJaEN5RzklBPZpVSic4GdaAsKjQoeA [email protected]
+write_files:
+  - path: /etc/init.d/ping-service
+    permissions: "0755"
+    content: |
+      #!/sbin/openrc-run
+
+      command="/usr/bin/ping-service.sh"
+      pidfile="/var/run/${RC_SVCNAME}.pid"
+
+      depend() {
+        need localmount
+        after bootmisc
+      }
+
+      start() {
+        ebegin "Starting ${RC_SVCNAME}"
+        start-stop-daemon --start --background --exec $command --pidfile $pidfile -- $command_args
+        eend $?
+      }
+
+      stop() {
+        ebegin "Stopping ${RC_SVCNAME}"
+        start-stop-daemon --stop --exec $command --pidfile $pidfile
+        eend $?
+      }
+  - path: /usr/bin/ping-service.sh
+    permissions: "0755"
+    content: |
+      #!/bin/bash
+
+      while true; do
+        ping -W1 -D 1.1.1.1 >> /tmp/ping.log
+      done
+final_message: "\U0001F525\U0001F525\U0001F525 The system is finally up, after $UPTIME seconds \U0001F525\U0001F525\U0001F525"
+runcmd:
+  - "echo \"\U0001F7E1 Starting runcmd at $(date +%H:%M:%S)\""
+  - rc-update add ping-service default
+  - rc-service ping-service start
+  - "echo \"\U0001F7E1 Finished runcmd at $(date +%H:%M:%S)\""

tests/e2e/testdata/affinity-toleration/vm/base/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./vm.yaml
+  - ./vd-root.yaml
+configurations:
+  - transformer.yaml
+generatorOptions:
+  disableNameSuffixHash: true
+secretGenerator:
+  - files:
+      - userData=cfg/cloudinit.yaml
+    name: cloud-init
+    type: provisioning.virtualization.deckhouse.io/cloud-init

tests/e2e/testdata/affinity-toleration/vm/base/transformer.yaml

@@ -0,0 +1,54 @@
+# https://github.com/kubernetes-sigs/kustomize/blob/master/examples/transformerconfigs/README.md#transformer-configurations
+
+namespace:
+  - kind: ClusterVirtualImage
+    path: spec/dataSource/objectRef/namespace
+nameReference:
+  - kind: VirtualImage
+    version: v1alpha2 # optional
+    fieldSpecs:
+      - path: spec/dataSource/objectRef/name
+        kind: ClusterVirtualImage
+      - path: spec/dataSource/objectRef/name
+        kind: VirtualImage
+      - path: spec/dataSource/objectRef/name
+        kind: VirtualDisk
+      - path: spec/blockDeviceRefs/name
+        kind: VirtualMachine
+  - kind: ClusterVirtualImage
+    version: v1alpha2 # optional
+    fieldSpecs:
+      - path: spec/dataSource/objectRef/name
+        kind: ClusterVirtualImage
+      - path: spec/dataSource/objectRef/name
+        kind: VirtualImage
+      - path: spec/dataSource/objectRef/name
+        kind: VirtualDisk
+      - path: spec/blockDeviceRefs/name
+        kind: VirtualMachine
+  - kind: VirtualDisk
+    version: v1alpha2 # optional
+    fieldSpecs:
+      - path: spec/blockDeviceRefs/name
+        kind: VirtualMachine
+      - path: spec/blockDeviceRef/name
+        kind: VirtualMachineBlockDeviceAttachment
+  - kind: Secret
+    fieldSpecs:
+      - path: spec/provisioning/userDataRef/name
+        kind: VirtualMachine
+  - kind: VirtualMachineIPAddress
+    version: v1alpha2
+    fieldSpecs:
+      - path: spec/virtualMachineIPAddressName
+        kind: VirtualMachine
+  - kind: VirtualMachine
+    version: v1alpha2
+    fieldSpecs:
+      - path: spec/virtualMachineName
+        kind: VirtualMachineBlockDeviceAttachment
+  - kind: VirtualMachineClass
+    version: v1alpha2
+    fieldSpecs:
+      - path: spec/virtualMachineClassName
+        kind: VirtualMachine