From 9a44a391594b710f2d7b0fad63d995b5b899307a Mon Sep 17 00:00:00 2001 From: Orel Misan Date: Thu, 25 Jan 2024 11:43:30 +0200 Subject: [PATCH 01/18] vm-customize: Automatically mount hugepages Currently, the mounting of hugepages is performed by the `/usr/bin/dpdk-checkup-boot.sh` script which is executed on every boot. Move it to the VM configuration phase, so it would be done automatically by the operating system. Signed-off-by: Ram Lavi Signed-off-by: Orel Misan (cherry picked from commit e9eabe195bf73e811464a79559d090ca6ef7ddc0) --- vms/traffic-gen/scripts/customize-vm | 6 +++--- vms/vm-under-test/scripts/customize-vm | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vms/traffic-gen/scripts/customize-vm b/vms/traffic-gen/scripts/customize-vm index 069f88c3..f3b8b9a4 100644 --- a/vms/traffic-gen/scripts/customize-vm +++ b/vms/traffic-gen/scripts/customize-vm @@ -46,6 +46,9 @@ install_trex() { # Setup hugepages in cmdline setup_hugepages() { + mkdir -p /mnt/huge + echo "hugetlbfs /mnt/huge hugetlbfs defaults,pagesize=1GB 0 0" >> /etc/fstab + grubby --update-kernel=ALL --args="default_hugepagesz=1GB hugepagesz=1G hugepages=1" } @@ -109,9 +112,6 @@ checkup_tuned_adm_set_marker_full_path=$1 driverctl set-override 0000:06:00.0 vfio-pci driverctl set-override 0000:07:00.0 vfio-pci -mkdir -p /mnt/huge -mount /mnt/huge --source nodev -t hugetlbfs -o pagesize=1GB - if [ ! -f "$checkup_tuned_adm_set_marker_full_path" ]; then echo "isolated_cores=2-7" > /etc/tuned/cpu-partitioning-variables.conf tuned-adm profile cpu-partitioning diff --git a/vms/vm-under-test/scripts/customize-vm b/vms/vm-under-test/scripts/customize-vm index 0814c576..e51ab06e 100644 --- a/vms/vm-under-test/scripts/customize-vm +++ b/vms/vm-under-test/scripts/customize-vm @@ -29,6 +29,9 @@ disable_services() { # Setup hugepages in cmdline setup_hugepages() { + mkdir -p /mnt/huge + echo "hugetlbfs /mnt/huge hugetlbfs defaults,pagesize=1GB 0 0" >> /etc/fstab + grubby --update-kernel=ALL --args="default_hugepagesz=1GB hugepagesz=1G hugepages=1" } @@ -92,9 +95,6 @@ checkup_tuned_adm_set_marker_full_path=$1 driverctl set-override 0000:06:00.0 vfio-pci driverctl set-override 0000:07:00.0 vfio-pci -mkdir -p /mnt/huge -mount /mnt/huge --source nodev -t hugetlbfs -o pagesize=1GB - if [ ! -f "$checkup_tuned_adm_set_marker_full_path" ]; then echo "isolated_cores=2-7" > /etc/tuned/cpu-partitioning-variables.conf tuned-adm profile cpu-partitioning From 5fc7737d095444a12496f356500312c7b74a149e Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Wed, 24 Jan 2024 00:18:43 +0200 Subject: [PATCH 02/18] checkup: Add vmi-under-test configmap This commit adds a new configmap that will be consumed by the vmi-under-test in future commits. The create and delete configmap functions are generalized to accept both configmaps. The unit test is also generalized in order to include any configmap deleted/created. Signed-off-by: Ram Lavi (cherry picked from commit f6686afb418d69f0016f3621d41e73069d16624a) --- pkg/internal/checkup/checkup.go | 73 +++++++++++++++++++--------- pkg/internal/checkup/checkup_test.go | 2 +- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/pkg/internal/checkup/checkup.go b/pkg/internal/checkup/checkup.go index 25cd5603..4ee94458 100644 --- a/pkg/internal/checkup/checkup.go +++ b/pkg/internal/checkup/checkup.go @@ -54,18 +54,20 @@ type testExecutor interface { } type Checkup struct { - client kubeVirtVMIClient - namespace string - params config.Config - vmiUnderTest *kvcorev1.VirtualMachineInstance - trafficGen *kvcorev1.VirtualMachineInstance - trafficGenConfigMap *k8scorev1.ConfigMap - results status.Results - executor testExecutor + client kubeVirtVMIClient + namespace string + params config.Config + vmiUnderTest *kvcorev1.VirtualMachineInstance + trafficGen *kvcorev1.VirtualMachineInstance + trafficGenConfigMap *k8scorev1.ConfigMap + vmiUnderTestConfigMap *k8scorev1.ConfigMap + results status.Results + executor testExecutor } const ( - TrafficGenConfigMapNamePrefix = "dpdk-traffic-gen-config" + TrafficGenConfigMapNamePrefix = "dpdk-traffic-gen-config" + vmiUnderTestConfigMapNamePrefix = "vmi-under-test-config" ) func New(client kubeVirtVMIClient, namespace string, checkupConfig config.Config, executor testExecutor) *Checkup { @@ -73,15 +75,17 @@ func New(client kubeVirtVMIClient, namespace string, checkupConfig config.Config randomSuffix := rand.String(randomStringLen) trafficGenCMName := trafficGenConfigMapName(randomSuffix) + vmiUnderTestCMName := vmiUnderTestConfigMapName(randomSuffix) return &Checkup{ - client: client, - namespace: namespace, - params: checkupConfig, - vmiUnderTest: newVMIUnderTest(vmiUnderTestName(randomSuffix), checkupConfig), - trafficGen: newTrafficGen(trafficGenName(randomSuffix), checkupConfig, trafficGenCMName), - trafficGenConfigMap: newTrafficGenConfigMap(trafficGenCMName, checkupConfig), - executor: executor, + client: client, + namespace: namespace, + params: checkupConfig, + vmiUnderTest: newVMIUnderTest(vmiUnderTestName(randomSuffix), checkupConfig), + vmiUnderTestConfigMap: newVMIUnderTestConfigMap(vmiUnderTestCMName, checkupConfig), + trafficGen: newTrafficGen(trafficGenName(randomSuffix), checkupConfig, trafficGenCMName), + trafficGenConfigMap: newTrafficGenConfigMap(trafficGenCMName, checkupConfig), + executor: executor, } } @@ -93,7 +97,11 @@ func (c *Checkup) Setup(ctx context.Context) (setupErr error) { const errMessagePrefix = "setup" var err error - if err = c.createTrafficGenCM(setupCtx); err != nil { + if err = c.createConfigmap(setupCtx, c.trafficGenConfigMap); err != nil { + return fmt.Errorf("%s: %w", errMessagePrefix, err) + } + + if err = c.createConfigmap(setupCtx, c.vmiUnderTestConfigMap); err != nil { return fmt.Errorf("%s: %w", errMessagePrefix, err) } @@ -178,7 +186,11 @@ func (c *Checkup) Teardown(ctx context.Context) error { teardownErrors = append(teardownErrors, fmt.Sprintf("%s: %v", errMessagePrefix, err)) } - if err := c.deleteTrafficGenCM(ctx); err != nil { + if err := c.deleteConfigmap(ctx, c.trafficGenConfigMap); err != nil { + teardownErrors = append(teardownErrors, fmt.Sprintf("%s: %v", errMessagePrefix, err)) + } + + if err := c.deleteConfigmap(ctx, c.vmiUnderTestConfigMap); err != nil { teardownErrors = append(teardownErrors, fmt.Sprintf("%s: %v", errMessagePrefix, err)) } @@ -201,17 +213,17 @@ func (c *Checkup) Results() status.Results { return c.results } -func (c *Checkup) createTrafficGenCM(ctx context.Context) error { - log.Printf("Creating ConfigMap %q...", ObjectFullName(c.namespace, c.trafficGenConfigMap.Name)) +func (c *Checkup) createConfigmap(ctx context.Context, configMap *k8scorev1.ConfigMap) error { + log.Printf("Creating ConfigMap %q...", ObjectFullName(c.namespace, configMap.Name)) - _, err := c.client.CreateConfigMap(ctx, c.namespace, c.trafficGenConfigMap) + _, err := c.client.CreateConfigMap(ctx, c.namespace, configMap) return err } -func (c *Checkup) deleteTrafficGenCM(ctx context.Context) error { - log.Printf("Deleting ConfigMap %q...", ObjectFullName(c.namespace, c.trafficGenConfigMap.Name)) +func (c *Checkup) deleteConfigmap(ctx context.Context, configMap *k8scorev1.ConfigMap) error { + log.Printf("Deleting ConfigMap %q...", ObjectFullName(c.namespace, configMap.Name)) - return c.client.DeleteConfigMap(ctx, c.namespace, c.trafficGenConfigMap.Name) + return c.client.DeleteConfigMap(ctx, c.namespace, configMap.Name) } func (c *Checkup) createVMI(ctx context.Context, vmiToCreate *kvcorev1.VirtualMachineInstance) error { @@ -304,6 +316,15 @@ func ObjectFullName(namespace, name string) string { return fmt.Sprintf("%s/%s", namespace, name) } +func newVMIUnderTestConfigMap(name string, checkupConfig config.Config) *k8scorev1.ConfigMap { + return configmap.New( + name, + checkupConfig.PodName, + checkupConfig.PodUID, + nil, + ) +} + func newTrafficGenConfigMap(name string, checkupConfig config.Config) *k8scorev1.ConfigMap { trexConfig := trex.NewConfig(checkupConfig) trafficGenConfigData := map[string]string{ @@ -332,3 +353,7 @@ func trafficGenName(suffix string) string { func trafficGenConfigMapName(suffix string) string { return TrafficGenConfigMapNamePrefix + "-" + suffix } + +func vmiUnderTestConfigMapName(suffix string) string { + return vmiUnderTestConfigMapNamePrefix + "-" + suffix +} diff --git a/pkg/internal/checkup/checkup_test.go b/pkg/internal/checkup/checkup_test.go index eb6eb77e..dd2d5983 100644 --- a/pkg/internal/checkup/checkup_test.go +++ b/pkg/internal/checkup/checkup_test.go @@ -200,7 +200,7 @@ func TestTeardownShouldFailWhen(t *testing.T) { }) } -func TestTrafficGenCMTeardownFailure(t *testing.T) { +func TestVMConfigMapTeardownFailure(t *testing.T) { testClient := newClientStub() testConfig := newTestConfig() From 03da6cad75f2d834e0119e83b06545b08d5ec017 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Thu, 25 Jan 2024 11:46:57 +0200 Subject: [PATCH 03/18] checkup/vmi: mount configmap to vmi-under-test Currently the configmap is mounted but has no data. Signed-off-by: Ram Lavi (cherry picked from commit d63b3179ce38132290bec1bf6e3717457154e08f) --- pkg/internal/checkup/checkup.go | 2 +- pkg/internal/checkup/vmi.go | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/pkg/internal/checkup/checkup.go b/pkg/internal/checkup/checkup.go index 4ee94458..616c4842 100644 --- a/pkg/internal/checkup/checkup.go +++ b/pkg/internal/checkup/checkup.go @@ -81,7 +81,7 @@ func New(client kubeVirtVMIClient, namespace string, checkupConfig config.Config client: client, namespace: namespace, params: checkupConfig, - vmiUnderTest: newVMIUnderTest(vmiUnderTestName(randomSuffix), checkupConfig), + vmiUnderTest: newVMIUnderTest(vmiUnderTestName(randomSuffix), checkupConfig, vmiUnderTestCMName), vmiUnderTestConfigMap: newVMIUnderTestConfigMap(vmiUnderTestCMName, checkupConfig), trafficGen: newTrafficGen(trafficGenName(randomSuffix), checkupConfig, trafficGenCMName), trafficGenConfigMap: newTrafficGenConfigMap(trafficGenCMName, checkupConfig), diff --git a/pkg/internal/checkup/vmi.go b/pkg/internal/checkup/vmi.go index 71f68d66..f9dbb8b1 100644 --- a/pkg/internal/checkup/vmi.go +++ b/pkg/internal/checkup/vmi.go @@ -53,7 +53,12 @@ const ( terminationGracePeriodSeconds = 0 ) -func newVMIUnderTest(name string, checkupConfig config.Config) *kvcorev1.VirtualMachineInstance { +func newVMIUnderTest(name string, checkupConfig config.Config, configMapName string) *kvcorev1.VirtualMachineInstance { + const ( + configDiskSerial = "DEADBEEF" + configVolumeName = "vmi-under-test-config" + ) + optionsToApply := baseOptions(checkupConfig) optionsToApply = append(optionsToApply, @@ -61,7 +66,10 @@ func newVMIUnderTest(name string, checkupConfig config.Config) *kvcorev1.Virtual vmi.WithSRIOVInterface(eastNetworkName, checkupConfig.VMUnderTestEastMacAddress.String(), config.VMIEastNICPCIAddress), vmi.WithSRIOVInterface(westNetworkName, checkupConfig.VMUnderTestWestMacAddress.String(), config.VMIWestNICPCIAddress), vmi.WithContainerDisk(rootDiskName, checkupConfig.VMUnderTestContainerDiskImage), - vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, CloudInit(config.VMIUsername, config.VMIPassword, nil)), + vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, + CloudInit(config.VMIUsername, config.VMIPassword, vmiUnderTestBootCommands(configDiskSerial))), + vmi.WithConfigMapVolume(configVolumeName, configMapName), + vmi.WithConfigMapDisk(configVolumeName, configDiskSerial), ) return vmi.New(name, optionsToApply...) @@ -156,3 +164,12 @@ func trafficGenBootCommands(configDiskSerial string) []string { fmt.Sprintf("cp %s/*.py %s", configMountDirectory, trex.StreamsPyPath), } } + +func vmiUnderTestBootCommands(configDiskSerial string) []string { + const configMountDirectory = "/mnt/app-config" + + return []string{ + fmt.Sprintf("mkdir %s", configMountDirectory), + fmt.Sprintf("mount /dev/$(lsblk --nodeps -no name,serial | grep %s | cut -f1 -d' ') %s", configDiskSerial, configMountDirectory), + } +} From 96272925d505367563a36125903732c5578e8fdc Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Thu, 18 Jan 2024 12:49:06 +0200 Subject: [PATCH 04/18] vms/customize-vm: Remove unneeded variable passing There is no need to pass the tuned-adm-set-marker file to the function. Removing it from the service and hard-coding it to the service script on both VM image scripts. Signed-off-by: Ram Lavi (cherry picked from commit 5633c4d62952d3e1f7c840f175166fe002b56158) --- vms/traffic-gen/scripts/customize-vm | 5 ++--- vms/vm-under-test/scripts/customize-vm | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/vms/traffic-gen/scripts/customize-vm b/vms/traffic-gen/scripts/customize-vm index f3b8b9a4..f1d7dc06 100644 --- a/vms/traffic-gen/scripts/customize-vm +++ b/vms/traffic-gen/scripts/customize-vm @@ -62,7 +62,6 @@ setup_boot_service() { local service_name="dpdk-checkup-boot.service" local checkup_boot_script_full_path="/usr/bin/dpdk-checkup-boot.sh" local checkup_boot_service_full_path="/usr/lib/systemd/system/$service_name" - local checkup_tuned_adm_set_marker_full_path="/var/dpdk-checkup-tuned-adm-set-marker" setup_checkup_boot_script "$checkup_boot_script_full_path" @@ -71,7 +70,7 @@ setup_boot_service() { Description=Checkup Boot Script Before=qemu-guest-agent.service [Service] -ExecStart=$checkup_boot_script_full_path $checkup_tuned_adm_set_marker_full_path +ExecStart=$checkup_boot_script_full_path Restart=no User=root Group=root @@ -107,7 +106,7 @@ cat <<'EOF' > "$checkup_boot_script_full_path" set -e -checkup_tuned_adm_set_marker_full_path=$1 +checkup_tuned_adm_set_marker_full_path="/var/dpdk-checkup-tuned-adm-set-marker" driverctl set-override 0000:06:00.0 vfio-pci driverctl set-override 0000:07:00.0 vfio-pci diff --git a/vms/vm-under-test/scripts/customize-vm b/vms/vm-under-test/scripts/customize-vm index e51ab06e..3126b5ed 100644 --- a/vms/vm-under-test/scripts/customize-vm +++ b/vms/vm-under-test/scripts/customize-vm @@ -45,7 +45,6 @@ setup_boot_service() { local service_name="dpdk-checkup-boot.service" local checkup_boot_script_full_path="/usr/bin/dpdk-checkup-boot.sh" local checkup_boot_service_full_path="/usr/lib/systemd/system/$service_name" - local checkup_tuned_adm_set_marker_full_path="/var/dpdk-checkup-tuned-adm-set-marker" setup_checkup_boot_script "$checkup_boot_script_full_path" @@ -54,7 +53,7 @@ setup_boot_service() { Description=Checkup Boot Script Before=qemu-guest-agent.service [Service] -ExecStart=$checkup_boot_script_full_path $checkup_tuned_adm_set_marker_full_path +ExecStart=$checkup_boot_script_full_path Restart=no User=root Group=root @@ -90,7 +89,7 @@ cat <<'EOF' > "$checkup_boot_script_full_path" set -e -checkup_tuned_adm_set_marker_full_path=$1 +checkup_tuned_adm_set_marker_full_path="/var/dpdk-checkup-tuned-adm-set-marker" driverctl set-override 0000:06:00.0 vfio-pci driverctl set-override 0000:07:00.0 vfio-pci From 8d39edf4914e2a9e42e3bf7b55166c3fca8e28a8 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Thu, 25 Jan 2024 12:37:52 +0200 Subject: [PATCH 05/18] checkup: Add boot script to VMs' configmaps Currently the VM guests are configured on a service added on the customize-vm script created in the image building process. There is no real need for the script to run on that context, and it is easier to manage if it runs as part of the cloud-init context. This commits: - copies the script to a checkup function. - adds the boot script to the traffic-gen and vmi-under-test VMs - mounts the script with the correct permissions. Signed-off-by: Ram Lavi (cherry picked from commit 9dfcf78b9289771996c982c9220fc14d9f73a5b7) --- pkg/internal/checkup/checkup.go | 7 ++++++- pkg/internal/checkup/vmi.go | 25 +++++++++++++++++++++++++ pkg/internal/config/config.go | 4 ++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pkg/internal/checkup/checkup.go b/pkg/internal/checkup/checkup.go index 616c4842..0f119117 100644 --- a/pkg/internal/checkup/checkup.go +++ b/pkg/internal/checkup/checkup.go @@ -317,11 +317,15 @@ func ObjectFullName(namespace, name string) string { } func newVMIUnderTestConfigMap(name string, checkupConfig config.Config) *k8scorev1.ConfigMap { + vmiUnderTestConfigData := map[string]string{ + config.BootScriptName: generateBootScript(), + } + return configmap.New( name, checkupConfig.PodName, checkupConfig.PodUID, - nil, + vmiUnderTestConfigData, ) } @@ -333,6 +337,7 @@ func newTrafficGenConfigMap(name string, checkupConfig config.Config) *k8scorev1 trex.CfgFileName: trexConfig.GenerateCfgFile(), trex.StreamPyFileName: trexConfig.GenerateStreamPyFile(), trex.StreamPeerParamsPyFileName: trexConfig.GenerateStreamAddrPyFile(), + config.BootScriptName: generateBootScript(), } return configmap.New( name, diff --git a/pkg/internal/checkup/vmi.go b/pkg/internal/checkup/vmi.go index f9dbb8b1..ec20c446 100644 --- a/pkg/internal/checkup/vmi.go +++ b/pkg/internal/checkup/vmi.go @@ -131,6 +131,27 @@ func Affinity(nodeName, ownerUID string) *k8scorev1.Affinity { return &affinity } +func generateBootScript() string { + const isolatedCores = "2-7" + sb := strings.Builder{} + + sb.WriteString("#!/bin/bash\n") + sb.WriteString("set -e\n") + sb.WriteString("\n") + sb.WriteString("checkup_tuned_adm_set_marker_full_path=" + config.BootScriptTunedAdmSetMarkerFileFullPath + "\n") + sb.WriteString("driverctl set-override " + config.VMIEastNICPCIAddress + " vfio-pci\n") + sb.WriteString("driverctl set-override " + config.VMIWestNICPCIAddress + " vfio-pci\n") + sb.WriteString("\n") + sb.WriteString("if [ ! -f \"$checkup_tuned_adm_set_marker_full_path\" ]; then\n") + sb.WriteString(" echo \"isolated_cores=" + isolatedCores + "\" > /etc/tuned/cpu-partitioning-variables.conf\n") + sb.WriteString(" tuned-adm profile cpu-partitioning\n\n") + sb.WriteString(" touch $checkup_tuned_adm_set_marker_full_path\n") + sb.WriteString(" reboot\n") + sb.WriteString("fi\n") + + return sb.String() +} + func CloudInit(username, password string, bootCommands []string) string { sb := strings.Builder{} sb.WriteString("#cloud-config\n") @@ -162,6 +183,8 @@ func trafficGenBootCommands(configDiskSerial string) []string { fmt.Sprintf("cp %s /etc", path.Join(configMountDirectory, trex.CfgFileName)), fmt.Sprintf("mkdir -p %s", trex.StreamsPyPath), fmt.Sprintf("cp %s/*.py %s", configMountDirectory, trex.StreamsPyPath), + fmt.Sprintf("cp %s %s", path.Join(configMountDirectory, config.BootScriptName), config.BootScriptBinDirectory), + fmt.Sprintf("chmod 744 %s", path.Join(config.BootScriptBinDirectory, config.BootScriptName)), } } @@ -171,5 +194,7 @@ func vmiUnderTestBootCommands(configDiskSerial string) []string { return []string{ fmt.Sprintf("mkdir %s", configMountDirectory), fmt.Sprintf("mount /dev/$(lsblk --nodeps -no name,serial | grep %s | cut -f1 -d' ') %s", configDiskSerial, configMountDirectory), + fmt.Sprintf("cp %s %s", path.Join(configMountDirectory, config.BootScriptName), config.BootScriptBinDirectory), + fmt.Sprintf("chmod 744 %s", path.Join(config.BootScriptBinDirectory, config.BootScriptName)), } } diff --git a/pkg/internal/config/config.go b/pkg/internal/config/config.go index 1c48743e..90a8e0b2 100644 --- a/pkg/internal/config/config.go +++ b/pkg/internal/config/config.go @@ -62,6 +62,10 @@ const ( VMIEastNICPCIAddress = "0000:06:00.0" VMIWestNICPCIAddress = "0000:07:00.0" + + BootScriptName = "dpdk-checkup-boot.sh" + BootScriptBinDirectory = "/usr/bin/" + BootScriptTunedAdmSetMarkerFileFullPath = "/var/dpdk-checkup-tuned-adm-set-marker" ) var ( From 6f4264ec0cee751f3e84fe75aa0baf48bd163f26 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Thu, 25 Jan 2024 12:54:12 +0200 Subject: [PATCH 06/18] Replace customize-vm service with boot script Currently the VM guests are configured on a service added on the customize-vm script created in the image building process. There is no real need for the script to run on that context, and it is easier to manage if it runs as part of the cloud-init context. This commits: - Runs the boot script on the cloud-init context. - Removes it from the images customize-vm scripts. Signed-off-by: Ram Lavi (cherry picked from commit a5d4f68d311c40dcd2ae3a14b76e54e28080d895) --- pkg/internal/checkup/vmi.go | 2 + vms/traffic-gen/scripts/customize-vm | 67 -------------------------- vms/vm-under-test/scripts/customize-vm | 67 -------------------------- 3 files changed, 2 insertions(+), 134 deletions(-) diff --git a/pkg/internal/checkup/vmi.go b/pkg/internal/checkup/vmi.go index ec20c446..4c059a56 100644 --- a/pkg/internal/checkup/vmi.go +++ b/pkg/internal/checkup/vmi.go @@ -185,6 +185,7 @@ func trafficGenBootCommands(configDiskSerial string) []string { fmt.Sprintf("cp %s/*.py %s", configMountDirectory, trex.StreamsPyPath), fmt.Sprintf("cp %s %s", path.Join(configMountDirectory, config.BootScriptName), config.BootScriptBinDirectory), fmt.Sprintf("chmod 744 %s", path.Join(config.BootScriptBinDirectory, config.BootScriptName)), + path.Join(config.BootScriptBinDirectory, config.BootScriptName), } } @@ -196,5 +197,6 @@ func vmiUnderTestBootCommands(configDiskSerial string) []string { fmt.Sprintf("mount /dev/$(lsblk --nodeps -no name,serial | grep %s | cut -f1 -d' ') %s", configDiskSerial, configMountDirectory), fmt.Sprintf("cp %s %s", path.Join(configMountDirectory, config.BootScriptName), config.BootScriptBinDirectory), fmt.Sprintf("chmod 744 %s", path.Join(config.BootScriptBinDirectory, config.BootScriptName)), + path.Join(config.BootScriptBinDirectory, config.BootScriptName), } } diff --git a/vms/traffic-gen/scripts/customize-vm b/vms/traffic-gen/scripts/customize-vm index f1d7dc06..d0914c36 100644 --- a/vms/traffic-gen/scripts/customize-vm +++ b/vms/traffic-gen/scripts/customize-vm @@ -57,74 +57,7 @@ set_unsafe_no_io_mmu_mode() { echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf } -# Create and enable the boot checkup service -setup_boot_service() { - local service_name="dpdk-checkup-boot.service" - local checkup_boot_script_full_path="/usr/bin/dpdk-checkup-boot.sh" - local checkup_boot_service_full_path="/usr/lib/systemd/system/$service_name" - - setup_checkup_boot_script "$checkup_boot_script_full_path" - - cat < "$checkup_boot_service_full_path" -[Unit] -Description=Checkup Boot Script -Before=qemu-guest-agent.service -[Service] -ExecStart=$checkup_boot_script_full_path -Restart=no -User=root -Group=root -[Install] -WantedBy=multi-user.target -EOF - - systemctl enable "$checkup_boot_service_full_path" - systemctl start "$checkup_boot_service_full_path" -} - -setup_checkup_boot_script() { - local checkup_boot_script_full_path=$1 -cat <<'EOF' > "$checkup_boot_script_full_path" -#!/usr/bin/env bash -# -# This file is part of the kiagnose project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2024 Red Hat, Inc. -# - -set -e - -checkup_tuned_adm_set_marker_full_path="/var/dpdk-checkup-tuned-adm-set-marker" - -driverctl set-override 0000:06:00.0 vfio-pci -driverctl set-override 0000:07:00.0 vfio-pci - -if [ ! -f "$checkup_tuned_adm_set_marker_full_path" ]; then - echo "isolated_cores=2-7" > /etc/tuned/cpu-partitioning-variables.conf - tuned-adm profile cpu-partitioning - - touch $checkup_tuned_adm_set_marker_full_path - reboot -fi -EOF - - chmod +x "$checkup_boot_script_full_path" -} - disable_services setup_hugepages install_trex set_unsafe_no_io_mmu_mode -setup_boot_service diff --git a/vms/vm-under-test/scripts/customize-vm b/vms/vm-under-test/scripts/customize-vm index 3126b5ed..15ea165b 100644 --- a/vms/vm-under-test/scripts/customize-vm +++ b/vms/vm-under-test/scripts/customize-vm @@ -40,73 +40,6 @@ set_unsafe_no_io_mmu_mode() { echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf } -# Create and enable the boot checkup service -setup_boot_service() { - local service_name="dpdk-checkup-boot.service" - local checkup_boot_script_full_path="/usr/bin/dpdk-checkup-boot.sh" - local checkup_boot_service_full_path="/usr/lib/systemd/system/$service_name" - - setup_checkup_boot_script "$checkup_boot_script_full_path" - - cat < "$checkup_boot_service_full_path" -[Unit] -Description=Checkup Boot Script -Before=qemu-guest-agent.service -[Service] -ExecStart=$checkup_boot_script_full_path -Restart=no -User=root -Group=root -[Install] -WantedBy=multi-user.target -EOF - - systemctl enable "$checkup_boot_service_full_path" - systemctl start "$checkup_boot_service_full_path" -} - -setup_checkup_boot_script() { - local checkup_boot_script_full_path=$1 -cat <<'EOF' > "$checkup_boot_script_full_path" -#!/usr/bin/env bash -# -# This file is part of the kiagnose project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2024 Red Hat, Inc. -# - -set -e - -checkup_tuned_adm_set_marker_full_path="/var/dpdk-checkup-tuned-adm-set-marker" - -driverctl set-override 0000:06:00.0 vfio-pci -driverctl set-override 0000:07:00.0 vfio-pci - -if [ ! -f "$checkup_tuned_adm_set_marker_full_path" ]; then - echo "isolated_cores=2-7" > /etc/tuned/cpu-partitioning-variables.conf - tuned-adm profile cpu-partitioning - - touch $checkup_tuned_adm_set_marker_full_path - reboot -fi -EOF - - chmod +x "$checkup_boot_script_full_path" -} - disable_services setup_hugepages set_unsafe_no_io_mmu_mode -setup_boot_service From ab6b5249ed804299a8a8006f3d2f77a87d2fee34 Mon Sep 17 00:00:00 2001 From: Orel Misan Date: Thu, 25 Jan 2024 19:51:40 +0200 Subject: [PATCH 07/18] VM boot script: Run driverctl from second boot Currently, `driverctl` is executed on every boot. There is no need to run it before we configure `tuned-adm profile` and reboot the VM. Signed-off-by: Orel Misan (cherry picked from commit b9c39cc36f99d2ce9d3255ca884695feae2df756) --- pkg/internal/checkup/vmi.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/internal/checkup/vmi.go b/pkg/internal/checkup/vmi.go index 4c059a56..bee6bad0 100644 --- a/pkg/internal/checkup/vmi.go +++ b/pkg/internal/checkup/vmi.go @@ -139,8 +139,6 @@ func generateBootScript() string { sb.WriteString("set -e\n") sb.WriteString("\n") sb.WriteString("checkup_tuned_adm_set_marker_full_path=" + config.BootScriptTunedAdmSetMarkerFileFullPath + "\n") - sb.WriteString("driverctl set-override " + config.VMIEastNICPCIAddress + " vfio-pci\n") - sb.WriteString("driverctl set-override " + config.VMIWestNICPCIAddress + " vfio-pci\n") sb.WriteString("\n") sb.WriteString("if [ ! -f \"$checkup_tuned_adm_set_marker_full_path\" ]; then\n") sb.WriteString(" echo \"isolated_cores=" + isolatedCores + "\" > /etc/tuned/cpu-partitioning-variables.conf\n") @@ -148,6 +146,9 @@ func generateBootScript() string { sb.WriteString(" touch $checkup_tuned_adm_set_marker_full_path\n") sb.WriteString(" reboot\n") sb.WriteString("fi\n") + sb.WriteString("\n") + sb.WriteString("driverctl set-override " + config.VMIEastNICPCIAddress + " vfio-pci\n") + sb.WriteString("driverctl set-override " + config.VMIWestNICPCIAddress + " vfio-pci\n") return sb.String() } From 7a2d8585776c027a2e602b180af4d34da0d8fa53 Mon Sep 17 00:00:00 2001 From: Orel Misan Date: Thu, 25 Jan 2024 20:18:56 +0200 Subject: [PATCH 08/18] Traffic-gen, customize-vm: Align with VM-under-test The traffic-gen's `customize-vm` script is the same as the one used by the VM-under-test, with an additional step of TRex installation. Reorder the file so the two scripts would be aligned. Signed-off-by: Orel Misan (cherry picked from commit 2e4df107d53728ec1f1c85ab88361392a6f0ae81) --- vms/traffic-gen/scripts/customize-vm | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/vms/traffic-gen/scripts/customize-vm b/vms/traffic-gen/scripts/customize-vm index d0914c36..53ec2896 100644 --- a/vms/traffic-gen/scripts/customize-vm +++ b/vms/traffic-gen/scripts/customize-vm @@ -27,6 +27,19 @@ disable_services() { done } +# Setup hugepages in cmdline +setup_hugepages() { + mkdir -p /mnt/huge + echo "hugetlbfs /mnt/huge hugetlbfs defaults,pagesize=1GB 0 0" >> /etc/fstab + + grubby --update-kernel=ALL --args="default_hugepagesz=1GB hugepagesz=1G hugepages=1" +} + +# Enable unsafe noiommu mode +set_unsafe_no_io_mmu_mode() { + echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf +} + # Install trex package install_trex() { local TREX_URL=https://trex-tgn.cisco.com/trex/release @@ -44,20 +57,7 @@ install_trex() { rm ${TREX_ARCHIVE_NAME} } -# Setup hugepages in cmdline -setup_hugepages() { - mkdir -p /mnt/huge - echo "hugetlbfs /mnt/huge hugetlbfs defaults,pagesize=1GB 0 0" >> /etc/fstab - - grubby --update-kernel=ALL --args="default_hugepagesz=1GB hugepagesz=1G hugepages=1" -} - -# Enable unsafe noiommu mode -set_unsafe_no_io_mmu_mode() { - echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf -} - disable_services setup_hugepages -install_trex set_unsafe_no_io_mmu_mode +install_trex From 6a1587d6538bc3fb9da418e91753d367e3a3ec83 Mon Sep 17 00:00:00 2001 From: Orel Misan Date: Sun, 28 Jan 2024 11:26:26 +0200 Subject: [PATCH 09/18] console, login: Fix Regex when logging in as root Currently, when logging in as root - the login operation fails because of a mistake in the directory name expected by `loggedInPromptRegex`. Signed-off-by: Orel Misan (cherry picked from commit 9108b986df908bb5a1ac9371abcc8844f7bf7d47) --- pkg/internal/checkup/executor/console/login.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/internal/checkup/executor/console/login.go b/pkg/internal/checkup/executor/console/login.go index b392fc28..782aaa24 100644 --- a/pkg/internal/checkup/executor/console/login.go +++ b/pkg/internal/checkup/executor/console/login.go @@ -48,7 +48,7 @@ func (e Expecter) LoginToCentOS(username, password string) error { // Do not login, if we already logged in loggedInPromptRegex := fmt.Sprintf( - `(\[%s@(localhost|centos|%s) ~\]\$ |\[root@(localhost|centos|%s) centos\]\# )`, username, e.vmiName, e.vmiName, + `(\[%s@(localhost|centos|%s) ~\]\$ |\[root@(localhost|centos|%s) ~\]\# )`, username, e.vmiName, e.vmiName, ) b := []expect.Batcher{ &expect.BSnd{S: "\n"}, From 91f33976acabaa12c48a4d01eada52db2dada850 Mon Sep 17 00:00:00 2001 From: Orel Misan Date: Sun, 28 Jan 2024 11:27:08 +0200 Subject: [PATCH 10/18] Console: Log in as root Currently, the checkup creates a `cloud-user` user using cloud-init, and logs in to the VMs using its credentials. Immediately after logging in, the checkup issues the `sudo su` command, thus effectively running as the root user. Logging in as root simplifies the login operation, and remove problems that can occur when certain assumptions are made when performing the `sudo su` step - for example: 1. Not expecting the sudo warning message 2. Not expecting to enter the user's password again. Also remove the creation of `cloud-user` from cloud-init, since it is no longer needed. Signed-off-by: Orel Misan (cherry picked from commit f646c65a7b852f37392cd81eb34b119499a71531) --- .../checkup/executor/console/login.go | 2 -- pkg/internal/checkup/vmi.go | 14 +++---------- pkg/internal/checkup/vmi_test.go | 20 +------------------ pkg/internal/config/config.go | 4 ++-- 4 files changed, 6 insertions(+), 34 deletions(-) diff --git a/pkg/internal/checkup/executor/console/login.go b/pkg/internal/checkup/executor/console/login.go index 782aaa24..e79eeef1 100644 --- a/pkg/internal/checkup/executor/console/login.go +++ b/pkg/internal/checkup/executor/console/login.go @@ -87,8 +87,6 @@ func (e Expecter) LoginToCentOS(username, password string) error { T: expect.OK(), }, }}, - &expect.BSnd{S: "sudo su\n"}, - &expect.BExp{R: PromptExpression}, } const loginTimeout = 2 * time.Minute res, err := genExpect.ExpectBatch(b, loginTimeout) diff --git a/pkg/internal/checkup/vmi.go b/pkg/internal/checkup/vmi.go index bee6bad0..43fda337 100644 --- a/pkg/internal/checkup/vmi.go +++ b/pkg/internal/checkup/vmi.go @@ -66,8 +66,7 @@ func newVMIUnderTest(name string, checkupConfig config.Config, configMapName str vmi.WithSRIOVInterface(eastNetworkName, checkupConfig.VMUnderTestEastMacAddress.String(), config.VMIEastNICPCIAddress), vmi.WithSRIOVInterface(westNetworkName, checkupConfig.VMUnderTestWestMacAddress.String(), config.VMIWestNICPCIAddress), vmi.WithContainerDisk(rootDiskName, checkupConfig.VMUnderTestContainerDiskImage), - vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, - CloudInit(config.VMIUsername, config.VMIPassword, vmiUnderTestBootCommands(configDiskSerial))), + vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, CloudInit(vmiUnderTestBootCommands(configDiskSerial))), vmi.WithConfigMapVolume(configVolumeName, configMapName), vmi.WithConfigMapDisk(configVolumeName, configDiskSerial), ) @@ -86,10 +85,7 @@ func newTrafficGen(name string, checkupConfig config.Config, configMapName strin vmi.WithSRIOVInterface(eastNetworkName, checkupConfig.TrafficGenEastMacAddress.String(), config.VMIEastNICPCIAddress), vmi.WithSRIOVInterface(westNetworkName, checkupConfig.TrafficGenWestMacAddress.String(), config.VMIWestNICPCIAddress), vmi.WithContainerDisk(rootDiskName, checkupConfig.TrafficGenContainerDiskImage), - vmi.WithCloudInitNoCloudVolume( - cloudInitDiskName, - CloudInit(config.VMIUsername, config.VMIPassword, trafficGenBootCommands(configDiskSerial)), - ), + vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, CloudInit(trafficGenBootCommands(configDiskSerial))), vmi.WithConfigMapVolume(configVolumeName, configMapName), vmi.WithConfigMapDisk(configVolumeName, configDiskSerial), ) @@ -153,13 +149,9 @@ func generateBootScript() string { return sb.String() } -func CloudInit(username, password string, bootCommands []string) string { +func CloudInit(bootCommands []string) string { sb := strings.Builder{} sb.WriteString("#cloud-config\n") - sb.WriteString(fmt.Sprintf("user: %s\n", username)) - sb.WriteString(fmt.Sprintf("password: %s\n", password)) - sb.WriteString("chpasswd:\n") - sb.WriteString(" expire: false\n") if len(bootCommands) != 0 { sb.WriteString("bootcmd:\n") diff --git a/pkg/internal/checkup/vmi_test.go b/pkg/internal/checkup/vmi_test.go index f4338300..361d7611 100644 --- a/pkg/internal/checkup/vmi_test.go +++ b/pkg/internal/checkup/vmi_test.go @@ -90,32 +90,14 @@ func TestAffinityCalculation(t *testing.T) { } func TestCloudInitString(t *testing.T) { - const username = "user" - const password = "password" - t.Run("without boot commands", func(t *testing.T) { - actualString := checkup.CloudInit(username, password, nil) - expectedString := `#cloud-config -user: user -password: password -chpasswd: - expire: false -` - - assert.Equal(t, expectedString, actualString) - }) - t.Run("with boot commands", func(t *testing.T) { bootCommands := []string{ "sudo mkdir /mnt/app-config", "sudo mount /dev/$(lsblk --nodeps -no name,serial | grep DEADBEEF | cut -f1 -d' ') /mnt/app-config", } - actualString := checkup.CloudInit(username, password, bootCommands) + actualString := checkup.CloudInit(bootCommands) expectedString := `#cloud-config -user: user -password: password -chpasswd: - expire: false bootcmd: - "sudo mkdir /mnt/app-config" - "sudo mount /dev/$(lsblk --nodeps -no name,serial | grep DEADBEEF | cut -f1 -d' ') /mnt/app-config" diff --git a/pkg/internal/config/config.go b/pkg/internal/config/config.go index 90a8e0b2..3b399934 100644 --- a/pkg/internal/config/config.go +++ b/pkg/internal/config/config.go @@ -57,8 +57,8 @@ const ( ) const ( - VMIUsername = "cloud-user" - VMIPassword = "0tli-pxem-xknu" // #nosec + VMIUsername = "root" + VMIPassword = "redhat" // #nosec VMIEastNICPCIAddress = "0000:06:00.0" VMIWestNICPCIAddress = "0000:07:00.0" From 9b022462c05bc1cc02fce34bd2fbd32f5daafcc6 Mon Sep 17 00:00:00 2001 From: Orel Misan Date: Sun, 28 Jan 2024 11:30:08 +0200 Subject: [PATCH 11/18] Move configureConsole to login.go `configureConsole` is only used in login.go. Signed-off-by: Orel Misan (cherry picked from commit 5161cae31b63b52cc2d441e246d8425247c2d40e) --- .../checkup/executor/console/console.go | 23 ------------------- .../checkup/executor/console/login.go | 23 +++++++++++++++++++ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pkg/internal/checkup/executor/console/console.go b/pkg/internal/checkup/executor/console/console.go index 4bc23514..2788434f 100644 --- a/pkg/internal/checkup/executor/console/console.go +++ b/pkg/internal/checkup/executor/console/console.go @@ -182,26 +182,3 @@ func expectBatchWithValidatedSend(expecter expect.Expecter, batch []expect.Batch res, err := expecter.ExpectBatch(batch, timeout) return res, err } - -func configureConsole(expecter expect.Expecter, shouldSudo bool) error { - sudoString := "" - if shouldSudo { - sudoString = "sudo " - } - batch := []expect.Batcher{ - &expect.BSnd{S: "stty cols 500 rows 500\n"}, - &expect.BExp{R: PromptExpression}, - &expect.BSnd{S: "echo $?\n"}, - &expect.BExp{R: RetValue("0")}, - &expect.BSnd{S: fmt.Sprintf("%sdmesg -n 1\n", sudoString)}, - &expect.BExp{R: PromptExpression}, - &expect.BSnd{S: "echo $?\n"}, - &expect.BExp{R: RetValue("0")}, - } - const configureConsoleTimeout = 30 * time.Second - resp, err := expecter.ExpectBatch(batch, configureConsoleTimeout) - if err != nil { - log.Printf("%v", resp) - } - return err -} diff --git a/pkg/internal/checkup/executor/console/login.go b/pkg/internal/checkup/executor/console/login.go index e79eeef1..c6f5d703 100644 --- a/pkg/internal/checkup/executor/console/login.go +++ b/pkg/internal/checkup/executor/console/login.go @@ -105,3 +105,26 @@ func (e Expecter) LoginToCentOS(username, password string) error { } return nil } + +func configureConsole(expecter expect.Expecter, shouldSudo bool) error { + sudoString := "" + if shouldSudo { + sudoString = "sudo " + } + batch := []expect.Batcher{ + &expect.BSnd{S: "stty cols 500 rows 500\n"}, + &expect.BExp{R: PromptExpression}, + &expect.BSnd{S: "echo $?\n"}, + &expect.BExp{R: RetValue("0")}, + &expect.BSnd{S: fmt.Sprintf("%sdmesg -n 1\n", sudoString)}, + &expect.BExp{R: PromptExpression}, + &expect.BSnd{S: "echo $?\n"}, + &expect.BExp{R: RetValue("0")}, + } + const configureConsoleTimeout = 30 * time.Second + resp, err := expecter.ExpectBatch(batch, configureConsoleTimeout) + if err != nil { + log.Printf("%v", resp) + } + return err +} From e4c63a796ff1e92400342ed67b2ac69e9771b89b Mon Sep 17 00:00:00 2001 From: Orel Misan Date: Sun, 28 Jan 2024 11:30:53 +0200 Subject: [PATCH 12/18] login, configureConsole: Remove shouldSudo param Since we log in as root, there is no need to have support for sudo. Signed-off-by: Orel Misan (cherry picked from commit 85a5b09296ef2f4da812df912489e80399eac318) --- pkg/internal/checkup/executor/console/login.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pkg/internal/checkup/executor/console/login.go b/pkg/internal/checkup/executor/console/login.go index c6f5d703..afe21257 100644 --- a/pkg/internal/checkup/executor/console/login.go +++ b/pkg/internal/checkup/executor/console/login.go @@ -99,24 +99,20 @@ func (e Expecter) LoginToCentOS(username, password string) error { } } - err = configureConsole(genExpect, false) + err = configureConsole(genExpect) if err != nil { return err } return nil } -func configureConsole(expecter expect.Expecter, shouldSudo bool) error { - sudoString := "" - if shouldSudo { - sudoString = "sudo " - } +func configureConsole(expecter expect.Expecter) error { batch := []expect.Batcher{ &expect.BSnd{S: "stty cols 500 rows 500\n"}, &expect.BExp{R: PromptExpression}, &expect.BSnd{S: "echo $?\n"}, &expect.BExp{R: RetValue("0")}, - &expect.BSnd{S: fmt.Sprintf("%sdmesg -n 1\n", sudoString)}, + &expect.BSnd{S: "dmesg -n 1\n"}, &expect.BExp{R: PromptExpression}, &expect.BSnd{S: "echo $?\n"}, &expect.BExp{R: RetValue("0")}, From d3416be18b00917265a03b8754ef0bbd081b78eb Mon Sep 17 00:00:00 2001 From: Orel Misan Date: Sun, 28 Jan 2024 11:38:49 +0200 Subject: [PATCH 13/18] login: Change LoginToCentOS to LoginToCentOSAsRoot Since we only log in as `root` - simply `LoginToCentOS` and rename it to match its abilities. Signed-off-by: Orel Misan (cherry picked from commit 6b627dcc3bc3d033529563e44363b31261900fc3) --- pkg/internal/checkup/executor/console/login.go | 8 +++----- pkg/internal/checkup/executor/executor.go | 6 ++---- pkg/internal/config/config.go | 1 - 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/internal/checkup/executor/console/login.go b/pkg/internal/checkup/executor/console/login.go index afe21257..43d452ae 100644 --- a/pkg/internal/checkup/executor/console/login.go +++ b/pkg/internal/checkup/executor/console/login.go @@ -29,7 +29,7 @@ import ( "google.golang.org/grpc/codes" ) -func (e Expecter) LoginToCentOS(username, password string) error { +func (e Expecter) LoginToCentOSAsRoot(password string) error { const ( connectionTimeout = 10 * time.Second promptTimeout = 5 * time.Second @@ -47,9 +47,7 @@ func (e Expecter) LoginToCentOS(username, password string) error { } // Do not login, if we already logged in - loggedInPromptRegex := fmt.Sprintf( - `(\[%s@(localhost|centos|%s) ~\]\$ |\[root@(localhost|centos|%s) ~\]\# )`, username, e.vmiName, e.vmiName, - ) + loggedInPromptRegex := fmt.Sprintf(`(\[root@(localhost|centos|%s) ~\]\# )`, e.vmiName) b := []expect.Batcher{ &expect.BSnd{S: "\n"}, &expect.BExp{R: loggedInPromptRegex}, @@ -67,7 +65,7 @@ func (e Expecter) LoginToCentOS(username, password string) error { // Using only "login: " would match things like "Last failed login: Tue Jun 9 22:25:30 UTC 2020 on ttyS0" // and in case the VM's did not get hostname form DHCP server try the default hostname R: regexp.MustCompile(fmt.Sprintf(`(localhost|centos|%s) login: `, e.vmiName)), - S: fmt.Sprintf("%s\n", username), + S: "root\n", T: expect.Next(), Rt: 10, }, diff --git a/pkg/internal/checkup/executor/executor.go b/pkg/internal/checkup/executor/executor.go index 5589b607..0e918a78 100644 --- a/pkg/internal/checkup/executor/executor.go +++ b/pkg/internal/checkup/executor/executor.go @@ -43,7 +43,6 @@ type vmiSerialConsoleClient interface { type Executor struct { vmiSerialClient vmiSerialConsoleClient namespace string - vmiUsername string vmiPassword string vmiUnderTestEastNICPCIAddress string trafficGenEastMACAddress string @@ -58,7 +57,6 @@ func New(client vmiSerialConsoleClient, namespace string, cfg config.Config) Exe return Executor{ vmiSerialClient: client, namespace: namespace, - vmiUsername: config.VMIUsername, vmiPassword: config.VMIPassword, vmiUnderTestEastNICPCIAddress: config.VMIEastNICPCIAddress, trafficGenEastMACAddress: cfg.TrafficGenEastMacAddress.String(), @@ -73,13 +71,13 @@ func New(client vmiSerialConsoleClient, namespace string, cfg config.Config) Exe func (e Executor) Execute(ctx context.Context, vmiUnderTestName, trafficGenVMIName string) (status.Results, error) { log.Printf("Login to VMI under test...") vmiUnderTestConsoleExpecter := console.NewExpecter(e.vmiSerialClient, e.namespace, vmiUnderTestName) - if err := vmiUnderTestConsoleExpecter.LoginToCentOS(e.vmiUsername, e.vmiPassword); err != nil { + if err := vmiUnderTestConsoleExpecter.LoginToCentOSAsRoot(e.vmiPassword); err != nil { return status.Results{}, fmt.Errorf("failed to login to VMI \"%s/%s\": %w", e.namespace, vmiUnderTestName, err) } log.Printf("Login to traffic generator...") trafficGenConsoleExpecter := console.NewExpecter(e.vmiSerialClient, e.namespace, trafficGenVMIName) - if err := trafficGenConsoleExpecter.LoginToCentOS(e.vmiUsername, e.vmiPassword); err != nil { + if err := trafficGenConsoleExpecter.LoginToCentOSAsRoot(e.vmiPassword); err != nil { return status.Results{}, fmt.Errorf("failed to login to VMI \"%s/%s\": %w", e.namespace, trafficGenVMIName, err) } diff --git a/pkg/internal/config/config.go b/pkg/internal/config/config.go index 3b399934..18cad901 100644 --- a/pkg/internal/config/config.go +++ b/pkg/internal/config/config.go @@ -57,7 +57,6 @@ const ( ) const ( - VMIUsername = "root" VMIPassword = "redhat" // #nosec VMIEastNICPCIAddress = "0000:06:00.0" From 5b25c9136479ca9cd6ab2737262dff800052a999 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Sun, 28 Jan 2024 11:24:18 +0200 Subject: [PATCH 14/18] checkup/vmi: Change boot script set flags Since the checkup relies on the boot script finishing, it is better to not fail on error. Replacing the "-e" option with a "-x" for better verbosity should the checkup fail. Setting the "-x" flag makes the script run visible in /var/log/cloud-init-output.log Signed-off-by: Ram Lavi (cherry picked from commit 3d063e7d9b5d92cb6c5a9d4c1e8785b3db2c6ec6) --- pkg/internal/checkup/vmi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/internal/checkup/vmi.go b/pkg/internal/checkup/vmi.go index 43fda337..7b13e2e8 100644 --- a/pkg/internal/checkup/vmi.go +++ b/pkg/internal/checkup/vmi.go @@ -132,7 +132,7 @@ func generateBootScript() string { sb := strings.Builder{} sb.WriteString("#!/bin/bash\n") - sb.WriteString("set -e\n") + sb.WriteString("set -x\n") sb.WriteString("\n") sb.WriteString("checkup_tuned_adm_set_marker_full_path=" + config.BootScriptTunedAdmSetMarkerFileFullPath + "\n") sb.WriteString("\n") From 4a1693aef2d3315d69bd2cbe2673f890fc619381 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Thu, 25 Jan 2024 21:59:45 +0200 Subject: [PATCH 15/18] checkup/vmi: Add "exit 0" after reboot After initially configuring the VM, a reboot is performed. However it is not immediatly taking affect. To make sure that the script exits right after the reboot is set, initiating manual exit from the script. Signed-off-by: Ram Lavi (cherry picked from commit 9b199c4285b3617d8cdcd6cb69dce1cd2a5c0d0f) --- pkg/internal/checkup/vmi.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/internal/checkup/vmi.go b/pkg/internal/checkup/vmi.go index 7b13e2e8..75269269 100644 --- a/pkg/internal/checkup/vmi.go +++ b/pkg/internal/checkup/vmi.go @@ -141,6 +141,7 @@ func generateBootScript() string { sb.WriteString(" tuned-adm profile cpu-partitioning\n\n") sb.WriteString(" touch $checkup_tuned_adm_set_marker_full_path\n") sb.WriteString(" reboot\n") + sb.WriteString(" exit 0\n") sb.WriteString("fi\n") sb.WriteString("\n") sb.WriteString("driverctl set-override " + config.VMIEastNICPCIAddress + " vfio-pci\n") From faa558acfb645a0a9dfd7900f6773c2c815e5164 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Wed, 24 Jan 2024 09:57:32 +0200 Subject: [PATCH 16/18] vms/customize-vm: Enable guest-agent-exec, guest-agent-exec-status This commit enables the guest-agent-exec option on the guest, in order to use the probe polling option, that will be introduced in future commits. Signed-off-by: Ram Lavi (cherry picked from commit 91c401ba693d2b5ec6514f3772525be0c84f8b75) --- vms/traffic-gen/scripts/customize-vm | 7 +++++++ vms/vm-under-test/scripts/customize-vm | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/vms/traffic-gen/scripts/customize-vm b/vms/traffic-gen/scripts/customize-vm index 53ec2896..d411ea96 100644 --- a/vms/traffic-gen/scripts/customize-vm +++ b/vms/traffic-gen/scripts/customize-vm @@ -40,6 +40,12 @@ set_unsafe_no_io_mmu_mode() { echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf } +# Enable guest-exec on the qemu-guest-agent configuration +enable_guest_exec() { + sed -i '/^BLACKLIST_RPC=/ { s/guest-exec-status//; s/guest-exec//g }' /etc/sysconfig/qemu-ga + sed -i '/^BLACKLIST_RPC=/ { s/,\+/,/g; s/^,\|,$//g }' /etc/sysconfig/qemu-ga +} + # Install trex package install_trex() { local TREX_URL=https://trex-tgn.cisco.com/trex/release @@ -60,4 +66,5 @@ install_trex() { disable_services setup_hugepages set_unsafe_no_io_mmu_mode +enable_guest_exec install_trex diff --git a/vms/vm-under-test/scripts/customize-vm b/vms/vm-under-test/scripts/customize-vm index 15ea165b..7364ba6d 100644 --- a/vms/vm-under-test/scripts/customize-vm +++ b/vms/vm-under-test/scripts/customize-vm @@ -40,6 +40,13 @@ set_unsafe_no_io_mmu_mode() { echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf } +# Enable guest-exec on the qemu-guest-agent configuration +enable_guest_exec() { + sed -i '/^BLACKLIST_RPC=/ { s/guest-exec-status//; s/guest-exec//g }' /etc/sysconfig/qemu-ga + sed -i '/^BLACKLIST_RPC=/ { s/,\+/,/g; s/^,\|,$//g }' /etc/sysconfig/qemu-ga +} + disable_services setup_hugepages set_unsafe_no_io_mmu_mode +enable_guest_exec From 635a46e9b441db6968dbd550d07fce7e721e6723 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Thu, 25 Jan 2024 15:51:08 +0200 Subject: [PATCH 17/18] checkup: Add readiness marker file In order for the VMI to be properly configured for DPDK, adding a new marker file that will be polled by guest-agent-exec in future commits. Also setting the correct selinux context of the marker file so it will be properly polled [0]. [0] https://kubevirt.io/user-guide/virtual_machines/liveness_and_readiness_probes/#defining-guest-agent-ping-probes Signed-off-by: Ram Lavi (cherry picked from commit beb3ede87b15ef71a162a59eff9d792fd22a4b4f) --- pkg/internal/checkup/vmi.go | 2 ++ pkg/internal/config/config.go | 1 + 2 files changed, 3 insertions(+) diff --git a/pkg/internal/checkup/vmi.go b/pkg/internal/checkup/vmi.go index 75269269..57c0c9aa 100644 --- a/pkg/internal/checkup/vmi.go +++ b/pkg/internal/checkup/vmi.go @@ -146,6 +146,8 @@ func generateBootScript() string { sb.WriteString("\n") sb.WriteString("driverctl set-override " + config.VMIEastNICPCIAddress + " vfio-pci\n") sb.WriteString("driverctl set-override " + config.VMIWestNICPCIAddress + " vfio-pci\n") + sb.WriteString("touch " + config.BootScriptReadinessMarkerFileFullPath + "\n") + sb.WriteString("chcon -t virt_qemu_ga_exec_t " + config.BootScriptReadinessMarkerFileFullPath + "\n") return sb.String() } diff --git a/pkg/internal/config/config.go b/pkg/internal/config/config.go index 18cad901..23755a13 100644 --- a/pkg/internal/config/config.go +++ b/pkg/internal/config/config.go @@ -65,6 +65,7 @@ const ( BootScriptName = "dpdk-checkup-boot.sh" BootScriptBinDirectory = "/usr/bin/" BootScriptTunedAdmSetMarkerFileFullPath = "/var/dpdk-checkup-tuned-adm-set-marker" + BootScriptReadinessMarkerFileFullPath = "/tmp/dpdk-checkup-ready-marker" ) var ( From 85898cfcafb97942780d8ef6753323fbc7b7f531 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Wed, 24 Jan 2024 09:56:13 +0200 Subject: [PATCH 18/18] checkup: Add guest-agent-ping probe replace wait cond to ready state Currently the checkup setup only waits for the VMI to finish "booting", i.e. the guest agent service to be ready. However this is not enough in order to ensure that the VMI has been configured, a procedure currently done on the cloud-init service. When the configuration is complete, the script configuring the guest in the cloud-init service adds a marker file. Changing the wait condition to "ready" is stronger than waiting for "guest-agent", since the guest-agent ping needs guest agent in order to set the VMI to ready. This commit: - introduces a new waiting mechanism, using guest-agent-ping probe [0] to poll-wait the guest until the file is present, and only then sets the VMI ready condition to true. - adds a wait for the VMI ready condition to be true. - removed the now unnecessary wait for guest-agent-ready condition. [0] https://kubevirt.io/user-guide/virtual_machines/liveness_and_readiness_probes/#defining-guest-agent-ping-probes Signed-off-by: Ram Lavi (cherry picked from commit ee2b43303666003ebc141ec2357803c9e286c7ad) --- pkg/internal/checkup/checkup.go | 15 +++++++-------- pkg/internal/checkup/checkup_test.go | 9 +++++---- pkg/internal/checkup/vmi.go | 2 ++ pkg/internal/checkup/vmi/spec.go | 17 +++++++++++++++++ 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/pkg/internal/checkup/checkup.go b/pkg/internal/checkup/checkup.go index 0f119117..b7896235 100644 --- a/pkg/internal/checkup/checkup.go +++ b/pkg/internal/checkup/checkup.go @@ -124,15 +124,14 @@ func (c *Checkup) Setup(ctx context.Context) (setupErr error) { }() var updatedVMIUnderTest *kvcorev1.VirtualMachineInstance - updatedVMIUnderTest, err = c.waitForVMIToBoot(setupCtx, c.vmiUnderTest.Name) + updatedVMIUnderTest, err = c.waitForVMIToBeReady(setupCtx, c.vmiUnderTest.Name) if err != nil { return err } c.vmiUnderTest = updatedVMIUnderTest - var updatedTrafficGen *kvcorev1.VirtualMachineInstance - updatedTrafficGen, err = c.waitForVMIToBoot(setupCtx, c.trafficGen.Name) + updatedTrafficGen, err = c.waitForVMIToBeReady(setupCtx, c.trafficGen.Name) if err != nil { return err } @@ -233,9 +232,9 @@ func (c *Checkup) createVMI(ctx context.Context, vmiToCreate *kvcorev1.VirtualMa return err } -func (c *Checkup) waitForVMIToBoot(ctx context.Context, name string) (*kvcorev1.VirtualMachineInstance, error) { +func (c *Checkup) waitForVMIToBeReady(ctx context.Context, name string) (*kvcorev1.VirtualMachineInstance, error) { vmiFullName := ObjectFullName(c.namespace, name) - log.Printf("Waiting for VMI %q to boot...", vmiFullName) + log.Printf("Waiting for VMI %q to be ready...", vmiFullName) var updatedVMI *kvcorev1.VirtualMachineInstance conditionFn := func(ctx context.Context) (bool, error) { @@ -246,7 +245,7 @@ func (c *Checkup) waitForVMIToBoot(ctx context.Context, name string) (*kvcorev1. } for _, condition := range updatedVMI.Status.Conditions { - if condition.Type == kvcorev1.VirtualMachineInstanceAgentConnected && condition.Status == k8scorev1.ConditionTrue { + if condition.Type == kvcorev1.VirtualMachineInstanceReady && condition.Status == k8scorev1.ConditionTrue { return true, nil } } @@ -255,10 +254,10 @@ func (c *Checkup) waitForVMIToBoot(ctx context.Context, name string) (*kvcorev1. } const pollInterval = 5 * time.Second if err := wait.PollImmediateUntilWithContext(ctx, pollInterval, conditionFn); err != nil { - return nil, fmt.Errorf("failed to wait for VMI %q to boot: %v", vmiFullName, err) + return nil, fmt.Errorf("failed to wait for VMI %q to be ready: %v", vmiFullName, err) } - log.Printf("VMI %q had successfully booted", vmiFullName) + log.Printf("VMI %q has successfully reached ready condition", vmiFullName) return updatedVMI, nil } diff --git a/pkg/internal/checkup/checkup_test.go b/pkg/internal/checkup/checkup_test.go index dd2d5983..30af6cb3 100644 --- a/pkg/internal/checkup/checkup_test.go +++ b/pkg/internal/checkup/checkup_test.go @@ -417,10 +417,11 @@ func (cs *clientStub) GetVirtualMachineInstance(_ context.Context, namespace, na return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachineinstances"}, name) } - vmi.Status.Conditions = append(vmi.Status.Conditions, kvcorev1.VirtualMachineInstanceCondition{ - Type: kvcorev1.VirtualMachineInstanceAgentConnected, - Status: k8scorev1.ConditionTrue, - }) + vmi.Status.Conditions = append(vmi.Status.Conditions, + kvcorev1.VirtualMachineInstanceCondition{ + Type: kvcorev1.VirtualMachineInstanceReady, + Status: k8scorev1.ConditionTrue, + }) return vmi, nil } diff --git a/pkg/internal/checkup/vmi.go b/pkg/internal/checkup/vmi.go index 57c0c9aa..0e16f3a3 100644 --- a/pkg/internal/checkup/vmi.go +++ b/pkg/internal/checkup/vmi.go @@ -69,6 +69,7 @@ func newVMIUnderTest(name string, checkupConfig config.Config, configMapName str vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, CloudInit(vmiUnderTestBootCommands(configDiskSerial))), vmi.WithConfigMapVolume(configVolumeName, configMapName), vmi.WithConfigMapDisk(configVolumeName, configDiskSerial), + vmi.WithReadinessFileProbe(config.BootScriptReadinessMarkerFileFullPath), ) return vmi.New(name, optionsToApply...) @@ -88,6 +89,7 @@ func newTrafficGen(name string, checkupConfig config.Config, configMapName strin vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, CloudInit(trafficGenBootCommands(configDiskSerial))), vmi.WithConfigMapVolume(configVolumeName, configMapName), vmi.WithConfigMapDisk(configVolumeName, configDiskSerial), + vmi.WithReadinessFileProbe(config.BootScriptReadinessMarkerFileFullPath), ) return vmi.New(name, optionsToApply...) diff --git a/pkg/internal/checkup/vmi/spec.go b/pkg/internal/checkup/vmi/spec.go index 643bab5b..c60c175a 100644 --- a/pkg/internal/checkup/vmi/spec.go +++ b/pkg/internal/checkup/vmi/spec.go @@ -262,6 +262,23 @@ func WithAffinity(affinity *corev1.Affinity) Option { } } +func WithReadinessFileProbe(fileName string) Option { + return func(vmi *kvcorev1.VirtualMachineInstance) { + var readinessProbeCommand = []string{"cat", fileName} + vmi.Spec.ReadinessProbe = &kvcorev1.Probe{ + Handler: kvcorev1.Handler{ + Exec: &corev1.ExecAction{ + Command: readinessProbeCommand, + }, + }, + FailureThreshold: 30, + InitialDelaySeconds: 90, + PeriodSeconds: 10, + TimeoutSeconds: 10, + } + } +} + func Pointer[T any](v T) *T { return &v }