diff --git a/pkg/internal/checkup/executor/console/console.go b/pkg/internal/checkup/executor/console/console.go index d6bf7bc7..4bc23514 100644 --- a/pkg/internal/checkup/executor/console/console.go +++ b/pkg/internal/checkup/executor/console/console.go @@ -101,6 +101,17 @@ func RetValue(retcode string) string { return "\n" + retcode + CRLF + ".*" + PromptExpression } +func (e Expecter) GetGuestKernelArgs() (string, error) { + const cmdLineCmd = "cat /proc/cmdline\n" + batch := []expect.Batcher{ + &expect.BSnd{S: cmdLineCmd}, + &expect.BExp{R: PromptExpression}, + } + const printKernelArgsTimeout = 30 * time.Second + resp, err := e.SafeExpectBatchWithResponse(batch, printKernelArgsTimeout) + return resp[0].Output, err +} + // SafeExpectBatchWithResponse runs the batch from `expected`, connecting to a VMI's console and // waiting for the batch to return with a response until timeout. // It validates that the commands arrive to the console. diff --git a/pkg/internal/checkup/executor/executor.go b/pkg/internal/checkup/executor/executor.go index c6174ae7..5589b607 100644 --- a/pkg/internal/checkup/executor/executor.go +++ b/pkg/internal/checkup/executor/executor.go @@ -83,6 +83,14 @@ func (e Executor) Execute(ctx context.Context, vmiUnderTestName, trafficGenVMINa return status.Results{}, fmt.Errorf("failed to login to VMI \"%s/%s\": %w", e.namespace, trafficGenVMIName, err) } + if e.verbosePrintsEnabled { + vmiUnderTestKernelArgs, _ := vmiUnderTestConsoleExpecter.GetGuestKernelArgs() + log.Printf("VMI under test guest kernel Args: %s", vmiUnderTestKernelArgs) + + trafficGenKernelArgs, _ := trafficGenConsoleExpecter.GetGuestKernelArgs() + log.Printf("traffic generator guest kernel Args: %s", trafficGenKernelArgs) + } + trexClient := trex.NewClient( trafficGenConsoleExpecter, e.trafficGeneratorPacketsPerSecond, diff --git a/vms/traffic-gen/scripts/build-vm-image b/vms/traffic-gen/scripts/build-vm-image index 770bd7d4..39821224 100755 --- a/vms/traffic-gen/scripts/build-vm-image +++ b/vms/traffic-gen/scripts/build-vm-image @@ -24,6 +24,5 @@ virt-builder centosstream-8 \ --root-password password:redhat \ --install cloud-init,driverctl,tuned-profiles-cpu-partitioning,tar,python3,pciutils \ --run /root/scripts/customize-vm \ - --firstboot /root/scripts/first-boot \ --selinux-relabel \ --output /output/kubevirt-dpdk-checkup-traffic-gen.qcow2 diff --git a/vms/traffic-gen/scripts/customize-vm b/vms/traffic-gen/scripts/customize-vm index 424205aa..069f88c3 100644 --- a/vms/traffic-gen/scripts/customize-vm +++ b/vms/traffic-gen/scripts/customize-vm @@ -19,25 +19,113 @@ set -e -systemctl disable NetworkManager-wait-online -systemctl disable sshd +# Disable unnecessary services +disable_services() { + local service_list=("NetworkManager-wait-online" "sshd") + for service in "${service_list[@]}"; do + systemctl disable "$service" + done +} -grubby --update-kernel=ALL --args="default_hugepagesz=1GB hugepagesz=1G hugepages=1" +# Install trex package +install_trex() { + local TREX_URL=https://trex-tgn.cisco.com/trex/release + local TREX_VERSION=v3.03 + local TREX_ARCHIVE_NAME=${TREX_VERSION}.tar.gz + local TREX_DIR=/opt/trex -echo isolated_cores=2-7 > /etc/tuned/cpu-partitioning-variables.conf -tuned-adm profile cpu-partitioning -echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf + mkdir -p ${TREX_DIR} + cd ${TREX_DIR} -TREX_URL=https://trex-tgn.cisco.com/trex/release -TREX_VERSION=v3.03 -TREX_ARCHIVE_NAME=${TREX_VERSION}.tar.gz -TREX_DIR=/opt/trex + curl --insecure ${TREX_URL}/${TREX_ARCHIVE_NAME} --output ${TREX_ARCHIVE_NAME} -mkdir -p ${TREX_DIR} -cd ${TREX_DIR} + tar -xzf ${TREX_ARCHIVE_NAME} --strip-components=1 -curl --insecure ${TREX_URL}/${TREX_ARCHIVE_NAME} --output ${TREX_ARCHIVE_NAME} + rm ${TREX_ARCHIVE_NAME} +} -tar -xzf ${TREX_ARCHIVE_NAME} --strip-components=1 +# Setup hugepages in cmdline +setup_hugepages() { + grubby --update-kernel=ALL --args="default_hugepagesz=1GB hugepagesz=1G hugepages=1" +} -rm ${TREX_ARCHIVE_NAME} +# Enable unsafe noiommu mode +set_unsafe_no_io_mmu_mode() { + echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf +} + +# Create and enable the boot checkup service +setup_boot_service() { + local service_name="dpdk-checkup-boot.service" + local checkup_boot_script_full_path="/usr/bin/dpdk-checkup-boot.sh" + local checkup_boot_service_full_path="/usr/lib/systemd/system/$service_name" + local checkup_tuned_adm_set_marker_full_path="/var/dpdk-checkup-tuned-adm-set-marker" + + setup_checkup_boot_script "$checkup_boot_script_full_path" + + cat < "$checkup_boot_service_full_path" +[Unit] +Description=Checkup Boot Script +Before=qemu-guest-agent.service +[Service] +ExecStart=$checkup_boot_script_full_path $checkup_tuned_adm_set_marker_full_path +Restart=no +User=root +Group=root +[Install] +WantedBy=multi-user.target +EOF + + systemctl enable "$checkup_boot_service_full_path" + systemctl start "$checkup_boot_service_full_path" +} + +setup_checkup_boot_script() { + local checkup_boot_script_full_path=$1 +cat <<'EOF' > "$checkup_boot_script_full_path" +#!/usr/bin/env bash +# +# This file is part of the kiagnose project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2024 Red Hat, Inc. +# + +set -e + +checkup_tuned_adm_set_marker_full_path=$1 + +driverctl set-override 0000:06:00.0 vfio-pci +driverctl set-override 0000:07:00.0 vfio-pci + +mkdir -p /mnt/huge +mount /mnt/huge --source nodev -t hugetlbfs -o pagesize=1GB + +if [ ! -f "$checkup_tuned_adm_set_marker_full_path" ]; then + echo "isolated_cores=2-7" > /etc/tuned/cpu-partitioning-variables.conf + tuned-adm profile cpu-partitioning + + touch $checkup_tuned_adm_set_marker_full_path + reboot +fi +EOF + + chmod +x "$checkup_boot_script_full_path" +} + +disable_services +setup_hugepages +install_trex +set_unsafe_no_io_mmu_mode +setup_boot_service diff --git a/vms/traffic-gen/scripts/first-boot b/vms/traffic-gen/scripts/first-boot deleted file mode 100755 index 54b21ab6..00000000 --- a/vms/traffic-gen/scripts/first-boot +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# -# This file is part of the kiagnose project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2023 Red Hat, Inc. -# - -driverctl set-override 0000:06:00.0 vfio-pci -driverctl set-override 0000:07:00.0 vfio-pci - -mkdir /mnt/huge -mount /mnt/huge --source nodev -t hugetlbfs -o pagesize=1GB diff --git a/vms/vm-under-test/scripts/build-vm-image b/vms/vm-under-test/scripts/build-vm-image index 04e0b493..772f926c 100755 --- a/vms/vm-under-test/scripts/build-vm-image +++ b/vms/vm-under-test/scripts/build-vm-image @@ -24,6 +24,5 @@ virt-builder centosstream-8 \ --root-password password:redhat \ --install cloud-init,dpdk,dpdk-tools,driverctl,tuned-profiles-cpu-partitioning \ --run /root/scripts/customize-vm \ - --firstboot /root/scripts/first-boot \ --selinux-relabel \ --output /output/kubevirt-dpdk-checkup-vm.qcow2 diff --git a/vms/vm-under-test/scripts/customize-vm b/vms/vm-under-test/scripts/customize-vm index 4692b95a..0814c576 100644 --- a/vms/vm-under-test/scripts/customize-vm +++ b/vms/vm-under-test/scripts/customize-vm @@ -17,11 +17,97 @@ # Copyright 2023 Red Hat, Inc. # -systemctl disable NetworkManager-wait-online -systemctl disable sshd +set -e -grubby --update-kernel=ALL --args="default_hugepagesz=1GB hugepagesz=1G hugepages=1" +# Disable unnecessary services +disable_services() { + local service_list=("NetworkManager-wait-online" "sshd") + for service in "${service_list[@]}"; do + systemctl disable "$service" + done +} -echo isolated_cores=2-7 > /etc/tuned/cpu-partitioning-variables.conf -tuned-adm profile cpu-partitioning -echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf +# Setup hugepages in cmdline +setup_hugepages() { + grubby --update-kernel=ALL --args="default_hugepagesz=1GB hugepagesz=1G hugepages=1" +} + +# Enable unsafe noiommu mode +set_unsafe_no_io_mmu_mode() { + echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf +} + +# Create and enable the boot checkup service +setup_boot_service() { + local service_name="dpdk-checkup-boot.service" + local checkup_boot_script_full_path="/usr/bin/dpdk-checkup-boot.sh" + local checkup_boot_service_full_path="/usr/lib/systemd/system/$service_name" + local checkup_tuned_adm_set_marker_full_path="/var/dpdk-checkup-tuned-adm-set-marker" + + setup_checkup_boot_script "$checkup_boot_script_full_path" + + cat < "$checkup_boot_service_full_path" +[Unit] +Description=Checkup Boot Script +Before=qemu-guest-agent.service +[Service] +ExecStart=$checkup_boot_script_full_path $checkup_tuned_adm_set_marker_full_path +Restart=no +User=root +Group=root +[Install] +WantedBy=multi-user.target +EOF + + systemctl enable "$checkup_boot_service_full_path" + systemctl start "$checkup_boot_service_full_path" +} + +setup_checkup_boot_script() { + local checkup_boot_script_full_path=$1 +cat <<'EOF' > "$checkup_boot_script_full_path" +#!/usr/bin/env bash +# +# This file is part of the kiagnose project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2024 Red Hat, Inc. +# + +set -e + +checkup_tuned_adm_set_marker_full_path=$1 + +driverctl set-override 0000:06:00.0 vfio-pci +driverctl set-override 0000:07:00.0 vfio-pci + +mkdir -p /mnt/huge +mount /mnt/huge --source nodev -t hugetlbfs -o pagesize=1GB + +if [ ! -f "$checkup_tuned_adm_set_marker_full_path" ]; then + echo "isolated_cores=2-7" > /etc/tuned/cpu-partitioning-variables.conf + tuned-adm profile cpu-partitioning + + touch $checkup_tuned_adm_set_marker_full_path + reboot +fi +EOF + + chmod +x "$checkup_boot_script_full_path" +} + +disable_services +setup_hugepages +set_unsafe_no_io_mmu_mode +setup_boot_service diff --git a/vms/vm-under-test/scripts/first-boot b/vms/vm-under-test/scripts/first-boot deleted file mode 100755 index 54b21ab6..00000000 --- a/vms/vm-under-test/scripts/first-boot +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# -# This file is part of the kiagnose project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2023 Red Hat, Inc. -# - -driverctl set-override 0000:06:00.0 vfio-pci -driverctl set-override 0000:07:00.0 vfio-pci - -mkdir /mnt/huge -mount /mnt/huge --source nodev -t hugetlbfs -o pagesize=1GB