From 7fb8d02e4c54017af218c62ed089adcc08d6cdc4 Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Wed, 16 Oct 2024 17:09:26 -0700 Subject: [PATCH 01/12] Pkg-Config: Allow update of PKG_CONFIG_PATH --- lisa/tools/pkgconfig.py | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/lisa/tools/pkgconfig.py b/lisa/tools/pkgconfig.py index bc2bdd75ea..23b97736bd 100644 --- a/lisa/tools/pkgconfig.py +++ b/lisa/tools/pkgconfig.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from typing import Optional + from assertpy import assert_that from semver import VersionInfo @@ -23,16 +25,34 @@ def install(self) -> bool: self.node.os.install_packages("pkg-config") return True - def package_info_exists(self, package_name: str) -> bool: - package_info_result = self.run(f"--modversion {package_name}", force_run=True) + def package_info_exists( + self, package_name: str, pkg_config_path: Optional[str] = None + ) -> bool: + if pkg_config_path: + update_env = {"PKG_CONFIG_PATH": f"{pkg_config_path}"} + else: + update_env = None + package_info_result = self.run( + f"--modversion {package_name}", + force_run=True, + shell=True, + update_envs=update_env, + ) return package_info_result.exit_code == 0 def get_package_info( self, package_name: str, update_cached: bool = False, + pkg_config_path: Optional[str] = None, ) -> str: - info_exists = self.package_info_exists(package_name=package_name) + info_exists = self.package_info_exists( + package_name=package_name, pkg_config_path=pkg_config_path + ) + if pkg_config_path: + update_env = {"PKG_CONFIG_PATH": f"{pkg_config_path}"} + else: + update_env = None assert_that(info_exists).described_as( ( f"pkg-config information was not available for {package_name}. " @@ -40,10 +60,17 @@ def get_package_info( f"ensure .pc file is available for {package_name} on this OS." ) ).is_true() - return self.run(f"--modversion {package_name}").stdout + return self.run( + f"--modversion {package_name}", shell=True, update_envs=update_env + ).stdout def get_package_version( - self, package_name: str, update_cached: bool = False + self, + package_name: str, + update_cached: bool = False, + pkg_config_path: Optional[str] = None, ) -> VersionInfo: - version_info = self.get_package_info(package_name, update_cached=update_cached) + version_info = self.get_package_info( + package_name, update_cached=update_cached, pkg_config_path=pkg_config_path + ) return parse_version(version_info) From c9c6ce35945344205fdd8ac11eb138b0b802b532 Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Tue, 15 Oct 2024 10:55:52 -0700 Subject: [PATCH 02/12] Give rdmacore a common parent class for type enforcement --- microsoft/testsuites/dpdk/rdmacore.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/microsoft/testsuites/dpdk/rdmacore.py b/microsoft/testsuites/dpdk/rdmacore.py index 0a0d650e1b..956a03a70e 100644 --- a/microsoft/testsuites/dpdk/rdmacore.py +++ b/microsoft/testsuites/dpdk/rdmacore.py @@ -115,7 +115,11 @@ ) -class RdmaCorePackageManagerInstall(PackageManagerInstall): +class RdmaCoreInstaller(Installer): + ... + + +class RdmaCorePackageManagerInstall(RdmaCoreInstaller, PackageManagerInstall): def _setup_node(self) -> None: if isinstance(self._os, Fedora): self._os.install_epel() @@ -131,7 +135,7 @@ def _check_if_installed(self) -> bool: # implement SourceInstall for DPDK -class RdmaCoreSourceInstaller(Installer): +class RdmaCoreSourceInstaller(RdmaCoreInstaller): def _check_if_installed(self) -> bool: try: package_manager_install = self._os.package_exists("rdma-core") From 2b46a64a05720ce9aa7a84161db13c48fd05177e Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Tue, 15 Oct 2024 11:06:29 -0700 Subject: [PATCH 03/12] DpdkInstall: move constant args to private variables Allowing extension of the class for the 32bit build --- microsoft/testsuites/dpdk/dpdktestpmd.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 14aa38e936..128c51fd7f 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -199,6 +199,12 @@ class DpdkSourceInstall(Installer): "multi_process/client_server_mp/mp_server", "multi_process/client_server_mp/mp_client", ] + _library_bashrc_lines = [ + "export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/local/lib64/pkgconfig/", + "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib64/", + ] + _meson_arguments: List[str] = [] + _build_dir: str = "build" def _check_if_installed(self) -> bool: try: @@ -258,14 +264,15 @@ def get_installed_version(self) -> VersionInfo: def _install(self) -> None: super()._install() if self._sample_applications: - sample_apps = f"-Dexamples={','.join(self._sample_applications)}" - else: - sample_apps = "" + self._meson_arguments += f"-Dexamples={','.join(self._sample_applications)}" + node = self._node # save the pythonpath for later python_path = node.tools[Python].get_python_path() self.dpdk_build_path = node.tools[Meson].setup( - args=sample_apps, build_dir="build", cwd=self.asset_path + args=" ".join(self._meson_arguments), + build_dir=self._build_dir, + cwd=self.asset_path, ) node.tools[Ninja].run( cwd=self.dpdk_build_path, @@ -300,12 +307,8 @@ def _install(self) -> None: expected_exit_code=0, expected_exit_code_failure_message="ldconfig failed, check for error spew.", ) - library_bashrc_lines = [ - "export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/local/lib64/pkgconfig/", - "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib64/", - ] node.tools[Echo].write_to_file( - ";".join(library_bashrc_lines), + ";".join(self._library_bashrc_lines), node.get_pure_path("$HOME/.bashrc"), append=True, ) From 9570b1d46cbccf2ce5b67442ebcfe4af89e15b86 Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Tue, 15 Oct 2024 11:10:49 -0700 Subject: [PATCH 04/12] Dpdk: 32bit test Allow building rdma-core and dpdk as 32bit applications before test. Add the 32bit test. This is a beefy commit because we must extend the Installer class to support 32bit builds. This means adding invalid arch checks, adding arch checks to the DependencyInstaller, adding changes to PKG_CONFIG_PATH and allowing updating environment variables everywhere. The result is that we can install DPDK and RDMA core and run their 32bit versions to test basic send/receive stuff. --- lisa/tools/meson.py | 13 +- microsoft/testsuites/dpdk/common.py | 114 +++++++++++++---- microsoft/testsuites/dpdk/dpdksuite.py | 24 ++++ microsoft/testsuites/dpdk/dpdktestpmd.py | 156 ++++++++++++++++++----- microsoft/testsuites/dpdk/dpdkutil.py | 26 +++- microsoft/testsuites/dpdk/rdmacore.py | 98 +++++++++++--- 6 files changed, 353 insertions(+), 78 deletions(-) diff --git a/lisa/tools/meson.py b/lisa/tools/meson.py index 9ca4d37ee6..05691eb069 100644 --- a/lisa/tools/meson.py +++ b/lisa/tools/meson.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. from pathlib import PurePath -from typing import cast +from typing import Dict, Optional, cast from semver import VersionInfo @@ -50,9 +50,15 @@ def _install(self) -> bool: ) return self._check_exists() - def setup(self, args: str, cwd: PurePath, build_dir: str = "build") -> PurePath: + def setup( + self, + args: str, + cwd: PurePath, + build_dir: str = "build", + update_envs: Optional[Dict[str, str]] = None, + ) -> PurePath: self.run( - f"{args} {build_dir}", + parameters=f"{args} {build_dir}", force_run=True, shell=True, cwd=cwd, @@ -60,5 +66,6 @@ def setup(self, args: str, cwd: PurePath, build_dir: str = "build") -> PurePath: expected_exit_code_failure_message=( f"Could not configure {str(cwd)} with meson using args {args}" ), + update_envs=update_envs, ) return cwd.joinpath(build_dir) diff --git a/microsoft/testsuites/dpdk/common.py b/microsoft/testsuites/dpdk/common.py index b7c54bded1..4be3579545 100644 --- a/microsoft/testsuites/dpdk/common.py +++ b/microsoft/testsuites/dpdk/common.py @@ -11,8 +11,9 @@ from lisa import Node from lisa.executable import Tool from lisa.operating_system import Debian, Fedora, Oracle, Posix, Redhat, Suse, Ubuntu -from lisa.tools import Git, Tar, Wget -from lisa.util import UnsupportedDistroException +from lisa.tools import Git, Lscpu, Tar, Wget +from lisa.tools.lscpu import CpuArchitecture +from lisa.util import UnsupportedCpuArchitectureException, UnsupportedDistroException DPDK_STABLE_GIT_REPO = "https://dpdk.org/git/dpdk-stable" @@ -20,6 +21,12 @@ # signals 'route all traffic on this subnet' AZ_ROUTE_ALL_TRAFFIC = "0.0.0.0/0" +ARCH_COMPATIBILITY_MATRIX = { + CpuArchitecture.X64: [CpuArchitecture.X64], + CpuArchitecture.I386: [CpuArchitecture.I386, CpuArchitecture.X64], + CpuArchitecture.ARM64: [CpuArchitecture.ARM64], +} + # Attempt to clean up the DPDK package dependency mess # Make a Installer class that implements the common steps @@ -34,7 +41,7 @@ class OsPackageDependencies: # the packages to install on that OS. def __init__( self, - matcher: Callable[[Posix], bool], + matcher: Callable[[Posix, Optional[CpuArchitecture]], bool], packages: Optional[Sequence[Union[str, Tool, Type[Tool]]]] = None, stop_on_match: bool = False, ) -> None: @@ -45,14 +52,21 @@ def __init__( class DependencyInstaller: # provide a list of OsPackageDependencies for a project - def __init__(self, requirements: List[OsPackageDependencies]) -> None: + def __init__( + self, + requirements: List[OsPackageDependencies], + arch: Optional[CpuArchitecture] = None, + ) -> None: self.requirements = requirements + self._arch = arch # evaluate the list of package dependencies, def install_required_packages( - self, node: Node, extra_args: Union[List[str], None] + self, + os: Posix, + extra_args: Union[List[str], None], + arch: Optional[CpuArchitecture] = None, ) -> None: - os = node.os assert isinstance(os, Posix), ( "DependencyInstaller is not compatible with this OS: " f"{os.information.vendor} {os.information.release}" @@ -61,13 +75,16 @@ def install_required_packages( # stop on list end or if exclusive_match parameter is true. packages: List[Union[str, Tool, Type[Tool]]] = [] for requirement in self.requirements: - if requirement.matcher(os) and requirement.packages: - packages += requirement.packages + if requirement.matcher(os, arch): + if requirement.packages is not None and len(requirement.packages) > 0: + packages += requirement.packages if requirement.stop_on_match: break + os.install_packages(packages=packages, extra_args=extra_args) # NOTE: It is up to the caller to raise an exception on an invalid OS + # see unsupported_os_thrower as a catch-all 'list end' function class Downloader: @@ -200,23 +217,43 @@ def _uninstall(self) -> None: def _install_dependencies(self) -> None: if self._os_dependencies is not None: self._os_dependencies.install_required_packages( - self._node, extra_args=self._package_manager_extra_args + self._os, extra_args=self._package_manager_extra_args, arch=self._arch ) # define how to check the installed version def get_installed_version(self) -> VersionInfo: raise NotImplementedError(f"get_installed_version {self._err_msg}") - def _should_install(self, required_version: Optional[VersionInfo] = None) -> bool: - return (not self._check_if_installed()) or ( - required_version is not None - and required_version > self.get_installed_version() + def _should_install( + self, + required_version: Optional[VersionInfo] = None, + required_arch: Optional[CpuArchitecture] = None, + ) -> bool: + return ( + (not self._check_if_installed()) + # NOTE: Taking advantage of longer-than-expected lifetimes here. + # If the tool still exists we ~should~ be able to check the old version + # from a previous test environment. + # At the moment, we use create() to force re-initialization. + # If we ever fix things so that we use .get, + # we will need this check. So add it now.q + or (required_arch != self._arch) + or ( + required_version is not None + and required_version > self.get_installed_version() + ) ) # run the defined setup and installation steps. - def do_installation(self, required_version: Optional[VersionInfo] = None) -> None: + def do_installation( + self, + required_version: Optional[VersionInfo] = None, + required_arch: Optional[CpuArchitecture] = None, + ) -> None: self._setup_node() - if self._should_install(): + if self._should_install( + required_version=required_version, required_arch=required_arch + ): self._uninstall() self._install_dependencies() self._install() @@ -226,6 +263,7 @@ def __init__( node: Node, os_dependencies: Optional[DependencyInstaller] = None, downloader: Optional[Downloader] = None, + arch: Optional[CpuArchitecture] = None, ) -> None: self._node = node if not isinstance(self._node.os, Posix): @@ -236,11 +274,22 @@ def __init__( self._package_manager_extra_args: List[str] = [] self._os_dependencies = os_dependencies self._downloader = downloader + self._arch = arch + if self._arch: + # avoid building/running arm64 on i386, etc + system_arch = self._node.tools[Lscpu].get_architecture() + if system_arch not in ARCH_COMPATIBILITY_MATRIX[self._arch]: + raise UnsupportedCpuArchitectureException(system_arch) # Base class for package manager installation class PackageManagerInstall(Installer): - def __init__(self, node: Node, os_dependencies: DependencyInstaller) -> None: + def __init__( + self, + node: Node, + os_dependencies: DependencyInstaller, + arch: Optional[CpuArchitecture], + ) -> None: super().__init__(node, os_dependencies) # uninstall from the package manager @@ -248,8 +297,11 @@ def _uninstall(self) -> None: if not (isinstance(self._os, Posix) and self._check_if_installed()): return if self._os_dependencies is not None: - for os_package_check in self._os_dependencies.requirements: - if os_package_check.matcher(self._os) and os_package_check.packages: + for os_package_check in self._os_dependencies: + if ( + os_package_check.matcher(self._os, self._arch) + and os_package_check.packages + ): self._os.uninstall_packages(os_package_check.packages) if os_package_check.stop_on_match: break @@ -261,7 +313,7 @@ def _check_if_installed(self) -> bool: # For dpdk, pkg-manager install is only for 'dpdk' and 'dpdk-dev' # This will take too long if it's more than a few packages. if self._os_dependencies is not None: - for os_package_check in self._os_dependencies.requirements: + for os_package_check in self._os_dependencies: if os_package_check.matcher(self._os) and os_package_check.packages: for pkg in os_package_check.packages: if not self._os.package_exists(pkg): @@ -271,9 +323,13 @@ def _check_if_installed(self) -> bool: return True -def force_dpdk_default_source(variables: Dict[str, Any]) -> None: +def force_dpdk_default_source( + variables: Dict[str, Any], build_arch: Optional[CpuArchitecture] = None +) -> None: if not variables.get("dpdk_source", None): variables["dpdk_source"] = DPDK_STABLE_GIT_REPO + if build_arch: + variables["build_arch"] = build_arch _UBUNTU_LTS_VERSIONS = ["24.4.0", "22.4.0", "20.4.0", "18.4.0"] @@ -368,13 +424,27 @@ def is_url_for_git_repo(url: str) -> bool: return scheme == "git" or check_for_git_https -def unsupported_os_thrower(os: Posix) -> bool: +# utility matcher to throw an OS error type for a match. +def unsupported_os_thrower(os: Posix, arch: Optional[CpuArchitecture]) -> bool: + if arch: + message_suffix = f"OS and Architecture ({arch})" + else: + message_suffix = "OS" raise UnsupportedDistroException( os, - message=("Installer did not define dependencies for this os."), + message=f"Installer did not define dependencies for this {message_suffix}", ) +# utility matcher to throw an OS error when i386 is not supported +def i386_not_implemented_thrower(os: Posix, arch: Optional[CpuArchitecture]) -> bool: + if arch and arch == CpuArchitecture.I386: + raise NotImplementedError( + "i386 is not implemented for this (installer,OS) combo." + ) + return False + + def get_debian_backport_repo_args(os: Debian) -> List[str]: # ex: 'bionic-backports' or 'buster-backports' # these backport repos are available for the older OS's diff --git a/microsoft/testsuites/dpdk/dpdksuite.py b/microsoft/testsuites/dpdk/dpdksuite.py index 2b66047a14..26e834a988 100644 --- a/microsoft/testsuites/dpdk/dpdksuite.py +++ b/microsoft/testsuites/dpdk/dpdksuite.py @@ -25,6 +25,7 @@ from lisa.testsuite import TestResult, simple_requirement from lisa.tools import Echo, Git, Hugepages, Ip, Kill, Lsmod, Make, Modprobe from lisa.tools.hugepages import HugePageSize +from lisa.tools.lscpu import CpuArchitecture from lisa.util.constants import SIGINT from microsoft.testsuites.dpdk.common import ( DPDK_STABLE_GIT_REPO, @@ -101,6 +102,29 @@ def verify_dpdk_build_netvsc( node, log, variables, "netvsc", HugePageSize.HUGE_2MB, result=result ) + @TestCaseMetadata( + description=""" + netvsc pmd version. + This test case checks DPDK can be built and installed correctly. + Prerequisites, accelerated networking must be enabled. + The VM should have at least two network interfaces, + with one interface for management. + More details refer https://docs.microsoft.com/en-us/azure/virtual-network/setup-dpdk#prerequisites # noqa: E501 + """, + priority=2, + requirement=simple_requirement( + min_core_count=8, + min_nic_count=2, + network_interface=Sriov(), + unsupported_features=[Gpu, Infiniband], + ), + ) + def verify_dpdk_build_netvsc_32bit( + self, node: Node, log: Logger, variables: Dict[str, Any] + ) -> None: + force_dpdk_default_source(variables, build_arch=CpuArchitecture.I386) + verify_dpdk_build(node, log, variables, "netvsc", HugePageSize.HUGE_2MB) + @TestCaseMetadata( description=""" netvsc pmd version with 1GiB hugepages diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 128c51fd7f..23bbd25435 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -3,7 +3,7 @@ import re from pathlib import PurePath, PurePosixPath -from typing import Any, List, Tuple, Type +from typing import Any, Dict, List, Optional, Tuple, Type from assertpy import assert_that, fail from semver import VersionInfo @@ -28,6 +28,7 @@ Timeout, Wget, ) +from lisa.tools.lscpu import CpuArchitecture from lisa.util import ( LisaException, SkippedException, @@ -44,6 +45,7 @@ PackageManagerInstall, TarDownloader, get_debian_backport_repo_args, + i386_not_implemented_thrower, is_url_for_git_repo, is_url_for_tarball, unsupported_os_thrower, @@ -53,38 +55,40 @@ # declare package dependencies for package manager DPDK installation -DPDK_PACKAGE_MANAGER_PACKAGES = DependencyInstaller( +DPDK_PACKAGE_MANAGER_DEPENDENCIES = DependencyInstaller( requirements=[ + # raise exception early if package manager install is used for i386 test. + OsPackageDependencies(matcher=i386_not_implemented_thrower), # install linux-modules-extra-azure if it's available for mana_ib # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian) - and bool(x.get_kernel_information().version >= "5.15.0") - and x.is_package_in_repo("linux-modules-extra-azure"), + matcher=lambda os, arch=None: isinstance(os, Debian) + and bool(os.get_kernel_information().version >= "5.15.0") + and os.is_package_in_repo("linux-modules-extra-azure"), packages=["linux-modules-extra-azure"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian), + matcher=lambda os, arch=None: isinstance(os, Debian), packages=["dpdk", "dpdk-dev"], stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Suse) - and bool(parse_version(x.information.release) == "15.5.0"), + matcher=lambda os, arch=None: isinstance(os, Suse) + and bool(parse_version(os.information.release) == "15.5.0"), packages=["dpdk22", "dpdk22-devel"], stop_on_match=True, ), OsPackageDependencies( # alma/rocky have started # including testpmd by default in 'dpdk' - matcher=lambda x: isinstance(x, Fedora) - and not x.is_package_in_repo("dpdk-devel"), + matcher=lambda os, arch=None: isinstance(os, Fedora) + and not os.is_package_in_repo("dpdk-devel"), packages=["dpdk"], stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, (Fedora, Suse)), + matcher=lambda os, arch=None: isinstance(os, (Fedora, Suse)), packages=["dpdk", "dpdk-devel"], stop_on_match=True, ), @@ -92,11 +96,11 @@ ] ) # declare package/tool dependencies for DPDK source installation -DPDK_SOURCE_INSTALL_PACKAGES = DependencyInstaller( +DPDK_SOURCE_DEPENDENCIES = DependencyInstaller( requirements=[ OsPackageDependencies( - matcher=lambda x: isinstance(x, Ubuntu) - and x.information.codename == "bionic", + matcher=lambda os, arch=None: isinstance(os, Ubuntu) + and os.information.codename == "bionic", packages=[ "build-essential", "libmnl-dev", @@ -116,13 +120,29 @@ # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian) - and bool(x.get_kernel_information().version >= "5.15.0") - and x.is_package_in_repo("linux-modules-extra-azure"), + matcher=lambda os, arch=None: isinstance(os, Debian) + and bool(os.get_kernel_information().version >= "5.15.0") + and os.is_package_in_repo("linux-modules-extra-azure"), packages=["linux-modules-extra-azure"], ), + # Install 32-bit dependencies if we're building for i386 OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian), + matcher=lambda os, arch=None: isinstance(os, Debian) + and arch == CpuArchitecture.I386, + packages=[ + "python3-pyelftools", + "libelf-dev:i386", + "libnuma-dev:i386", + "pkg-config", + "python3-pip", + "cmake", + "libnl-3-dev:i386", + "meson", + "gcc-i686-linux-gnu", + ], + ), + OsPackageDependencies( + matcher=lambda os, arch=None: isinstance(os, Debian), packages=[ "build-essential", "libnuma-dev", @@ -134,7 +154,7 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Suse), + matcher=lambda os, arch=None: isinstance(os, Suse), packages=[ "psmisc", "libnuma-devel", @@ -145,7 +165,7 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, (Fedora)), + matcher=lambda os, arch=None: isinstance(os, (Fedora)), packages=[ "psmisc", "numactl-devel", @@ -194,17 +214,57 @@ def _check_if_installed(self) -> bool: # implement SourceInstall for DPDK class DpdkSourceInstall(Installer): + def _get_pkgconfig_path(self) -> str: + if self._arch == CpuArchitecture.I386: + return "${PKG_CONFIG_PATH}:/usr/local/lib/i386-linux-gnu/pkgconfig" + else: + return "${PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig" + + def _get_bashrc_defines(self) -> List[str]: + if self._arch == CpuArchitecture.I386: + arch_folder = "i386-linux-gnu" + else: + arch_folder = "lib64" + return [ + f"export PKG_CONFIG_PATH={self._get_pkgconfig_path()}", + "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib/" + + f"{arch_folder}", + ] + _sample_applications = [ "l3fwd", "multi_process/client_server_mp/mp_server", "multi_process/client_server_mp/mp_client", ] - _library_bashrc_lines = [ - "export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/local/lib64/pkgconfig/", - "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib64/", - ] - _meson_arguments: List[str] = [] + + def _get_meson_defines(self) -> Optional[Dict[str, str]]: + if self._arch == CpuArchitecture.I386: + return { + "CC": "/usr/bin/i686-linux-gnu-gcc", + "LDFLAGS": "-m32", + "PKG_CONFIG_LIBDIR": "/usr/local/lib/i386-linux-gnu/pkgconfig", + } + else: + return {"PATH": "$PATH:/usr/local/bin:/home/$USER/.local/bin"} + + def _get_c_arguments(self) -> str: + if self._arch == CpuArchitecture.I386: + return "-Dc_link_args=-m32" + else: + return "" + _build_dir: str = "build" + _enable_drivers: List[str] = [ + "*/mlx*", + "net/*netvsc", + "net/ring", + "net/virtio", + "net/bonding", + "bus/*", + "common/*", + "mempool/*", + ] + _enable_apps: List[str] = ["app/test-pmd"] def _check_if_installed(self) -> bool: try: @@ -224,6 +284,15 @@ def _check_if_installed(self) -> bool: def _setup_node(self) -> None: super()._setup_node() if isinstance(self._os, Debian): + if self._arch == CpuArchitecture.I386: + self._node.execute( + "dpkg --add-architecture i386", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "Could not enable i386 packages." + ), + ) self._package_manager_extra_args = get_debian_backport_repo_args(self._os) if isinstance(self._os, Ubuntu) and self._os.information.version < "22.4.0": self._os.update_packages("linux-azure") @@ -258,22 +327,41 @@ def _uninstall(self) -> None: def get_installed_version(self) -> VersionInfo: return self._node.tools[Pkgconfig].get_package_version( - "libdpdk", update_cached=True + "libdpdk", update_cached=True, pkg_config_path=self._get_pkgconfig_path() ) + def _get_meson_parameters(self) -> str: + # enable any apps we need (namely, testpmd) + enable_apps = "-Denable_apps=" + ",".join(self._enable_apps) + # add net/mana to the pmd list if we need it + if self._node.nics.is_mana_device_present(): + self._enable_drivers += ["net/mana"] + # build the driver enable arg + enable_drivers = "-Denable_drivers=" + ",".join(self._enable_drivers) + # add any needed -Dc_flags or -Dc_link_args arguments + c_args = self._get_c_arguments() + return " ".join([c_args, enable_apps, enable_drivers]) + def _install(self) -> None: super()._install() if self._sample_applications: - self._meson_arguments += f"-Dexamples={','.join(self._sample_applications)}" + self._meson_arguments = f"-Dexamples={','.join(self._sample_applications)}" node = self._node # save the pythonpath for later python_path = node.tools[Python].get_python_path() + + # handle arch special cases for env vars + update_envs = self._get_meson_defines() + # run [DEFINE='...'] meson setup ... $build_dir + # configures the installation self.dpdk_build_path = node.tools[Meson].setup( - args=" ".join(self._meson_arguments), - build_dir=self._build_dir, + args=self._get_meson_parameters(), cwd=self.asset_path, + build_dir=self._build_dir, + update_envs=update_envs, ) + # run the build node.tools[Ninja].run( cwd=self.dpdk_build_path, shell=True, @@ -307,8 +395,9 @@ def _install(self) -> None: expected_exit_code=0, expected_exit_code_failure_message="ldconfig failed, check for error spew.", ) + # add the define lines we need node.tools[Echo].write_to_file( - ";".join(self._library_bashrc_lines), + ";".join(self._get_bashrc_defines()), node.get_pure_path("$HOME/.bashrc"), append=True, ) @@ -705,13 +794,14 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self._dpdk_source = kwargs.pop("dpdk_source", PACKAGE_MANAGER_SOURCE) self._dpdk_branch = kwargs.pop("dpdk_branch", "main") self._sample_apps_to_build = kwargs.pop("sample_apps", []) + self._build_arch: Optional[CpuArchitecture] = kwargs.pop("build_arch", None) self._dpdk_version_info = VersionInfo(0, 0) self._testpmd_install_path: str = "" self._expected_install_path = "" self._determine_network_hardware() if self.use_package_manager_install(): self.installer: Installer = DpdkPackageManagerInstall( - self.node, DPDK_PACKAGE_MANAGER_PACKAGES + self.node, DPDK_PACKAGE_MANAGER_DEPENDENCIES, arch=self._build_arch ) # if not package manager, choose source installation else: @@ -739,10 +829,12 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: " Expected https://___/___.git or /path/to/tar.tar[.gz] or " "https://__/__.tar[.gz]" ) + self.installer = DpdkSourceInstall( node=self.node, - os_dependencies=DPDK_SOURCE_INSTALL_PACKAGES, + os_dependencies=DPDK_SOURCE_DEPENDENCIES, downloader=downloader, + arch=self._build_arch, ) # if dpdk is already installed, find the binary and check the version if self.find_testpmd_binary(assert_on_fail=False): diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index 71753a730b..69e3dd07c1 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -45,6 +45,7 @@ Timeout, ) from lisa.tools.hugepages import HugePageSize +from lisa.tools.lscpu import CpuArchitecture from lisa.util.constants import DEVICE_TYPE_SRIOV, SIGINT from lisa.util.parallel import TaskManager, run_in_parallel, run_in_parallel_async from microsoft.testsuites.dpdk.common import ( @@ -62,7 +63,7 @@ from microsoft.testsuites.dpdk.dpdktestpmd import PACKAGE_MANAGER_SOURCE, DpdkTestpmd from microsoft.testsuites.dpdk.rdmacore import ( RDMA_CORE_MANA_DEFAULT_SOURCE, - RDMA_CORE_PACKAGE_MANAGER_DEPENDENCIES, + RDMA_CORE_PACKAGE_DEPENDENCIES, RDMA_CORE_SOURCE_DEPENDENCIES, RdmaCorePackageManagerInstall, RdmaCoreSourceInstaller, @@ -128,14 +129,21 @@ def _set_forced_source_by_distro(node: Node, variables: Dict[str, Any]) -> None: def get_rdma_core_installer( - node: Node, dpdk_source: str, dpdk_branch: str, rdma_source: str, rdma_branch: str + node: Node, + rdma_source: str, + rdma_branch: str, + build_arch: Optional[CpuArchitecture] = None, ) -> Installer: # set rdma-core installer type. + if not build_arch: + build_arch = node.tools[Lscpu].get_architecture() + if build_arch == CpuArchitecture.I386 and not rdma_source: + rdma_source = RDMA_CORE_MANA_DEFAULT_SOURCE if rdma_source: if is_url_for_git_repo(rdma_source): # else, if we have a user provided rdma-core source, use it downloader: Downloader = GitDownloader(node, rdma_source, rdma_branch) - elif is_url_for_tarball(rdma_branch): + elif is_url_for_tarball(rdma_source): downloader = TarDownloader(node, rdma_source) else: # throw on unrecognized rdma core source type @@ -148,11 +156,15 @@ def get_rdma_core_installer( else: # no rdma_source and not mana, just use the package manager return RdmaCorePackageManagerInstall( - node, os_dependencies=RDMA_CORE_PACKAGE_MANAGER_DEPENDENCIES + node, os_dependencies=RDMA_CORE_PACKAGE_DEPENDENCIES, arch=build_arch ) + # return the installer with the downloader we've picked return RdmaCoreSourceInstaller( - node, os_dependencies=RDMA_CORE_SOURCE_DEPENDENCIES, downloader=downloader + node, + os_dependencies=RDMA_CORE_SOURCE_DEPENDENCIES, + downloader=downloader, + arch=build_arch, ) @@ -300,6 +312,7 @@ def initialize_node_resources( "Dpdk initialize_node_resources running" f"found dpdk_source '{dpdk_source}' and dpdk_branch '{dpdk_branch}'" ) + build_arch = variables.get("build_arch", None) network_interface_feature = node.features[NetworkInterface] sriov_is_enabled = network_interface_feature.is_enabled_sriov() if not sriov_is_enabled: @@ -325,7 +338,7 @@ def initialize_node_resources( node.nics.check_pci_enabled(pci_enabled=True) update_kernel_from_repo(node) rdma_core = get_rdma_core_installer( - node, dpdk_source, dpdk_branch, rdma_source, rdma_branch + node, rdma_source, rdma_branch, build_arch=build_arch ) rdma_core.do_installation() # create tool, initialize testpmd tool (installs dpdk) @@ -337,6 +350,7 @@ def initialize_node_resources( dpdk_branch=dpdk_branch, sample_apps=sample_apps, force_net_failsafe_pmd=force_net_failsafe_pmd, + build_arch=build_arch, ) # Tools will skip installation if the binary is present, so # force invoke install. Installer will skip if the correct diff --git a/microsoft/testsuites/dpdk/rdmacore.py b/microsoft/testsuites/dpdk/rdmacore.py index 956a03a70e..fa29d7cce5 100644 --- a/microsoft/testsuites/dpdk/rdmacore.py +++ b/microsoft/testsuites/dpdk/rdmacore.py @@ -1,8 +1,10 @@ from assertpy import assert_that from semver import VersionInfo +from lisa import UnsupportedCpuArchitectureException from lisa.operating_system import Debian, Fedora, Suse from lisa.tools import Make, Pkgconfig +from lisa.tools.lscpu import CpuArchitecture from microsoft.testsuites.dpdk.common import ( DependencyInstaller, Installer, @@ -19,16 +21,48 @@ RDMA_CORE_SOURCE_DEPENDENCIES = DependencyInstaller( [ OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian) + matcher=lambda os, _arch=None: isinstance(os, Debian) # install linux-modules-extra-azure if it's available for mana_ib # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. - and bool(x.get_kernel_information().version >= "5.15.0") - and x.is_package_in_repo("linux-modules-extra-azure"), + and bool(os.get_kernel_information().version >= "5.15.0") + and os.is_package_in_repo("linux-modules-extra-azure"), packages=["linux-modules-extra-azure"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian), + matcher=lambda os, arch=None: isinstance(os, (Debian)) + and arch == CpuArchitecture.I386, + packages=[ + "python3-pyelftools", + "libelf-dev:i386", + "libnuma-dev:i386", + "pkg-config", + "python3-pip", + "cmake", + "libnl-3-dev:i386", + "libnl-route-3-dev:i386", + "meson", + "gcc-i686-linux-gnu", + "python3-dev:i386", + "libudev-dev:i386", + "libudev-dev", + "libnl-3-dev", + "libnl-route-3-dev", + "libssl-dev", + "libelf-dev", + ], + ), + OsPackageDependencies( + matcher=lambda os, arch=None: isinstance(os, (Debian)) + and arch == CpuArchitecture.I386, + # Weirdly, I've run into errors trying to + packages=[ + "cython3:i386", + ], + stop_on_match=True, + ), + OsPackageDependencies( + matcher=lambda os, _arch=None: isinstance(os, Debian), packages=[ "cmake", "libudev-dev", @@ -49,7 +83,7 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Fedora), + matcher=lambda os, _arch=None: isinstance(os, Fedora), packages=[ "cmake", "libudev-devel", @@ -82,31 +116,32 @@ ] ) -RDMA_CORE_PACKAGE_MANAGER_DEPENDENCIES = DependencyInstaller( + +RDMA_CORE_PACKAGE_DEPENDENCIES = DependencyInstaller( [ OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian) + matcher=lambda os, _=None: isinstance(os, Debian) # install linux-modules-extra-azure if it's available for mana_ib # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. - and bool(x.get_kernel_information().version >= "5.15.0") - and x.is_package_in_repo("linux-modules-extra-azure"), + and bool(os.get_kernel_information().version >= "5.15.0") + and os.is_package_in_repo("linux-modules-extra-azure"), packages=["linux-modules-extra-azure"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian), + matcher=lambda os, _=None: isinstance(os, Debian), packages=["ibverbs-providers", "libibverbs-dev"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Suse), + matcher=lambda os, _=None: isinstance(os, Suse), packages=["rdma-core-devel", "librdmacm1"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Fedora), + matcher=lambda os, _=None: isinstance(os, Fedora), packages=["librdmacm-devel"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, (Fedora, Debian, Suse)), + matcher=lambda os, _=None: isinstance(os, (Fedora, Debian, Suse)), packages=["rdma-core"], stop_on_match=True, ), @@ -115,6 +150,7 @@ ) +# Common parent for isinstance matching class RdmaCoreInstaller(Installer): ... @@ -152,10 +188,42 @@ def _check_if_installed(self) -> bool: return False def _setup_node(self) -> None: + self._pkg_config_path = None if isinstance(self._os, (Debian, Fedora, Suse)): self._os.uninstall_packages("rdma-core") if isinstance(self._os, Fedora): self._os.group_install_packages("Development Tools") + if not self._arch or self._arch in [CpuArchitecture.ARM64, CpuArchitecture.X64]: + self._cmake_command = ( + "cmake -DIN_PLACE=0 -DNO_MAN_PAGES=1 -DCMAKE_INSTALL_PREFIX=/usr" + ) + # Only support this 32bit build on one distro family. + elif isinstance(self._os, Debian) and self._arch == CpuArchitecture.I386: + self._pkg_config_path = "/usr/local/lib/i386-linux-gnu/pkgconfig" + # enable 32bit packages, needed for dependencies + self._node.execute( + "dpkg --add-architecture i386", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message="Could not enable i386 packages.", + ) + self._node.execute( + "apt update", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "Apt update after enabling i386 failed" + ), + ) + self._cmake_command = ( + "cmake " + "-DIN_PLACE=0 -DNO_MAN_PAGES=1 -DCMAKE_INSTALL_PREFIX=/usr " + "-DCMAKE_C_COMPILER=/usr/bin/i686-linux-gnu-gcc -DCMAKE_C_FLAGS=-m32" + ) + else: + # Will hit this when adding a new CPU Architecture, + # so hello to whoever is adding RISC-V :) + raise UnsupportedCpuArchitectureException(arch=self._arch) super()._setup_node() def _uninstall(self) -> None: @@ -182,7 +250,7 @@ def _uninstall(self) -> None: def get_installed_version(self) -> VersionInfo: return self._node.tools[Pkgconfig].get_package_version( - "libibverbs", update_cached=True + "libibverbs", update_cached=True, pkg_config_path=self._pkg_config_path ) def _install(self) -> None: @@ -190,7 +258,7 @@ def _install(self) -> None: node = self._node make = node.tools[Make] node.execute( - "cmake -DIN_PLACE=0 -DNO_MAN_PAGES=1 -DCMAKE_INSTALL_PREFIX=/usr", + self._cmake_command, shell=True, cwd=self.asset_path, sudo=True, From a2f9526e6374f49963e50f6bb0a26bd250f8b1f1 Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Mon, 28 Oct 2024 16:03:57 -0700 Subject: [PATCH 05/12] DPDK: Add 32bit multiqueue test --- microsoft/testsuites/dpdk/dpdksuite.py | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/microsoft/testsuites/dpdk/dpdksuite.py b/microsoft/testsuites/dpdk/dpdksuite.py index 26e834a988..c8ad3f9c86 100644 --- a/microsoft/testsuites/dpdk/dpdksuite.py +++ b/microsoft/testsuites/dpdk/dpdksuite.py @@ -125,6 +125,40 @@ def verify_dpdk_build_netvsc_32bit( force_dpdk_default_source(variables, build_arch=CpuArchitecture.I386) verify_dpdk_build(node, log, variables, "netvsc", HugePageSize.HUGE_2MB) + @TestCaseMetadata( + description=""" + netvsc pmd version. + This test case checks DPDK can be built and installed correctly. + Prerequisites, accelerated networking must be enabled. + The VM should have at least two network interfaces, + with one interface for management. + More details refer https://docs.microsoft.com/en-us/azure/virtual-network/setup-dpdk#prerequisites # noqa: E501 + """, + priority=2, + requirement=simple_requirement( + min_count=2, + min_core_count=8, + min_nic_count=2, + network_interface=Sriov(), + unsupported_features=[Gpu, Infiniband], + ), + ) + def verify_dpdk_send_receive_netvsc_32bit( + self, + environment: Environment, + log: Logger, + variables: Dict[str, Any], + ) -> None: + force_dpdk_default_source(variables, build_arch=CpuArchitecture.I386) + verify_dpdk_send_receive( + environment, + log, + variables, + "netvsc", + HugePageSize.HUGE_2MB, + multiple_queues=True, + ) + @TestCaseMetadata( description=""" netvsc pmd version with 1GiB hugepages From e13a5dfd08eaab8e9e453b1fda7e11267cf0660e Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Tue, 29 Oct 2024 11:53:06 -0700 Subject: [PATCH 06/12] # type: ignore to the 'rescue' --- microsoft/testsuites/dpdk/common.py | 9 ++++++--- microsoft/testsuites/dpdk/dpdktestpmd.py | 24 +++++++++++++----------- microsoft/testsuites/dpdk/rdmacore.py | 22 ++++++++++++---------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/microsoft/testsuites/dpdk/common.py b/microsoft/testsuites/dpdk/common.py index 4be3579545..de7b1192c0 100644 --- a/microsoft/testsuites/dpdk/common.py +++ b/microsoft/testsuites/dpdk/common.py @@ -297,7 +297,7 @@ def _uninstall(self) -> None: if not (isinstance(self._os, Posix) and self._check_if_installed()): return if self._os_dependencies is not None: - for os_package_check in self._os_dependencies: + for os_package_check in self._os_dependencies.requirements: if ( os_package_check.matcher(self._os, self._arch) and os_package_check.packages @@ -313,8 +313,11 @@ def _check_if_installed(self) -> bool: # For dpdk, pkg-manager install is only for 'dpdk' and 'dpdk-dev' # This will take too long if it's more than a few packages. if self._os_dependencies is not None: - for os_package_check in self._os_dependencies: - if os_package_check.matcher(self._os) and os_package_check.packages: + for os_package_check in self._os_dependencies.requirements: + if ( + os_package_check.matcher(self._os, self._arch) + and os_package_check.packages + ): for pkg in os_package_check.packages: if not self._os.package_exists(pkg): return False diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 23bbd25435..160ef050a0 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -63,18 +63,18 @@ # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, Debian) + matcher=lambda os, arch=None: isinstance(os, Debian) # type: ignore and bool(os.get_kernel_information().version >= "5.15.0") and os.is_package_in_repo("linux-modules-extra-azure"), packages=["linux-modules-extra-azure"], ), OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, Debian), + matcher=lambda os, arch=None: isinstance(os, Debian), # type: ignore packages=["dpdk", "dpdk-dev"], stop_on_match=True, ), OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, Suse) + matcher=lambda os, arch=None: isinstance(os, Suse) # type: ignore and bool(parse_version(os.information.release) == "15.5.0"), packages=["dpdk22", "dpdk22-devel"], stop_on_match=True, @@ -82,13 +82,15 @@ OsPackageDependencies( # alma/rocky have started # including testpmd by default in 'dpdk' - matcher=lambda os, arch=None: isinstance(os, Fedora) + matcher=lambda os, arch=None: isinstance(os, Fedora) # type: ignore and not os.is_package_in_repo("dpdk-devel"), packages=["dpdk"], stop_on_match=True, ), OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, (Fedora, Suse)), + matcher=lambda os, arch=None: isinstance( # type: ignore + os, (Fedora, Suse) + ), packages=["dpdk", "dpdk-devel"], stop_on_match=True, ), @@ -99,7 +101,7 @@ DPDK_SOURCE_DEPENDENCIES = DependencyInstaller( requirements=[ OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, Ubuntu) + matcher=lambda os, arch=None: isinstance(os, Ubuntu) # type: ignore and os.information.codename == "bionic", packages=[ "build-essential", @@ -120,14 +122,14 @@ # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, Debian) + matcher=lambda os, arch=None: isinstance(os, Debian) # type: ignore and bool(os.get_kernel_information().version >= "5.15.0") and os.is_package_in_repo("linux-modules-extra-azure"), packages=["linux-modules-extra-azure"], ), # Install 32-bit dependencies if we're building for i386 OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, Debian) + matcher=lambda os, arch=None: isinstance(os, Debian) # type: ignore and arch == CpuArchitecture.I386, packages=[ "python3-pyelftools", @@ -142,7 +144,7 @@ ], ), OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, Debian), + matcher=lambda os, arch=None: isinstance(os, Debian), # type: ignore packages=[ "build-essential", "libnuma-dev", @@ -154,7 +156,7 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, Suse), + matcher=lambda os, arch=None: isinstance(os, Suse), # type: ignore packages=[ "psmisc", "libnuma-devel", @@ -165,7 +167,7 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, (Fedora)), + matcher=lambda os, arch=None: isinstance(os, (Fedora)), # type: ignore packages=[ "psmisc", "numactl-devel", diff --git a/microsoft/testsuites/dpdk/rdmacore.py b/microsoft/testsuites/dpdk/rdmacore.py index fa29d7cce5..1fd572025c 100644 --- a/microsoft/testsuites/dpdk/rdmacore.py +++ b/microsoft/testsuites/dpdk/rdmacore.py @@ -21,7 +21,7 @@ RDMA_CORE_SOURCE_DEPENDENCIES = DependencyInstaller( [ OsPackageDependencies( - matcher=lambda os, _arch=None: isinstance(os, Debian) + matcher=lambda os, _arch=None: isinstance(os, Debian) # type: ignore # install linux-modules-extra-azure if it's available for mana_ib # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. @@ -30,7 +30,7 @@ packages=["linux-modules-extra-azure"], ), OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, (Debian)) + matcher=lambda os, arch=None: isinstance(os, (Debian)) # type: ignore and arch == CpuArchitecture.I386, packages=[ "python3-pyelftools", @@ -53,7 +53,7 @@ ], ), OsPackageDependencies( - matcher=lambda os, arch=None: isinstance(os, (Debian)) + matcher=lambda os, arch=None: isinstance(os, (Debian)) # type: ignore and arch == CpuArchitecture.I386, # Weirdly, I've run into errors trying to packages=[ @@ -62,7 +62,7 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda os, _arch=None: isinstance(os, Debian), + matcher=lambda os, _arch=None: isinstance(os, Debian), # type: ignore packages=[ "cmake", "libudev-dev", @@ -83,7 +83,7 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda os, _arch=None: isinstance(os, Fedora), + matcher=lambda os, _arch=None: isinstance(os, Fedora), # type: ignore packages=[ "cmake", "libudev-devel", @@ -120,7 +120,7 @@ RDMA_CORE_PACKAGE_DEPENDENCIES = DependencyInstaller( [ OsPackageDependencies( - matcher=lambda os, _=None: isinstance(os, Debian) + matcher=lambda os, _=None: isinstance(os, Debian) # type: ignore # install linux-modules-extra-azure if it's available for mana_ib # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. @@ -129,19 +129,21 @@ packages=["linux-modules-extra-azure"], ), OsPackageDependencies( - matcher=lambda os, _=None: isinstance(os, Debian), + matcher=lambda os, _=None: isinstance(os, Debian), # type: ignore packages=["ibverbs-providers", "libibverbs-dev"], ), OsPackageDependencies( - matcher=lambda os, _=None: isinstance(os, Suse), + matcher=lambda os, _=None: isinstance(os, Suse), # type: ignore packages=["rdma-core-devel", "librdmacm1"], ), OsPackageDependencies( - matcher=lambda os, _=None: isinstance(os, Fedora), + matcher=lambda os, _=None: isinstance(os, Fedora), # type: ignore packages=["librdmacm-devel"], ), OsPackageDependencies( - matcher=lambda os, _=None: isinstance(os, (Fedora, Debian, Suse)), + matcher=lambda os, _=None: isinstance( # type: ignore + os, (Fedora, Debian, Suse) + ), packages=["rdma-core"], stop_on_match=True, ), From db1f94407f59d96bbcc2353e4eb58bea3ef68e79 Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Tue, 29 Oct 2024 12:22:19 -0700 Subject: [PATCH 07/12] DPDK: skip 32bit test on unsupported distros --- microsoft/testsuites/dpdk/dpdksuite.py | 4 ++++ microsoft/testsuites/dpdk/dpdkutil.py | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/microsoft/testsuites/dpdk/dpdksuite.py b/microsoft/testsuites/dpdk/dpdksuite.py index c8ad3f9c86..dba76cfb9b 100644 --- a/microsoft/testsuites/dpdk/dpdksuite.py +++ b/microsoft/testsuites/dpdk/dpdksuite.py @@ -44,6 +44,7 @@ init_nodes_concurrent, initialize_node_resources, run_testpmd_concurrent, + skip_32bit_test_on_unsupported_distros, verify_dpdk_build, verify_dpdk_l3fwd_ntttcp_tcp, verify_dpdk_send_receive, @@ -122,6 +123,7 @@ def verify_dpdk_build_netvsc( def verify_dpdk_build_netvsc_32bit( self, node: Node, log: Logger, variables: Dict[str, Any] ) -> None: + skip_32bit_test_on_unsupported_distros(node.os) force_dpdk_default_source(variables, build_arch=CpuArchitecture.I386) verify_dpdk_build(node, log, variables, "netvsc", HugePageSize.HUGE_2MB) @@ -149,6 +151,8 @@ def verify_dpdk_send_receive_netvsc_32bit( log: Logger, variables: Dict[str, Any], ) -> None: + node = environment.default_node + skip_32bit_test_on_unsupported_distros(node.os) force_dpdk_default_source(variables, build_arch=CpuArchitecture.I386) verify_dpdk_send_receive( environment, diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index 69e3dd07c1..d0dbabc66e 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -1188,3 +1188,16 @@ def annotate_dpdk_test_result( log.debug(f"Adding nic version: {nic_hw}") except AssertionError as err: test_kit.node.log.debug(f"Could not fetch NIC short name: {str(err)}") + + +def skip_32bit_test_on_unsupported_distros(os: OperatingSystem) -> None: + if not ( + isinstance(os, Ubuntu) + and os.information.version >= "22.4.0" + and os.information.version < "22.5.0" + ): + raise SkippedException( + UnsupportedDistroException( + os, "32bit test is only implemented for Ubuntu 22.04" + ) + ) From 6d052d5f3fc29fa8c205018f3daf47409a725a69 Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Tue, 29 Oct 2024 12:32:20 -0700 Subject: [PATCH 08/12] DPDK+32bit: define default sources for 32bit test --- microsoft/testsuites/dpdk/common.py | 11 ++++++++--- microsoft/testsuites/dpdk/dpdkutil.py | 3 ++- microsoft/testsuites/dpdk/rdmacore.py | 4 ++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/microsoft/testsuites/dpdk/common.py b/microsoft/testsuites/dpdk/common.py index de7b1192c0..7676e06c15 100644 --- a/microsoft/testsuites/dpdk/common.py +++ b/microsoft/testsuites/dpdk/common.py @@ -16,7 +16,7 @@ from lisa.util import UnsupportedCpuArchitectureException, UnsupportedDistroException DPDK_STABLE_GIT_REPO = "https://dpdk.org/git/dpdk-stable" - +DPDK_32BIT_DEFAULT_BRANCH = "v24.11-rc1" # azure routing table magic subnet prefix # signals 'route all traffic on this subnet' AZ_ROUTE_ALL_TRAFFIC = "0.0.0.0/0" @@ -329,11 +329,16 @@ def _check_if_installed(self) -> bool: def force_dpdk_default_source( variables: Dict[str, Any], build_arch: Optional[CpuArchitecture] = None ) -> None: - if not variables.get("dpdk_source", None): - variables["dpdk_source"] = DPDK_STABLE_GIT_REPO if build_arch: variables["build_arch"] = build_arch + if build_arch == CpuArchitecture.I386 and not variables.get("dpdk_branch", None): + # assign a default branch with needed MANA commits for 32bit test + variables["dpdk_branch"] = DPDK_32BIT_DEFAULT_BRANCH + + if not variables.get("dpdk_source", None): + variables["dpdk_source"] = DPDK_STABLE_GIT_REPO + _UBUNTU_LTS_VERSIONS = ["24.4.0", "22.4.0", "20.4.0", "18.4.0"] diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index d0dbabc66e..06b57acd84 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -62,6 +62,7 @@ ) from microsoft.testsuites.dpdk.dpdktestpmd import PACKAGE_MANAGER_SOURCE, DpdkTestpmd from microsoft.testsuites.dpdk.rdmacore import ( + RDMA_CORE_I386_DEFAULT_SOURCE, RDMA_CORE_MANA_DEFAULT_SOURCE, RDMA_CORE_PACKAGE_DEPENDENCIES, RDMA_CORE_SOURCE_DEPENDENCIES, @@ -138,7 +139,7 @@ def get_rdma_core_installer( if not build_arch: build_arch = node.tools[Lscpu].get_architecture() if build_arch == CpuArchitecture.I386 and not rdma_source: - rdma_source = RDMA_CORE_MANA_DEFAULT_SOURCE + rdma_source = RDMA_CORE_I386_DEFAULT_SOURCE if rdma_source: if is_url_for_git_repo(rdma_source): # else, if we have a user provided rdma-core source, use it diff --git a/microsoft/testsuites/dpdk/rdmacore.py b/microsoft/testsuites/dpdk/rdmacore.py index 1fd572025c..54635fc2ba 100644 --- a/microsoft/testsuites/dpdk/rdmacore.py +++ b/microsoft/testsuites/dpdk/rdmacore.py @@ -18,6 +18,10 @@ "https://github.com/linux-rdma/rdma-core/" "releases/download/v50.1/rdma-core-50.1.tar.gz" ) +RDMA_CORE_I386_DEFAULT_SOURCE = ( + "https://github.com/linux-rdma/rdma-core/" + "releases/download/v53.1/rdma-core-53.1.tar.gz" +) RDMA_CORE_SOURCE_DEPENDENCIES = DependencyInstaller( [ OsPackageDependencies( From c4f52f83285157f255514acd8b733c3f02591713 Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Tue, 29 Oct 2024 13:04:02 -0700 Subject: [PATCH 09/12] Add default source for rc release --- microsoft/testsuites/dpdk/common.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/microsoft/testsuites/dpdk/common.py b/microsoft/testsuites/dpdk/common.py index 7676e06c15..7fd4857fb2 100644 --- a/microsoft/testsuites/dpdk/common.py +++ b/microsoft/testsuites/dpdk/common.py @@ -16,7 +16,11 @@ from lisa.util import UnsupportedCpuArchitectureException, UnsupportedDistroException DPDK_STABLE_GIT_REPO = "https://dpdk.org/git/dpdk-stable" +# 32bit test relies on newer versions of DPDK. +# Release candidates are not in stable, so use the github mirror. DPDK_32BIT_DEFAULT_BRANCH = "v24.11-rc1" +# TODO: update these when v24.11 is released to stable. +DPDK_32BIT_DEFAULT_SOURCE = "https://github.com/DPDK/dpdk.git" # azure routing table magic subnet prefix # signals 'route all traffic on this subnet' AZ_ROUTE_ALL_TRAFFIC = "0.0.0.0/0" @@ -332,9 +336,12 @@ def force_dpdk_default_source( if build_arch: variables["build_arch"] = build_arch - if build_arch == CpuArchitecture.I386 and not variables.get("dpdk_branch", None): - # assign a default branch with needed MANA commits for 32bit test - variables["dpdk_branch"] = DPDK_32BIT_DEFAULT_BRANCH + if build_arch == CpuArchitecture.I386: + if not variables.get("dpdk_branch", None): + # assign a default branch with needed MANA commits for 32bit test + variables["dpdk_branch"] = DPDK_32BIT_DEFAULT_BRANCH + if not variables.get("dpdk_source", None): + variables["dpdk_source"] = DPDK_32BIT_DEFAULT_SOURCE if not variables.get("dpdk_source", None): variables["dpdk_source"] = DPDK_STABLE_GIT_REPO From 6572cb6dbe23f9804d39e399bfa928bcc0684a65 Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Tue, 29 Oct 2024 15:24:38 -0700 Subject: [PATCH 10/12] Fix version and installation criteria in testpmd --- microsoft/testsuites/dpdk/common.py | 2 +- microsoft/testsuites/dpdk/dpdktestpmd.py | 2 +- microsoft/testsuites/dpdk/dpdkutil.py | 10 ++++++---- microsoft/testsuites/dpdk/rdmacore.py | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/microsoft/testsuites/dpdk/common.py b/microsoft/testsuites/dpdk/common.py index 7fd4857fb2..94a6846ee9 100644 --- a/microsoft/testsuites/dpdk/common.py +++ b/microsoft/testsuites/dpdk/common.py @@ -241,7 +241,7 @@ def _should_install( # At the moment, we use create() to force re-initialization. # If we ever fix things so that we use .get, # we will need this check. So add it now.q - or (required_arch != self._arch) + or (self._arch and required_arch != self._arch) or ( required_version is not None and required_version > self.get_installed_version() diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 160ef050a0..1554a81b56 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -898,7 +898,7 @@ def _install(self) -> bool: ): raise SkippedException("MANA DPDK test is not supported on this OS") - self.installer.do_installation() + self.installer.do_installation(required_arch=self._build_arch) self._dpdk_version_info = self.installer.get_installed_version() self._load_drivers_for_dpdk() self.find_testpmd_binary(check_path=self._expected_install_path) diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index 06b57acd84..21f1810889 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -62,7 +62,7 @@ ) from microsoft.testsuites.dpdk.dpdktestpmd import PACKAGE_MANAGER_SOURCE, DpdkTestpmd from microsoft.testsuites.dpdk.rdmacore import ( - RDMA_CORE_I386_DEFAULT_SOURCE, + RDMA_CORE_I386_MANA_DEFAULT_SOURCE, RDMA_CORE_MANA_DEFAULT_SOURCE, RDMA_CORE_PACKAGE_DEPENDENCIES, RDMA_CORE_SOURCE_DEPENDENCIES, @@ -138,8 +138,7 @@ def get_rdma_core_installer( # set rdma-core installer type. if not build_arch: build_arch = node.tools[Lscpu].get_architecture() - if build_arch == CpuArchitecture.I386 and not rdma_source: - rdma_source = RDMA_CORE_I386_DEFAULT_SOURCE + if rdma_source: if is_url_for_git_repo(rdma_source): # else, if we have a user provided rdma-core source, use it @@ -153,7 +152,10 @@ def get_rdma_core_installer( ) # handle MANA special case, build a default rdma-core with mana provider elif not rdma_source and node.nics.is_mana_device_present(): - downloader = TarDownloader(node, RDMA_CORE_MANA_DEFAULT_SOURCE) + if build_arch == CpuArchitecture.I386: + downloader = TarDownloader(node, RDMA_CORE_I386_MANA_DEFAULT_SOURCE) + else: + downloader = TarDownloader(node, RDMA_CORE_MANA_DEFAULT_SOURCE) else: # no rdma_source and not mana, just use the package manager return RdmaCorePackageManagerInstall( diff --git a/microsoft/testsuites/dpdk/rdmacore.py b/microsoft/testsuites/dpdk/rdmacore.py index 54635fc2ba..5c99ab1308 100644 --- a/microsoft/testsuites/dpdk/rdmacore.py +++ b/microsoft/testsuites/dpdk/rdmacore.py @@ -18,7 +18,7 @@ "https://github.com/linux-rdma/rdma-core/" "releases/download/v50.1/rdma-core-50.1.tar.gz" ) -RDMA_CORE_I386_DEFAULT_SOURCE = ( +RDMA_CORE_I386_MANA_DEFAULT_SOURCE = ( "https://github.com/linux-rdma/rdma-core/" "releases/download/v53.1/rdma-core-53.1.tar.gz" ) From 6ab0d27e9563ae3dde704f910e7ac058fa6716f7 Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Tue, 19 Nov 2024 10:23:05 -0800 Subject: [PATCH 11/12] Dpdk: annotation fixes for mypy --- microsoft/testsuites/dpdk/dpdkutil.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index 21f1810889..ce06dbf168 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -1127,7 +1127,7 @@ class NicType(Enum): # Short name for nic types -NIC_SHORT_NAMES = { +NIC_SHORT_NAMES: Dict[NicType, str] = { NicType.CX3: "cx3", NicType.CX4: "cx4", NicType.CX5: "cx5", @@ -1150,13 +1150,12 @@ def get_node_nic_short_name(node: Node) -> str: short_names = map(lambda x: x.value, non_mana_nics) known_nic_types = ",".join(short_names) found_nic_types = ",".join(map(str, [x.device_id for x in devices])) - node.log.debug( + # assert, this will be caught in annotate_dpdk_test_result + # and logged, rather than failing the test. + raise AssertionError( "Unknown NIC hardware was detected during DPDK test case. " f"Expected one of: {known_nic_types}. Found {found_nic_types}. " ) - # this is just a function for annotating a result, so don't assert - # if there's - return found_nic_types def _format_version_str(version: VersionInfo) -> str: @@ -1178,19 +1177,19 @@ def annotate_dpdk_test_result( test_result.information["dpdk_version"] = _format_version_str(dpdk_version) log.debug(f"Adding dpdk version: {dpdk_version}") except AssertionError as err: - test_kit.node.log.debug(f"Could not fetch DPDK version info: {str(err)}") + log.debug(f"Could not fetch DPDK version info: {str(err)}") try: rdma_version = test_kit.rdma_core.get_installed_version() test_result.information["rdma_version"] = _format_version_str(rdma_version) log.debug(f"Adding rdma version: {rdma_version}") except AssertionError as err: - test_kit.node.log.debug(f"Could not fetch RDMA version info: {str(err)}") + log.debug(f"Could not fetch RDMA version info: {str(err)}") try: nic_hw = get_node_nic_short_name(test_kit.node) test_result.information["nic_hw"] = nic_hw log.debug(f"Adding nic version: {nic_hw}") except AssertionError as err: - test_kit.node.log.debug(f"Could not fetch NIC short name: {str(err)}") + log.debug(f"Could not fetch NIC short name: {str(err)}") def skip_32bit_test_on_unsupported_distros(os: OperatingSystem) -> None: From cef07ea4a5a1e9c62ae0465aa56cf024c6322995 Mon Sep 17 00:00:00 2001 From: Matthew G McGovern Date: Tue, 19 Nov 2024 11:00:59 -0800 Subject: [PATCH 12/12] Dpdk: add test annotation to 32bit test --- microsoft/testsuites/dpdk/dpdksuite.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/microsoft/testsuites/dpdk/dpdksuite.py b/microsoft/testsuites/dpdk/dpdksuite.py index dba76cfb9b..5781a8b1cf 100644 --- a/microsoft/testsuites/dpdk/dpdksuite.py +++ b/microsoft/testsuites/dpdk/dpdksuite.py @@ -121,11 +121,13 @@ def verify_dpdk_build_netvsc( ), ) def verify_dpdk_build_netvsc_32bit( - self, node: Node, log: Logger, variables: Dict[str, Any] + self, node: Node, log: Logger, variables: Dict[str, Any], result: TestResult ) -> None: skip_32bit_test_on_unsupported_distros(node.os) force_dpdk_default_source(variables, build_arch=CpuArchitecture.I386) - verify_dpdk_build(node, log, variables, "netvsc", HugePageSize.HUGE_2MB) + verify_dpdk_build( + node, log, variables, "netvsc", HugePageSize.HUGE_2MB, result=result + ) @TestCaseMetadata( description=""" @@ -150,6 +152,7 @@ def verify_dpdk_send_receive_netvsc_32bit( environment: Environment, log: Logger, variables: Dict[str, Any], + result: TestResult, ) -> None: node = environment.default_node skip_32bit_test_on_unsupported_distros(node.os) @@ -161,6 +164,7 @@ def verify_dpdk_send_receive_netvsc_32bit( "netvsc", HugePageSize.HUGE_2MB, multiple_queues=True, + result=result, ) @TestCaseMetadata(