From 5ba88ef91998f5141d7882a394a10fc1face2464 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:09:23 +0200 Subject: [PATCH 01/39] Add a build script that creates a virtual environment and triggers a Scons build. --- .gitignore | 1 + .isort.cfg | 2 +- build | 32 +++++++++++++++ scons/modules.py | 27 +++++++++++++ scons/requirements.txt | 1 + scons/run.py | 90 ++++++++++++++++++++++++++++++++++++++++++ scons/sconstruct.py | 14 +++++++ 7 files changed, 166 insertions(+), 1 deletion(-) create mode 100755 build create mode 100644 scons/modules.py create mode 100644 scons/requirements.txt create mode 100644 scons/run.py create mode 100644 scons/sconstruct.py diff --git a/.gitignore b/.gitignore index e1dc86f2b2..e9b7d96c8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Build files __pycache__/ +scons/build/ python/**/build/ python/**/dist/ python/**/*egg-info/ diff --git a/.isort.cfg b/.isort.cfg index 4f384833b2..31528f3cdc 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -3,7 +3,7 @@ supported_extensions=py,pxd,pyx line_length=120 group_by_package=true known_first_party=mlrl -known_third_party=sklearn,scipy,numpy,tabulate,arff +known_third_party=sklearn,scipy,numpy,tabulate,arff,SCons forced_separate=mlrl.common,mlrl.boosting,mlrl.seco,mlrl.testbed lines_between_types=1 order_by_type=true diff --git a/build b/build new file mode 100755 index 0000000000..8ba0ee5610 --- /dev/null +++ b/build @@ -0,0 +1,32 @@ +#!/bin/sh + +VENV_DIR="venv" +SCONS_DIR="scons" +CLEAN=false + +if [ $# -eq 1 ]; then + if [ $1 = "--clean" ]; then + CLEAN=true + fi + if [ $1 = "-c" ]; then + CLEAN=true + fi +fi + +if [ ! -d $VENV_DIR ] && [ $CLEAN = false ]; then + echo "Creating virtual Python environment..." + python -m venv ${VENV_DIR} +fi + +if [ -d "$VENV_DIR" ]; then + source $VENV_DIR/bin/activate \ + && python -c "import sys; sys.path.append('$SCONS_DIR'); import run; run.install_build_dependencies('scons')" \ + && scons --enable-virtualenv --silent --file $SCONS_DIR/sconstruct.py $@ \ + && deactivate +fi + +if [ $CLEAN = true ] && [ -d $VENV_DIR ]; then + echo "Removing virtual Python environment..." + rm -rf $VENV_DIR + rm -rf $SCONS_DIR/build +fi diff --git a/scons/modules.py b/scons/modules.py new file mode 100644 index 0000000000..f7b599ca34 --- /dev/null +++ b/scons/modules.py @@ -0,0 +1,27 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides access to directories and files belonging to different modules that are part of the project. +""" +from os import path + + +class BuildModule: + """ + Provides access to directories and paths that belong to the build system. + """ + + @property + def root_dir(self) -> str: + return 'scons' + + @property + def build_dir(self) -> str: + return 'build' + + @property + def requirements_file(self) -> str: + return path.join(self.root_dir, 'requirements.txt') + + +BUILD_MODULE = BuildModule() diff --git a/scons/requirements.txt b/scons/requirements.txt new file mode 100644 index 0000000000..41e56894e1 --- /dev/null +++ b/scons/requirements.txt @@ -0,0 +1 @@ +scons >= 4.5, < 4.6 diff --git a/scons/run.py b/scons/run.py new file mode 100644 index 0000000000..e67b564e79 --- /dev/null +++ b/scons/run.py @@ -0,0 +1,90 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for installing and running external programs during the build process. +""" +import subprocess +import sys +from functools import reduce +from typing import List, Tuple + +from pkg_resources import DistributionNotFound, VersionConflict, parse_requirements, require + +from modules import BUILD_MODULE + + +def __run_command(cmd: str, *args, print_args: bool = False): + cmd_formatted = cmd + (reduce(lambda aggr, argument: aggr + ' ' + argument, args, '') if print_args else '') + print('Running external command "' + cmd_formatted + '"...') + cmd_args = [cmd] + + for arg in args: + cmd_args.append(str(arg)) + + out = subprocess.run(cmd_args, check=False) + exit_code = out.returncode + + if exit_code != 0: + print('External command "' + cmd_formatted + '" terminated with non-zero exit code ' + str(exit_code)) + sys.exit(exit_code) + + +def __is_dependency_missing(dependency: str) -> bool: + try: + require(dependency) + return False + except DistributionNotFound: + return True + except VersionConflict: + return False + + +def __is_dependency_outdated(dependency: str) -> bool: + try: + require(dependency) + return False + except DistributionNotFound: + return False + except VersionConflict: + return True + + +def __find_dependencies(requirements_file: str, *dependencies: str) -> List[str]: + with open(requirements_file, mode='r', encoding='utf-8') as f: + dependency_dict = {dependency.key: str(dependency) for dependency in parse_requirements(f.read())} + + if dependencies: + return [dependency_dict[dependency] for dependency in dependencies if dependency in dependency_dict] + + return [dependency for dependency in dependency_dict.values()] + + +def __find_missing_and_outdated_dependencies(requirements_file: str, *dependencies: str) -> Tuple[List[str], List[str]]: + dependencies = __find_dependencies(requirements_file, *dependencies) + missing_dependencies = [dependency for dependency in dependencies if __is_dependency_missing(dependency)] + outdated_dependencies = [dependency for dependency in dependencies if __is_dependency_outdated(dependency)] + return missing_dependencies, outdated_dependencies + + +def __pip_install(dependencies: List[str], force_reinstall: bool = False): + args = ['--prefer-binary'] + + if force_reinstall: + args.append('--force-reinstall') + + __run_command('python', '-m', 'pip', 'install', *args, *dependencies, print_args=True) + + +def __install_dependencies(requirements_file: str, *dependencies: str): + missing_dependencies, outdated_dependencies = __find_missing_and_outdated_dependencies( + requirements_file, *dependencies) + + if missing_dependencies: + __pip_install(missing_dependencies) + + if outdated_dependencies: + __pip_install(outdated_dependencies, force_reinstall=True) + + +def install_build_dependencies(*dependencies: str): + __install_dependencies(BUILD_MODULE.requirements_file, *dependencies) diff --git a/scons/sconstruct.py b/scons/sconstruct.py new file mode 100644 index 0000000000..7940f785ee --- /dev/null +++ b/scons/sconstruct.py @@ -0,0 +1,14 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines the individual targets of the build process. +""" +from os import path + +from SCons.Script.SConscript import SConsEnvironment + +from modules import BUILD_MODULE + +# Create temporary file ".sconsign.dblite" in the build directory... +env = SConsEnvironment() +env.SConsignFile(name=path.join(BUILD_MODULE.build_dir, '.sconsign')) From 5b2708fbaa84eeb0c72327827fd3d1d4eefb34a1 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:26:33 +0200 Subject: [PATCH 02/39] Add build target for formatting Python code. --- scons/code_style.py | 37 ++++++++++++++++++++++++++++++++++ scons/modules.py | 45 ++++++++++++++++++++++++++++++++++++++---- scons/requirements.txt | 2 ++ scons/run.py | 20 ++++++++++++++++--- scons/sconstruct.py | 14 ++++++++++++- 5 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 scons/code_style.py diff --git a/scons/code_style.py b/scons/code_style.py new file mode 100644 index 0000000000..7daf294cdc --- /dev/null +++ b/scons/code_style.py @@ -0,0 +1,37 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for checking and enforcing code style definitions. +""" +from modules import BUILD_MODULE, PYTHON_MODULE +from run import run_program + + +def __isort(directory: str, enforce_changes: bool = False): + args = ['--settings-path', '.', '--virtual-env', 'venv', '--skip-gitignore'] + + if not enforce_changes: + args.append('--check') + + run_program('isort', *args, directory) + + +def __yapf(directory: str, enforce_changes: bool = False): + args = ['-r', '-p', '--style=.style.yapf', '--exclude', '**/build/*.py', '-i' if enforce_changes else '--diff'] + run_program('yapf', *args, directory) + + +def __pylint(directory: str): + args = ['--jobs=0', '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', '--score=n'] + run_program('pylint', *args, directory) + + +def enforce_python_code_style(**_): + """ + Enforces Python source files to adhere to the code style definitions. + """ + for module in [BUILD_MODULE, PYTHON_MODULE]: + directory = module.root_dir + print('Formatting Python code in directory "' + directory + '"...') + __isort(directory, enforce_changes=True) + __yapf(directory, enforce_changes=True) diff --git a/scons/modules.py b/scons/modules.py index f7b599ca34..5fdf38679e 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -3,25 +3,62 @@ Provides access to directories and files belonging to different modules that are part of the project. """ +from abc import ABC, abstractmethod from os import path -class BuildModule: +class Module(ABC): """ - Provides access to directories and paths that belong to the build system. + An abstract base class for all classes that provide access to directories and files that belong to a module. """ @property + @abstractmethod def root_dir(self) -> str: - return 'scons' + """ + The path to the module's root directory. + """ + pass @property def build_dir(self) -> str: - return 'build' + """ + The path to the directory, where build files should be stored. + """ + return path.join(self.root_dir, 'build') @property def requirements_file(self) -> str: + """ + The path to the requirements.txt file that specifies dependencies required by a module. + """ return path.join(self.root_dir, 'requirements.txt') +class PythonModule(Module): + """ + Provides access to directories and files that belong to the project's Python code. + """ + + @property + def root_dir(self) -> str: + return 'python' + + +class BuildModule(Module): + """ + Provides access to directories and files that belong to the build system. + """ + + @property + def root_dir(self) -> str: + return 'scons' + + @property + def build_dir(self) -> str: + return 'build' + + BUILD_MODULE = BuildModule() + +PYTHON_MODULE = PythonModule() diff --git a/scons/requirements.txt b/scons/requirements.txt index 41e56894e1..853b701a7b 100644 --- a/scons/requirements.txt +++ b/scons/requirements.txt @@ -1 +1,3 @@ +isort >= 5.12, < 5.13 scons >= 4.5, < 4.6 +yapf >= 0.40, < 0.41 diff --git a/scons/run.py b/scons/run.py index e67b564e79..3b2e575390 100644 --- a/scons/run.py +++ b/scons/run.py @@ -5,12 +5,12 @@ """ import subprocess import sys -from functools import reduce -from typing import List, Tuple -from pkg_resources import DistributionNotFound, VersionConflict, parse_requirements, require +from functools import reduce +from typing import List, Optional, Tuple from modules import BUILD_MODULE +from pkg_resources import DistributionNotFound, VersionConflict, parse_requirements, require def __run_command(cmd: str, *args, print_args: bool = False): @@ -88,3 +88,17 @@ def __install_dependencies(requirements_file: str, *dependencies: str): def install_build_dependencies(*dependencies: str): __install_dependencies(BUILD_MODULE.requirements_file, *dependencies) + + +def run_program(program: str, + *args, + print_args: bool = False, + additional_dependencies: Optional[List[str]] = None, + requirements_file: str = BUILD_MODULE.requirements_file): + dependencies = [program] + + if additional_dependencies: + dependencies.extend(additional_dependencies) + + __install_dependencies(requirements_file, *dependencies) + __run_command(program, *args, print_args=print_args) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 7940f785ee..422959686c 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -5,10 +5,22 @@ """ from os import path +from code_style import enforce_python_code_style +from modules import BUILD_MODULE from SCons.Script.SConscript import SConsEnvironment -from modules import BUILD_MODULE + +def __create_phony_target(environment, target, action=None): + return environment.AlwaysBuild(environment.Alias(target, None, action)) + + +# Define target names... +TARGET_NAME_FORMAT = 'format' +TARGET_NAME_FORMAT_PYTHON = TARGET_NAME_FORMAT + '_python' # Create temporary file ".sconsign.dblite" in the build directory... env = SConsEnvironment() env.SConsignFile(name=path.join(BUILD_MODULE.build_dir, '.sconsign')) + +# Define targets for enforcing code style definitions... +target_format_python = __create_phony_target(env, TARGET_NAME_FORMAT_PYTHON, action=enforce_python_code_style) From 325bdf04962355354b1a9d3285f1ff907f5550b0 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:29:45 +0200 Subject: [PATCH 03/39] Add build target for checking the Python code style. --- scons/code_style.py | 14 +++++++++++++- scons/requirements.txt | 1 + scons/sconstruct.py | 7 ++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scons/code_style.py b/scons/code_style.py index 7daf294cdc..20af218377 100644 --- a/scons/code_style.py +++ b/scons/code_style.py @@ -26,9 +26,21 @@ def __pylint(directory: str): run_program('pylint', *args, directory) +def check_python_code_style(**_): + """ + Checks if the Python source files adhere to the code style definitions. If this is not the case, an error is raised. + """ + for module in [BUILD_MODULE, PYTHON_MODULE]: + directory = module.root_dir + print('Checking Python code style in directory "' + directory + '"...') + __isort(directory) + __yapf(directory) + __pylint(directory) + + def enforce_python_code_style(**_): """ - Enforces Python source files to adhere to the code style definitions. + Enforces the Python source files to adhere to the code style definitions. """ for module in [BUILD_MODULE, PYTHON_MODULE]: directory = module.root_dir diff --git a/scons/requirements.txt b/scons/requirements.txt index 853b701a7b..13c6ef42d7 100644 --- a/scons/requirements.txt +++ b/scons/requirements.txt @@ -1,3 +1,4 @@ isort >= 5.12, < 5.13 +pylint >= 2.17, < 2.18 scons >= 4.5, < 4.6 yapf >= 0.40, < 0.41 diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 422959686c..9f9dd2ce83 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -5,7 +5,7 @@ """ from os import path -from code_style import enforce_python_code_style +from code_style import check_python_code_style, enforce_python_code_style from modules import BUILD_MODULE from SCons.Script.SConscript import SConsEnvironment @@ -15,6 +15,8 @@ def __create_phony_target(environment, target, action=None): # Define target names... +TARGET_NAME_TEST_FORMAT = 'test_format' +TARGET_NAME_TEST_FORMAT_PYTHON = TARGET_NAME_TEST_FORMAT + '_python' TARGET_NAME_FORMAT = 'format' TARGET_NAME_FORMAT_PYTHON = TARGET_NAME_FORMAT + '_python' @@ -22,5 +24,8 @@ def __create_phony_target(environment, target, action=None): env = SConsEnvironment() env.SConsignFile(name=path.join(BUILD_MODULE.build_dir, '.sconsign')) +# Define targets for checking code style definitions... +target_test_format_python = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_PYTHON, action=check_python_code_style) + # Define targets for enforcing code style definitions... target_format_python = __create_phony_target(env, TARGET_NAME_FORMAT_PYTHON, action=enforce_python_code_style) From e4d4610ba7d714d89b80131b18fcf7e3345586d2 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:30:08 +0200 Subject: [PATCH 04/39] Remove unnecessary pass statement. --- scons/modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scons/modules.py b/scons/modules.py index 5fdf38679e..4db64c75eb 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -18,7 +18,6 @@ def root_dir(self) -> str: """ The path to the module's root directory. """ - pass @property def build_dir(self) -> str: From 25605245797be4b0731e4afc99970d069c5cbfb3 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:34:01 +0200 Subject: [PATCH 05/39] Fix Pylint warnings. --- scons/run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scons/run.py b/scons/run.py index 3b2e575390..252017975b 100644 --- a/scons/run.py +++ b/scons/run.py @@ -50,13 +50,13 @@ def __is_dependency_outdated(dependency: str) -> bool: def __find_dependencies(requirements_file: str, *dependencies: str) -> List[str]: - with open(requirements_file, mode='r', encoding='utf-8') as f: - dependency_dict = {dependency.key: str(dependency) for dependency in parse_requirements(f.read())} + with open(requirements_file, mode='r', encoding='utf-8') as file: + dependency_dict = {dependency.key: str(dependency) for dependency in parse_requirements(file.read())} if dependencies: return [dependency_dict[dependency] for dependency in dependencies if dependency in dependency_dict] - return [dependency for dependency in dependency_dict.values()] + return list(dependency_dict.values()) def __find_missing_and_outdated_dependencies(requirements_file: str, *dependencies: str) -> Tuple[List[str], List[str]]: From 6efe0de77f00953698e1d1a1381868ae5857b3fd Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:34:17 +0200 Subject: [PATCH 06/39] Add missing comments. --- scons/run.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scons/run.py b/scons/run.py index 252017975b..7f5eda1a78 100644 --- a/scons/run.py +++ b/scons/run.py @@ -87,6 +87,11 @@ def __install_dependencies(requirements_file: str, *dependencies: str): def install_build_dependencies(*dependencies: str): + """ + Installs one or several dependencies that are required by the build system. + + :param dependencies: The names of the dependencies that should be installed + """ __install_dependencies(BUILD_MODULE.requirements_file, *dependencies) @@ -95,6 +100,15 @@ def run_program(program: str, print_args: bool = False, additional_dependencies: Optional[List[str]] = None, requirements_file: str = BUILD_MODULE.requirements_file): + """ + Runs an external program. + + :param program: The name of the program to be run + :param args: Optional arguments that should be passed to the program + :param print_args: True, if the arguments should be included in log statements, False otherwise + :param additional_dependencies: The names of dependencies that should be installed before running the program + :param requirements_file: The path of the requirements.txt file that specifies the dependency versions + """ dependencies = [program] if additional_dependencies: From 1eafa091c543513b5940af037e7e9ca7fd860eb7 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:39:33 +0200 Subject: [PATCH 07/39] Add build target for formatting C++ code. --- scons/code_style.py | 28 +++++++++++++++++++++++++++- scons/modules.py | 12 ++++++++++++ scons/requirements.txt | 1 + scons/sconstruct.py | 6 +++++- 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/scons/code_style.py b/scons/code_style.py index 20af218377..fbc0e4f99b 100644 --- a/scons/code_style.py +++ b/scons/code_style.py @@ -3,7 +3,10 @@ Provides utility functions for checking and enforcing code style definitions. """ -from modules import BUILD_MODULE, PYTHON_MODULE +from glob import glob +from os import path + +from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE from run import run_program @@ -26,6 +29,20 @@ def __pylint(directory: str): run_program('pylint', *args, directory) +def __clang_format(directory: str, enforce_changes: bool = True): + cpp_header_files = glob(path.join(directory, '**', '*.hpp'), recursive=True) + cpp_source_files = glob(path.join(directory, '**', '*.cpp'), recursive=True) + args = ['--style=file'] + + if enforce_changes: + args.append('-i') + else: + args.append('-n') + args.append('--Werror') + + run_program('clang-format', *args, *cpp_header_files, *cpp_source_files) + + def check_python_code_style(**_): """ Checks if the Python source files adhere to the code style definitions. If this is not the case, an error is raised. @@ -47,3 +64,12 @@ def enforce_python_code_style(**_): print('Formatting Python code in directory "' + directory + '"...') __isort(directory, enforce_changes=True) __yapf(directory, enforce_changes=True) + + +def enforce_cpp_code_style(**_): + """ + Enforces the C++ source files to adhere to the code style definitions. + """ + directory = CPP_MODULE.root_dir + print('Formatting C++ code in directory "' + directory + '"...') + __clang_format(directory, enforce_changes=True) diff --git a/scons/modules.py b/scons/modules.py index 4db64c75eb..6001eeb649 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -44,6 +44,16 @@ def root_dir(self) -> str: return 'python' +class CppModule(Module): + """ + Provides access to directories and files that belong to the project's C++ code. + """ + + @property + def root_dir(self) -> str: + return 'cpp' + + class BuildModule(Module): """ Provides access to directories and files that belong to the build system. @@ -61,3 +71,5 @@ def build_dir(self) -> str: BUILD_MODULE = BuildModule() PYTHON_MODULE = PythonModule() + +CPP_MODULE = CppModule() diff --git a/scons/requirements.txt b/scons/requirements.txt index 13c6ef42d7..4693698941 100644 --- a/scons/requirements.txt +++ b/scons/requirements.txt @@ -1,3 +1,4 @@ +clang-format >= 16.0, < 16.1 isort >= 5.12, < 5.13 pylint >= 2.17, < 2.18 scons >= 4.5, < 4.6 diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 9f9dd2ce83..ddc0b886e9 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -5,7 +5,7 @@ """ from os import path -from code_style import check_python_code_style, enforce_python_code_style +from code_style import check_python_code_style, enforce_cpp_code_style, enforce_python_code_style from modules import BUILD_MODULE from SCons.Script.SConscript import SConsEnvironment @@ -19,6 +19,7 @@ def __create_phony_target(environment, target, action=None): TARGET_NAME_TEST_FORMAT_PYTHON = TARGET_NAME_TEST_FORMAT + '_python' TARGET_NAME_FORMAT = 'format' TARGET_NAME_FORMAT_PYTHON = TARGET_NAME_FORMAT + '_python' +TARGET_NAME_FORMAT_CPP = TARGET_NAME_FORMAT + '_cpp' # Create temporary file ".sconsign.dblite" in the build directory... env = SConsEnvironment() @@ -29,3 +30,6 @@ def __create_phony_target(environment, target, action=None): # Define targets for enforcing code style definitions... target_format_python = __create_phony_target(env, TARGET_NAME_FORMAT_PYTHON, action=enforce_python_code_style) +target_format_cpp = __create_phony_target(env, TARGET_NAME_FORMAT_CPP, action=enforce_cpp_code_style) +target_format = __create_phony_target(env, TARGET_NAME_FORMAT) +env.Depends(target_format, [target_format_python, target_format_cpp]) From 2bebe7974baa83f754a093182d629352647307a1 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:41:39 +0200 Subject: [PATCH 08/39] Add build target for checking the C++ code style. --- scons/code_style.py | 9 +++++++++ scons/sconstruct.py | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scons/code_style.py b/scons/code_style.py index fbc0e4f99b..51dd8a729b 100644 --- a/scons/code_style.py +++ b/scons/code_style.py @@ -66,6 +66,15 @@ def enforce_python_code_style(**_): __yapf(directory, enforce_changes=True) +def check_cpp_code_style(**_): + """ + Checks if the C++ source files adhere to the code style definitions. If this is not the case, an error is raised. + """ + directory = CPP_MODULE.root_dir + print('Checking C++ code style in directory "' + directory + '"...') + __clang_format(directory) + + def enforce_cpp_code_style(**_): """ Enforces the C++ source files to adhere to the code style definitions. diff --git a/scons/sconstruct.py b/scons/sconstruct.py index ddc0b886e9..85dc40a100 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -5,7 +5,7 @@ """ from os import path -from code_style import check_python_code_style, enforce_cpp_code_style, enforce_python_code_style +from code_style import check_cpp_code_style, check_python_code_style, enforce_cpp_code_style, enforce_python_code_style from modules import BUILD_MODULE from SCons.Script.SConscript import SConsEnvironment @@ -17,6 +17,7 @@ def __create_phony_target(environment, target, action=None): # Define target names... TARGET_NAME_TEST_FORMAT = 'test_format' TARGET_NAME_TEST_FORMAT_PYTHON = TARGET_NAME_TEST_FORMAT + '_python' +TARGET_NAME_TEST_FORMAT_CPP = TARGET_NAME_TEST_FORMAT + '_cpp' TARGET_NAME_FORMAT = 'format' TARGET_NAME_FORMAT_PYTHON = TARGET_NAME_FORMAT + '_python' TARGET_NAME_FORMAT_CPP = TARGET_NAME_FORMAT + '_cpp' @@ -27,6 +28,9 @@ def __create_phony_target(environment, target, action=None): # Define targets for checking code style definitions... target_test_format_python = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_PYTHON, action=check_python_code_style) +target_test_format_cpp = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_CPP, action=check_cpp_code_style) +target_test_format = __create_phony_target(env, TARGET_NAME_TEST_FORMAT) +env.Depends(target_test_format, [target_test_format_python, target_test_format_cpp]) # Define targets for enforcing code style definitions... target_format_python = __create_phony_target(env, TARGET_NAME_FORMAT_PYTHON, action=enforce_python_code_style) From 4cf064cdd384c95c5a55d255164a7c8b8a49331f Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:44:52 +0200 Subject: [PATCH 09/39] Exit build if any invalid targets are given. --- scons/sconstruct.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 85dc40a100..cc9ad05af0 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -3,10 +3,14 @@ Defines the individual targets of the build process. """ +import sys + +from functools import reduce from os import path from code_style import check_cpp_code_style, check_python_code_style, enforce_cpp_code_style, enforce_python_code_style from modules import BUILD_MODULE +from SCons.Script import COMMAND_LINE_TARGETS from SCons.Script.SConscript import SConsEnvironment @@ -22,6 +26,19 @@ def __create_phony_target(environment, target, action=None): TARGET_NAME_FORMAT_PYTHON = TARGET_NAME_FORMAT + '_python' TARGET_NAME_FORMAT_CPP = TARGET_NAME_FORMAT + '_cpp' +VALID_TARGETS = { + TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_FORMAT, + TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP +} + +# Raise an error if any invalid targets are given... +invalid_targets = [target for target in COMMAND_LINE_TARGETS if target not in VALID_TARGETS] + +if invalid_targets: + print('The following targets are unknown: ' + + reduce(lambda aggr, target: aggr + (', ' if len(aggr) > 0 else '') + target, invalid_targets, '')) + sys.exit(-1) + # Create temporary file ".sconsign.dblite" in the build directory... env = SConsEnvironment() env.SConsignFile(name=path.join(BUILD_MODULE.build_dir, '.sconsign')) From 98983de865aa0f753260482a6cd6590c389e8365 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:49:17 +0200 Subject: [PATCH 10/39] The class BuildModule does not override the property "build_dir" anymore. --- scons/modules.py | 4 ---- scons/sconstruct.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/scons/modules.py b/scons/modules.py index 6001eeb649..69937de410 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -63,10 +63,6 @@ class BuildModule(Module): def root_dir(self) -> str: return 'scons' - @property - def build_dir(self) -> str: - return 'build' - BUILD_MODULE = BuildModule() diff --git a/scons/sconstruct.py b/scons/sconstruct.py index cc9ad05af0..e827cb4c5d 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -41,7 +41,7 @@ def __create_phony_target(environment, target, action=None): # Create temporary file ".sconsign.dblite" in the build directory... env = SConsEnvironment() -env.SConsignFile(name=path.join(BUILD_MODULE.build_dir, '.sconsign')) +env.SConsignFile(name=path.relpath(path.join(BUILD_MODULE.build_dir, '.sconsign'), BUILD_MODULE.root_dir)) # Define targets for checking code style definitions... target_test_format_python = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_PYTHON, action=check_python_code_style) From 2ce468105b810173de24ed9dd801d761d2f32104 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:52:39 +0200 Subject: [PATCH 11/39] Add build target for installing runtime dependencies. --- scons/run.py | 14 +++++++++++++- scons/sconstruct.py | 7 ++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/scons/run.py b/scons/run.py index 7f5eda1a78..2ff3735814 100644 --- a/scons/run.py +++ b/scons/run.py @@ -7,9 +7,10 @@ import sys from functools import reduce +from os import path from typing import List, Optional, Tuple -from modules import BUILD_MODULE +from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE from pkg_resources import DistributionNotFound, VersionConflict, parse_requirements, require @@ -95,6 +96,17 @@ def install_build_dependencies(*dependencies: str): __install_dependencies(BUILD_MODULE.requirements_file, *dependencies) +def install_runtime_dependencies(**_): + """ + Installs all runtime dependencies that are required by the Python and C++ module. + """ + for module in [PYTHON_MODULE, CPP_MODULE]: + requirements_file = module.requirements_file + + if path.isfile(requirements_file): + __install_dependencies(requirements_file) + + def run_program(program: str, *args, print_args: bool = False, diff --git a/scons/sconstruct.py b/scons/sconstruct.py index e827cb4c5d..19fb10a6b6 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -10,6 +10,7 @@ from code_style import check_cpp_code_style, check_python_code_style, enforce_cpp_code_style, enforce_python_code_style from modules import BUILD_MODULE +from run import install_runtime_dependencies from SCons.Script import COMMAND_LINE_TARGETS from SCons.Script.SConscript import SConsEnvironment @@ -25,10 +26,11 @@ def __create_phony_target(environment, target, action=None): TARGET_NAME_FORMAT = 'format' TARGET_NAME_FORMAT_PYTHON = TARGET_NAME_FORMAT + '_python' TARGET_NAME_FORMAT_CPP = TARGET_NAME_FORMAT + '_cpp' +TARGET_NAME_VENV = 'venv' VALID_TARGETS = { TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_FORMAT, - TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP + TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_VENV } # Raise an error if any invalid targets are given... @@ -54,3 +56,6 @@ def __create_phony_target(environment, target, action=None): target_format_cpp = __create_phony_target(env, TARGET_NAME_FORMAT_CPP, action=enforce_cpp_code_style) target_format = __create_phony_target(env, TARGET_NAME_FORMAT) env.Depends(target_format, [target_format_python, target_format_cpp]) + +# Define target for installing runtime dependencies... +target_venv = __create_phony_target(env, TARGET_NAME_VENV, action=install_runtime_dependencies) From e8e107fe86ed453865597e5d6c6a98bfaebd4083 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 21:59:00 +0200 Subject: [PATCH 12/39] Add build target for compiling the C++ code. --- scons/compilation.py | 28 ++++++++++++++++++++++++++++ scons/requirements.txt | 2 ++ scons/sconstruct.py | 12 ++++++++++-- 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 scons/compilation.py diff --git a/scons/compilation.py b/scons/compilation.py new file mode 100644 index 0000000000..9b8ff873cd --- /dev/null +++ b/scons/compilation.py @@ -0,0 +1,28 @@ +from typing import List, Optional + +from modules import CPP_MODULE, Module +from run import run_program + + +def __meson_setup(root_dir: str, build_dir: str, dependencies: Optional[List[str]] = None): + print('Setting up build directory "' + build_dir + '"...') + run_program('meson', 'setup', build_dir, root_dir, print_args=True, additional_dependencies=dependencies) + + +def __meson_compile(build_dir: str): + run_program('meson', 'compile', '-C', build_dir, print_args=True) + + +def setup_cpp(**_): + """ + Sets up the build system for compiling the C++ code. + """ + __meson_setup(CPP_MODULE.root_dir, CPP_MODULE.build_dir, dependencies=['ninja']) + + +def compile_cpp(**_): + """ + Compiles the C++ code. + """ + print('Compiling C++ code...') + __meson_compile(CPP_MODULE.build_dir) diff --git a/scons/requirements.txt b/scons/requirements.txt index 4693698941..e9ad493013 100644 --- a/scons/requirements.txt +++ b/scons/requirements.txt @@ -1,5 +1,7 @@ clang-format >= 16.0, < 16.1 isort >= 5.12, < 5.13 +meson >= 1.2, < 1.3 +ninja >= 1.11, < 1.12 pylint >= 2.17, < 2.18 scons >= 4.5, < 4.6 yapf >= 0.40, < 0.41 diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 19fb10a6b6..1cb478f6ed 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -9,7 +9,8 @@ from os import path from code_style import check_cpp_code_style, check_python_code_style, enforce_cpp_code_style, enforce_python_code_style -from modules import BUILD_MODULE +from compilation import compile_cpp, setup_cpp +from modules import BUILD_MODULE, CPP_MODULE from run import install_runtime_dependencies from SCons.Script import COMMAND_LINE_TARGETS from SCons.Script.SConscript import SConsEnvironment @@ -27,10 +28,12 @@ def __create_phony_target(environment, target, action=None): TARGET_NAME_FORMAT_PYTHON = TARGET_NAME_FORMAT + '_python' TARGET_NAME_FORMAT_CPP = TARGET_NAME_FORMAT + '_cpp' TARGET_NAME_VENV = 'venv' +TARGET_NAME_COMPILE = 'compile' +TARGET_NAME_COMPILE_CPP = TARGET_NAME_COMPILE + '_cpp' VALID_TARGETS = { TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_FORMAT, - TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_VENV + TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP } # Raise an error if any invalid targets are given... @@ -59,3 +62,8 @@ def __create_phony_target(environment, target, action=None): # Define target for installing runtime dependencies... target_venv = __create_phony_target(env, TARGET_NAME_VENV, action=install_runtime_dependencies) + +# Define targets for compiling the C++ and Cython code... +env.Command(CPP_MODULE.build_dir, None, action=setup_cpp) +target_compile_cpp = __create_phony_target(env, TARGET_NAME_COMPILE_CPP, action=compile_cpp) +env.Depends(target_compile_cpp, [target_venv, CPP_MODULE.build_dir]) From 4f7d308607759a00b9b5026c1cc5f7eb98d207a3 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 22:03:06 +0200 Subject: [PATCH 13/39] Add build target for compiling the Cython code. --- scons/compilation.py | 22 +++++++++++++++++++++- scons/requirements.txt | 1 + scons/sconstruct.py | 15 ++++++++++++--- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/scons/compilation.py b/scons/compilation.py index 9b8ff873cd..620c4880c4 100644 --- a/scons/compilation.py +++ b/scons/compilation.py @@ -1,6 +1,11 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for compiling C++ and Cython code. +""" from typing import List, Optional -from modules import CPP_MODULE, Module +from modules import CPP_MODULE, PYTHON_MODULE from run import run_program @@ -26,3 +31,18 @@ def compile_cpp(**_): """ print('Compiling C++ code...') __meson_compile(CPP_MODULE.build_dir) + + +def setup_cython(**_): + """ + Sets up the build system for compiling the Cython code. + """ + __meson_setup(PYTHON_MODULE.root_dir, PYTHON_MODULE.build_dir, dependencies=['cython']) + + +def compile_cython(**_): + """ + Compiles the Cython code. + """ + print('Compiling Cython code...') + __meson_compile(PYTHON_MODULE.build_dir) diff --git a/scons/requirements.txt b/scons/requirements.txt index e9ad493013..8e93e75ea7 100644 --- a/scons/requirements.txt +++ b/scons/requirements.txt @@ -1,4 +1,5 @@ clang-format >= 16.0, < 16.1 +cython >= 3.0, < 3.1 isort >= 5.12, < 5.13 meson >= 1.2, < 1.3 ninja >= 1.11, < 1.12 diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 1cb478f6ed..4481c842ec 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -9,8 +9,8 @@ from os import path from code_style import check_cpp_code_style, check_python_code_style, enforce_cpp_code_style, enforce_python_code_style -from compilation import compile_cpp, setup_cpp -from modules import BUILD_MODULE, CPP_MODULE +from compilation import compile_cpp, compile_cython, setup_cpp, setup_cython +from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE from run import install_runtime_dependencies from SCons.Script import COMMAND_LINE_TARGETS from SCons.Script.SConscript import SConsEnvironment @@ -30,10 +30,12 @@ def __create_phony_target(environment, target, action=None): TARGET_NAME_VENV = 'venv' TARGET_NAME_COMPILE = 'compile' TARGET_NAME_COMPILE_CPP = TARGET_NAME_COMPILE + '_cpp' +TARGET_NAME_COMPILE_CYTHON = TARGET_NAME_COMPILE + '_cython' VALID_TARGETS = { TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_FORMAT, - TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP + TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, + TARGET_NAME_COMPILE_CYTHON } # Raise an error if any invalid targets are given... @@ -67,3 +69,10 @@ def __create_phony_target(environment, target, action=None): env.Command(CPP_MODULE.build_dir, None, action=setup_cpp) target_compile_cpp = __create_phony_target(env, TARGET_NAME_COMPILE_CPP, action=compile_cpp) env.Depends(target_compile_cpp, [target_venv, CPP_MODULE.build_dir]) + +env.Command(PYTHON_MODULE.build_dir, None, action=setup_cython) +target_compile_cython = __create_phony_target(env, TARGET_NAME_COMPILE_CYTHON, action=compile_cython) +env.Depends(target_compile_cython, [target_compile_cpp, PYTHON_MODULE.build_dir]) + +target_compile = __create_phony_target(env, TARGET_NAME_COMPILE) +env.Depends(target_compile, [target_compile_cpp, target_compile_cython]) From 4a925d4a017eab6b63369329ea9c61f3299806fb Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 22:06:15 +0200 Subject: [PATCH 14/39] Add build targets for removing C++ and Cython build files. --- scons/sconstruct.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 4481c842ec..90483e5114 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -20,6 +20,11 @@ def __create_phony_target(environment, target, action=None): return environment.AlwaysBuild(environment.Alias(target, None, action)) +def __print_if_clean(environment, message: str): + if environment.GetOption('clean'): + print(message) + + # Define target names... TARGET_NAME_TEST_FORMAT = 'test_format' TARGET_NAME_TEST_FORMAT_PYTHON = TARGET_NAME_TEST_FORMAT + '_python' @@ -38,6 +43,8 @@ def __create_phony_target(environment, target, action=None): TARGET_NAME_COMPILE_CYTHON } +DEFAULT_TARGET = 'undefined' + # Raise an error if any invalid targets are given... invalid_targets = [target for target in COMMAND_LINE_TARGETS if target not in VALID_TARGETS] @@ -76,3 +83,16 @@ def __create_phony_target(environment, target, action=None): target_compile = __create_phony_target(env, TARGET_NAME_COMPILE) env.Depends(target_compile, [target_compile_cpp, target_compile_cython]) + +# Define targets for cleaning up C++ and Cython build directories... +if not COMMAND_LINE_TARGETS \ + or TARGET_NAME_COMPILE_CPP in COMMAND_LINE_TARGETS \ + or TARGET_NAME_COMPILE in COMMAND_LINE_TARGETS: + __print_if_clean(env, 'Removing C++ build files...') + env.Clean([target_compile_cpp, DEFAULT_TARGET], CPP_MODULE.build_dir) + +if not COMMAND_LINE_TARGETS \ + or TARGET_NAME_COMPILE_CYTHON in COMMAND_LINE_TARGETS \ + or TARGET_NAME_COMPILE in COMMAND_LINE_TARGETS: + __print_if_clean(env, 'Removing Cython build files...') + env.Clean([target_compile_cython, DEFAULT_TARGET], PYTHON_MODULE.build_dir) From 38046badc14df29f5aa800a142061d0807d38cdf Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 22:07:57 +0200 Subject: [PATCH 15/39] Add build target for installing shared libraries into the source tree. --- scons/compilation.py | 12 ++++++++++++ scons/sconstruct.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scons/compilation.py b/scons/compilation.py index 620c4880c4..a65a482a29 100644 --- a/scons/compilation.py +++ b/scons/compilation.py @@ -18,6 +18,10 @@ def __meson_compile(build_dir: str): run_program('meson', 'compile', '-C', build_dir, print_args=True) +def __meson_install(build_dir: str): + run_program('meson', 'install', '--no-rebuild', '--only-changed', '-C', build_dir, print_args=True) + + def setup_cpp(**_): """ Sets up the build system for compiling the C++ code. @@ -33,6 +37,14 @@ def compile_cpp(**_): __meson_compile(CPP_MODULE.build_dir) +def install_cpp(**_): + """ + Installs shared libraries into the source tree. + """ + print('Installing shared libraries into source tree...') + __meson_install(CPP_MODULE.build_dir) + + def setup_cython(**_): """ Sets up the build system for compiling the Cython code. diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 90483e5114..8f7ffc5fbb 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -9,7 +9,7 @@ from os import path from code_style import check_cpp_code_style, check_python_code_style, enforce_cpp_code_style, enforce_python_code_style -from compilation import compile_cpp, compile_cython, setup_cpp, setup_cython +from compilation import compile_cpp, compile_cython, install_cpp, setup_cpp, setup_cython from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE from run import install_runtime_dependencies from SCons.Script import COMMAND_LINE_TARGETS @@ -36,11 +36,13 @@ def __print_if_clean(environment, message: str): TARGET_NAME_COMPILE = 'compile' TARGET_NAME_COMPILE_CPP = TARGET_NAME_COMPILE + '_cpp' TARGET_NAME_COMPILE_CYTHON = TARGET_NAME_COMPILE + '_cython' +TARGET_NAME_INSTALL = 'install' +TARGET_NAME_INSTALL_CPP = TARGET_NAME_INSTALL + '_cpp' VALID_TARGETS = { TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_FORMAT, TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, - TARGET_NAME_COMPILE_CYTHON + TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP } DEFAULT_TARGET = 'undefined' @@ -96,3 +98,7 @@ def __print_if_clean(environment, message: str): or TARGET_NAME_COMPILE in COMMAND_LINE_TARGETS: __print_if_clean(env, 'Removing Cython build files...') env.Clean([target_compile_cython, DEFAULT_TARGET], PYTHON_MODULE.build_dir) + +# Define targets for installing shared libraries and extension modules into the source tree... +target_install_cpp = __create_phony_target(env, TARGET_NAME_INSTALL_CPP, action=install_cpp) +env.Depends(target_install_cpp, target_compile_cpp) From f90e5bdb1c9a9f7ad69ec3f90138a042a2750093 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 22:10:38 +0200 Subject: [PATCH 16/39] Add build target for installing extension modules into the source tree. --- scons/compilation.py | 8 ++++++++ scons/sconstruct.py | 11 +++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scons/compilation.py b/scons/compilation.py index a65a482a29..fb3f9b558c 100644 --- a/scons/compilation.py +++ b/scons/compilation.py @@ -58,3 +58,11 @@ def compile_cython(**_): """ print('Compiling Cython code...') __meson_compile(PYTHON_MODULE.build_dir) + + +def install_cython(**_): + """ + Installs extension modules into the source tree. + """ + print('Installing extension modules into source tree...') + __meson_install(PYTHON_MODULE.build_dir) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 8f7ffc5fbb..de56c1fb60 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -9,7 +9,7 @@ from os import path from code_style import check_cpp_code_style, check_python_code_style, enforce_cpp_code_style, enforce_python_code_style -from compilation import compile_cpp, compile_cython, install_cpp, setup_cpp, setup_cython +from compilation import compile_cpp, compile_cython, install_cpp, install_cython, setup_cpp, setup_cython from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE from run import install_runtime_dependencies from SCons.Script import COMMAND_LINE_TARGETS @@ -38,11 +38,12 @@ def __print_if_clean(environment, message: str): TARGET_NAME_COMPILE_CYTHON = TARGET_NAME_COMPILE + '_cython' TARGET_NAME_INSTALL = 'install' TARGET_NAME_INSTALL_CPP = TARGET_NAME_INSTALL + '_cpp' +TARGET_NAME_INSTALL_CYTHON = TARGET_NAME_INSTALL + '_cython' VALID_TARGETS = { TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_FORMAT, TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, - TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP + TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON } DEFAULT_TARGET = 'undefined' @@ -102,3 +103,9 @@ def __print_if_clean(environment, message: str): # Define targets for installing shared libraries and extension modules into the source tree... target_install_cpp = __create_phony_target(env, TARGET_NAME_INSTALL_CPP, action=install_cpp) env.Depends(target_install_cpp, target_compile_cpp) + +target_install_cython = __create_phony_target(env, TARGET_NAME_INSTALL_CYTHON, action=install_cython) +env.Depends(target_install_cython, target_compile_cython) + +target_install = env.Alias(TARGET_NAME_INSTALL, None, None) +env.Depends(target_install, [target_install_cpp, target_install_cython]) From 9a6ac9ae97879da1ef215038faa58da2c65e4238 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Tue, 5 Sep 2023 22:21:08 +0200 Subject: [PATCH 17/39] Add build targets for removing shared libraries and extension modules from the source tree. --- scons/modules.py | 101 +++++++++++++++++++++++++++++++++++++++++++- scons/sconstruct.py | 17 ++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/scons/modules.py b/scons/modules.py index 69937de410..0f2092e1c9 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -4,7 +4,32 @@ Provides access to directories and files belonging to different modules that are part of the project. """ from abc import ABC, abstractmethod -from os import path +from glob import glob +from os import path, walk +from typing import Callable, List + + +def find_files_recursively(directory: str, + directory_filter: Callable[[str], bool] = lambda _: True, + file_filter: Callable[[str], bool] = lambda _: True) -> List[str]: + """ + Finds and returns files in a directory and its subdirectories that match a given filter. + + :param directory: The directory to be searched + :param directory_filter: A function to be used for filtering subdirectories + :param file_filter: A function to be used for filtering files + :return: A list that contains the paths of all files that have been found + """ + result = [] + + for root_directory, subdirectories, files in walk(directory, topdown=True): + subdirectories[:] = [subdirectory for subdirectory in subdirectories if directory_filter(subdirectory)] + + for file in files: + if file_filter(file): + result.append(path.join(root_directory, file)) + + return result class Module(ABC): @@ -39,10 +64,84 @@ class PythonModule(Module): Provides access to directories and files that belong to the project's Python code. """ + class Subproject: + """ + Provides access to directories and files that belong to an individual subproject that is part of the project's + Python code. + """ + + @staticmethod + def __filter_pycache_directories(directory: str) -> bool: + return directory != '__pycache__' + + def __init__(self, root_dir: str): + """ + :param root_dir: The root directory of the subproject + """ + self.root_dir = root_dir + + @property + def name(self) -> str: + """ + The name of the subproject. + """ + return path.basename(self.root_dir) + + @property + def source_dir(self) -> str: + """ + The directory that contains the subproject's source code. + """ + return path.join(self.root_dir, 'mlrl') + + def find_shared_libraries(self) -> List[str]: + """ + Finds and returns all shared libraries that are contained in the subproject's source tree. + + :return: A list that contains all shared libraries that have been found + """ + + def file_filter(file) -> bool: + return (file.startswith('lib') and file.find('.so') >= 0) \ + or file.endswith('.dylib') \ + or (file.startswith('mlrl') and file.endswith('.lib')) \ + or file.endswith('.dll') + + return find_files_recursively(self.source_dir, + directory_filter=self.__filter_pycache_directories, + file_filter=file_filter) + + def find_extension_modules(self) -> List[str]: + """ + Finds and returns all extension modules that are contained in the subproject's source tree. + + :return: A list that contains all extension modules that have been found + """ + + def file_filter(file) -> bool: + return (not file.startswith('lib') and file.endswith('.so')) \ + or file.endswith('.pyd') \ + or (not file.startswith('mlrl') and file.endswith('.lib')) + + return find_files_recursively(self.source_dir, + directory_filter=self.__filter_pycache_directories, + file_filter=file_filter) + @property def root_dir(self) -> str: return 'python' + def find_subprojects(self) -> List[Subproject]: + """ + Finds and returns all subprojects that are part of the Python code. + + :return: A list that contains all subrojects that have been found + """ + return [ + PythonModule.Subproject(file) for file in glob(path.join(self.root_dir, 'subprojects', '*')) + if path.isdir(file) + ] + class CppModule(Module): """ diff --git a/scons/sconstruct.py b/scons/sconstruct.py index de56c1fb60..e75de0268c 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -109,3 +109,20 @@ def __print_if_clean(environment, message: str): target_install = env.Alias(TARGET_NAME_INSTALL, None, None) env.Depends(target_install, [target_install_cpp, target_install_cython]) + +# Define targets for removing shared libraries and extension modules from the source tree... +if not COMMAND_LINE_TARGETS \ + or TARGET_NAME_INSTALL_CPP in COMMAND_LINE_TARGETS \ + or TARGET_NAME_INSTALL in COMMAND_LINE_TARGETS: + __print_if_clean(env, 'Removing shared libraries from source tree...') + + for subproject in PYTHON_MODULE.find_subprojects(): + env.Clean([target_install_cpp, DEFAULT_TARGET], subproject.find_shared_libraries()) + +if not COMMAND_LINE_TARGETS \ + or TARGET_NAME_INSTALL_CYTHON in COMMAND_LINE_TARGETS \ + or TARGET_NAME_INSTALL in COMMAND_LINE_TARGETS: + __print_if_clean(env, 'Removing extension modules from source tree...') + + for subproject in PYTHON_MODULE.find_subprojects(): + env.Clean([target_install_cython, DEFAULT_TARGET], subproject.find_extension_modules()) From 7558e279a003c1860e28eb7f5c2f8c81bfe613b4 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 6 Sep 2023 19:17:24 +0200 Subject: [PATCH 18/39] Add build targets for building and installing wheel packages. --- scons/modules.py | 36 +++++++++++++++++++++++++++++++ scons/packaging.py | 48 ++++++++++++++++++++++++++++++++++++++++++ scons/requirements.txt | 1 + scons/run.py | 23 ++++++++++++++++++++ scons/sconstruct.py | 27 +++++++++++++++++++++++- 5 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 scons/packaging.py diff --git a/scons/modules.py b/scons/modules.py index 0f2092e1c9..a40c04342c 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -94,6 +94,29 @@ def source_dir(self) -> str: """ return path.join(self.root_dir, 'mlrl') + @property + def dist_dir(self) -> str: + """ + The directory that contains all wheel packages that have been built for the subproject. + """ + return path.join(self.root_dir, 'dist') + + def find_wheels(self) -> List[str]: + """ + Finds and returns all wheel packages that have been built for the subproject. + + :return: A list that contains the paths of the wheel packages that have been found + """ + return glob(path.join(self.dist_dir, '*.whl')) + + def find_source_files(self) -> List[str]: + """ + Finds and returns all source files that are contained by the subproject. + + :return: A list that contains the paths of the source files that have been found + """ + return find_files_recursively(self.source_dir, directory_filter=self.__filter_pycache_directories) + def find_shared_libraries(self) -> List[str]: """ Finds and returns all shared libraries that are contained in the subproject's source tree. @@ -142,6 +165,19 @@ def find_subprojects(self) -> List[Subproject]: if path.isdir(file) ] + def find_subproject(self, file: str) -> Subproject: + """ + Finds and returns the subproject to which a given file belongs. + + :param file: The path of the file + :return: The subproject to which the given file belongs + """ + for subproject in self.find_subprojects(): + if file.startswith(subproject.root_dir): + return subproject + + raise ValueError('File "' + file + '" does not belong to a Python subproject') + class CppModule(Module): """ diff --git a/scons/packaging.py b/scons/packaging.py new file mode 100644 index 0000000000..be8e2a2910 --- /dev/null +++ b/scons/packaging.py @@ -0,0 +1,48 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for building and installing Python wheel packages. +""" +from typing import List + +from modules import PYTHON_MODULE +from run import run_python_program + + +def __build_python_wheel(package_dir: str): + run_python_program('build', '--wheel', package_dir, print_args=True) + + +def __install_python_wheels(wheels: List[str]): + run_python_program('pip', 'install', '--force-reinstall', '--no-deps', *wheels, print_args=True) + + +# pylint: disable=unused-argument +def build_python_wheel(env, target, source): + """ + Builds a Python wheel package for a single subproject. + + :param env: The scons environment + :param target: The path of the wheel package to be built, if it does already exist, or the path of the directory, + where the wheel package should be stored + :param source: The source files from which the wheel package should be built + """ + if target: + subproject = PYTHON_MODULE.find_subproject(target[0].path) + print('Building Python wheels for subproject "' + subproject.name + '"...') + __build_python_wheel(subproject.root_dir) + + +# pylint: disable=unused-argument +def install_python_wheels(env, target, source): + """ + Installs all Python wheel packages that have been built for a single subproject. + + :param env: The scons environment + :param target: The path of the subproject's root directory + :param source: The paths of the wheel packages to be installed + """ + if source: + subproject = PYTHON_MODULE.find_subproject(source[0].path) + print('Installing Python wheels for subproject "' + subproject.name + '"...') + __install_python_wheels(subproject.find_wheels()) diff --git a/scons/requirements.txt b/scons/requirements.txt index 8e93e75ea7..09ce76feb0 100644 --- a/scons/requirements.txt +++ b/scons/requirements.txt @@ -1,3 +1,4 @@ +build >= 1.0, < 1.1 clang-format >= 16.0, < 16.1 cython >= 3.0, < 3.1 isort >= 5.12, < 5.13 diff --git a/scons/run.py b/scons/run.py index 2ff3735814..a28428e971 100644 --- a/scons/run.py +++ b/scons/run.py @@ -128,3 +128,26 @@ def run_program(program: str, __install_dependencies(requirements_file, *dependencies) __run_command(program, *args, print_args=print_args) + + +def run_python_program(program: str, + *args, + print_args: bool = False, + additional_dependencies: Optional[List[str]] = None, + requirements_file: str = BUILD_MODULE.requirements_file): + """ + Runs an external Python program. + + :param program: The name of the program to be run + :param args: Optional arguments that should be passed to the program + :param print_args: True, if the arguments should be included in log statements, False otherwise + :param additional_dependencies: The names of dependencies that should be installed before running the program + :param requirements_file: The path of the requirements.txt file that specifies the dependency versions + """ + dependencies = [program] + + if additional_dependencies: + dependencies.extend(additional_dependencies) + + __install_dependencies(requirements_file, *dependencies) + __run_command('python', '-m', program, *args, print_args=print_args) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index e75de0268c..343bc41002 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -11,6 +11,7 @@ from code_style import check_cpp_code_style, check_python_code_style, enforce_cpp_code_style, enforce_python_code_style from compilation import compile_cpp, compile_cython, install_cpp, install_cython, setup_cpp, setup_cython from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE +from packaging import build_python_wheel, install_python_wheels from run import install_runtime_dependencies from SCons.Script import COMMAND_LINE_TARGETS from SCons.Script.SConscript import SConsEnvironment @@ -39,11 +40,14 @@ def __print_if_clean(environment, message: str): TARGET_NAME_INSTALL = 'install' TARGET_NAME_INSTALL_CPP = TARGET_NAME_INSTALL + '_cpp' TARGET_NAME_INSTALL_CYTHON = TARGET_NAME_INSTALL + '_cython' +TARGET_NAME_BUILD_WHEELS = 'build_wheels' +TARGET_NAME_INSTALL_WHEELS = 'install_wheels' VALID_TARGETS = { TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_FORMAT, TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, - TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON + TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, + TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS } DEFAULT_TARGET = 'undefined' @@ -126,3 +130,24 @@ def __print_if_clean(environment, message: str): for subproject in PYTHON_MODULE.find_subprojects(): env.Clean([target_install_cython, DEFAULT_TARGET], subproject.find_extension_modules()) + +# Define targets for building and installing Python wheels... +commands_build_wheels = [] +commands_install_wheels = [] + +for subproject in PYTHON_MODULE.find_subprojects(): + wheels = subproject.find_wheels() + targets_build_wheels = wheels if wheels else subproject.dist_dir + + command_build_wheels = env.Command(targets_build_wheels, subproject.find_source_files(), action=build_python_wheel) + commands_build_wheels.append(command_build_wheels) + + command_install_wheels = env.Command(subproject.root_dir, targets_build_wheels, action=install_python_wheels) + env.Depends(command_install_wheels, command_build_wheels) + commands_install_wheels.append(command_install_wheels) + +target_build_wheels = env.Alias(TARGET_NAME_BUILD_WHEELS, None, None) +env.Depends(target_build_wheels, [target_install] + commands_build_wheels) + +target_install_wheels = env.Alias(TARGET_NAME_INSTALL_WHEELS, None, None) +env.Depends(target_install_wheels, [target_install] + commands_install_wheels) From c21485d1cdd64fce61e8c0906459cd7dc18a0b1e Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 6 Sep 2023 19:20:07 +0200 Subject: [PATCH 19/39] Add build target for removing wheel packages. --- scons/modules.py | 9 ++++++++- scons/sconstruct.py | 7 +++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scons/modules.py b/scons/modules.py index a40c04342c..6e25335299 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -47,7 +47,7 @@ def root_dir(self) -> str: @property def build_dir(self) -> str: """ - The path to the directory, where build files should be stored. + The path to the directory, where build files are stored. """ return path.join(self.root_dir, 'build') @@ -101,6 +101,13 @@ def dist_dir(self) -> str: """ return path.join(self.root_dir, 'dist') + @property + def build_dirs(self) -> List[str]: + """ + A list that contains all directories, where the subproject's build files are stored. + """ + return [self.dist_dir, path.join(self.root_dir, 'build')] + glob(path.join(self.root_dir, '*.egg-info')) + def find_wheels(self) -> List[str]: """ Finds and returns all wheel packages that have been built for the subproject. diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 343bc41002..457b707f8d 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -151,3 +151,10 @@ def __print_if_clean(environment, message: str): target_install_wheels = env.Alias(TARGET_NAME_INSTALL_WHEELS, None, None) env.Depends(target_install_wheels, [target_install] + commands_install_wheels) + +# Define target for cleaning up Python wheels and associated build directories... +if not COMMAND_LINE_TARGETS or TARGET_NAME_BUILD_WHEELS in COMMAND_LINE_TARGETS: + __print_if_clean(env, 'Removing Python wheels...') + + for subproject in PYTHON_MODULE.find_subprojects(): + env.Clean([target_build_wheels, DEFAULT_TARGET], subproject.build_dirs) From 42cc7f812045daeed5abb4dc2b2cb336065258cf Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 6 Sep 2023 19:26:42 +0200 Subject: [PATCH 20/39] Add build target for running automated tests. --- scons/modules.py | 7 +++++++ scons/sconstruct.py | 8 +++++++- scons/testing.py | 25 +++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 scons/testing.py diff --git a/scons/modules.py b/scons/modules.py index 6e25335299..492885f6c7 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -94,6 +94,13 @@ def source_dir(self) -> str: """ return path.join(self.root_dir, 'mlrl') + @property + def test_dir(self) -> str: + """ + The directory that contains the subproject's automated tests. + """ + return path.join(self.root_dir, 'tests') + @property def dist_dir(self) -> str: """ diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 457b707f8d..cf9d939b46 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -13,6 +13,7 @@ from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels from run import install_runtime_dependencies +from testing import run_tests from SCons.Script import COMMAND_LINE_TARGETS from SCons.Script.SConscript import SConsEnvironment @@ -42,12 +43,13 @@ def __print_if_clean(environment, message: str): TARGET_NAME_INSTALL_CYTHON = TARGET_NAME_INSTALL + '_cython' TARGET_NAME_BUILD_WHEELS = 'build_wheels' TARGET_NAME_INSTALL_WHEELS = 'install_wheels' +TARGET_NAME_TESTS = 'tests' VALID_TARGETS = { TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_FORMAT, TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, - TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS + TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS } DEFAULT_TARGET = 'undefined' @@ -158,3 +160,7 @@ def __print_if_clean(environment, message: str): for subproject in PYTHON_MODULE.find_subprojects(): env.Clean([target_build_wheels, DEFAULT_TARGET], subproject.build_dirs) + +# Define targets for running automated tests... +target_test = __create_phony_target(env, TARGET_NAME_TESTS, action=run_tests) +env.Depends(target_test, target_install_wheels) diff --git a/scons/testing.py b/scons/testing.py new file mode 100644 index 0000000000..30a5b9d753 --- /dev/null +++ b/scons/testing.py @@ -0,0 +1,25 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for running automated tests. +""" +from os import path + +from modules import PYTHON_MODULE +from run import run_python_program + + +def __run_python_tests(directory: str): + run_python_program('unittest', 'discover', '-v', '-f', '-s', directory) + + +def run_tests(**_): + """ + Runs all automated tests. + """ + for subproject in PYTHON_MODULE.find_subprojects(): + test_dir = subproject.test_dir + + if path.isdir(test_dir): + print('Running automated tests for subpackage "' + subproject.name + '"...') + __run_python_tests(test_dir) From 191b12572789e96feb19b3668c2bde6bfd0e8d09 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 6 Sep 2023 19:27:00 +0200 Subject: [PATCH 21/39] Edit isort config. --- .isort.cfg | 2 +- scons/sconstruct.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.isort.cfg b/.isort.cfg index 31528f3cdc..e42dcea7db 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -4,7 +4,7 @@ line_length=120 group_by_package=true known_first_party=mlrl known_third_party=sklearn,scipy,numpy,tabulate,arff,SCons -forced_separate=mlrl.common,mlrl.boosting,mlrl.seco,mlrl.testbed +forced_separate=mlrl.common,mlrl.boosting,mlrl.seco,mlrl.testbed,SCons lines_between_types=1 order_by_type=true multi_line_output=2 diff --git a/scons/sconstruct.py b/scons/sconstruct.py index cf9d939b46..fb20f2b3ed 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -14,6 +14,7 @@ from packaging import build_python_wheel, install_python_wheels from run import install_runtime_dependencies from testing import run_tests + from SCons.Script import COMMAND_LINE_TARGETS from SCons.Script.SConscript import SConsEnvironment From dd51efe939ba710373bb3784ad0551b7adcd3b84 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 6 Sep 2023 20:17:18 +0200 Subject: [PATCH 22/39] A build targets for generating the documentation. --- doc/Doxyfile_boosting | 4 +- doc/Doxyfile_common | 4 +- scons/documentation.py | 90 +++++++++++++++ scons/modules.py | 246 ++++++++++++++++++++++++++++++++++++++--- scons/sconstruct.py | 53 ++++++++- 5 files changed, 378 insertions(+), 19 deletions(-) create mode 100644 scons/documentation.py diff --git a/doc/Doxyfile_boosting b/doc/Doxyfile_boosting index 9aac1eb085..6e017e88c1 100644 --- a/doc/Doxyfile_boosting +++ b/doc/Doxyfile_boosting @@ -68,7 +68,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = apidoc/api/cpp/boosting/ +OUTPUT_DIRECTORY = doc/apidoc/api/cpp/boosting/ # If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 # sub-directories (in 2 levels) under the output directory of each output format @@ -943,7 +943,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = ../cpp/subprojects/boosting/ +INPUT = cpp/subprojects/boosting/ # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/doc/Doxyfile_common b/doc/Doxyfile_common index 2ecc170179..2337f1cd91 100644 --- a/doc/Doxyfile_common +++ b/doc/Doxyfile_common @@ -68,7 +68,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = apidoc/api/cpp/common/ +OUTPUT_DIRECTORY = doc/apidoc/api/cpp/common/ # If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 # sub-directories (in 2 levels) under the output directory of each output format @@ -943,7 +943,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = ../cpp/subprojects/common/ +INPUT = cpp/subprojects/common/ # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/scons/documentation.py b/scons/documentation.py new file mode 100644 index 0000000000..911721d365 --- /dev/null +++ b/scons/documentation.py @@ -0,0 +1,90 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for generating the documentation. +""" +from os import makedirs, path +from typing import List, Optional + +from modules import DOC_MODULE +from run import run_program + + +def __doxygen(config_file: str, output_dir: str): + makedirs(output_dir, exist_ok=True) + run_program('doxygen', config_file, print_args=True) + + +def __sphinx_apidoc(source_dir: str, output_dir: str): + run_program('sphinx-apidoc', + '--tocfile', + 'index', + '-f', + '-o', + output_dir, + source_dir, + '**/cython', + print_args=True, + additional_dependencies=['sphinx', 'furo'], + requirements_file=DOC_MODULE.requirements_file) + + +def __sphinx_build(source_dir: str, output_dir: str, additional_dependencies: Optional[List[str]] = None): + run_program('sphinx-build', + '-M', + 'html', + source_dir, + output_dir, + print_args=True, + additional_dependencies=additional_dependencies, + requirements_file=DOC_MODULE.requirements_file) + + +# pylint: disable=unused-argument +def apidoc_cpp(env, target, source): + """ + Builds the API documentation for a single C++ subproject. + + :param env: The scons environment + :param target: The path of the files that belong to the API documentation, if it has already been built, or the + path of the directory, where the API documentation should be stored + :param source: The paths of the source files from which the API documentation should be built + """ + if target: + apidoc_subproject = DOC_MODULE.find_cpp_apidoc_subproject(target[0].path) + config_file = apidoc_subproject.config_file + + if path.isfile(config_file): + print('Generating C++ API documentation for subproject "' + apidoc_subproject.name + '"...') + __doxygen(config_file=config_file, output_dir=path.join(apidoc_subproject.apidoc_dir)) + + +# pylint: disable=unused-argument +def apidoc_python(env, target, source): + """ + Builds the API documentation for a single Python subproject. + + :param env: The scons environment + :param target: The path of the files that belong to the API documentation, if it has already been built, or the + path of the directory, where the API documentation should be stored + :param source: The paths of the source files from which the API documentation should be built + """ + if target: + apidoc_subproject = DOC_MODULE.find_python_apidoc_subproject(target[0].path) + tmp_dir = apidoc_subproject.build_dir + + if path.isdir(tmp_dir): + print('Generating Python API documentation for subproject "' + apidoc_subproject.name + '"...') + __sphinx_apidoc(source_dir=apidoc_subproject.source_subproject.source_dir, output_dir=tmp_dir) + __sphinx_build(source_dir=tmp_dir, output_dir=apidoc_subproject.apidoc_dir) + + +def doc(**_): + """ + Builds the documentation. + """ + print('Generating documentation...') + __sphinx_build( + source_dir=DOC_MODULE.root_dir, + output_dir=DOC_MODULE.build_dir, + additional_dependencies=['sphinxext-opengraph', 'sphinx-inline-tabs', 'sphinx-copybutton', 'myst-parser']) diff --git a/scons/modules.py b/scons/modules.py index 492885f6c7..90cfd0e0c5 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -59,25 +59,24 @@ def requirements_file(self) -> str: return path.join(self.root_dir, 'requirements.txt') -class PythonModule(Module): +class SourceModule(Module, ABC): """ - Provides access to directories and files that belong to the project's Python code. + An abstract base class for all classes that provide access to directories and files that belong to a module, which + contains source code. """ - class Subproject: + class Subproject(ABC): """ - Provides access to directories and files that belong to an individual subproject that is part of the project's - Python code. + An abstract base class for all classes that provide access to directories and files that belong to an individual + subproject that is part of a module, which contains source files. """ - @staticmethod - def __filter_pycache_directories(directory: str) -> bool: - return directory != '__pycache__' - - def __init__(self, root_dir: str): + def __init__(self, parent_module: 'SourceModule', root_dir: str): """ - :param root_dir: The root directory of the subproject + :param parent_module: The `SourceModule`, the subproject belongs to + :param root_dir: The root directory of the suproject """ + self.parent_module = parent_module self.root_dir = root_dir @property @@ -87,6 +86,22 @@ def name(self) -> str: """ return path.basename(self.root_dir) + +class PythonModule(SourceModule): + """ + Provides access to directories and files that belong to the project's Python code. + """ + + class Subproject(SourceModule.Subproject): + """ + Provides access to directories and files that belong to an individual subproject that is part of the project's + Python code. + """ + + @staticmethod + def __filter_pycache_directories(directory: str) -> bool: + return directory != '__pycache__' + @property def source_dir(self) -> str: """ @@ -175,7 +190,7 @@ def find_subprojects(self) -> List[Subproject]: :return: A list that contains all subrojects that have been found """ return [ - PythonModule.Subproject(file) for file in glob(path.join(self.root_dir, 'subprojects', '*')) + PythonModule.Subproject(self, file) for file in glob(path.join(self.root_dir, 'subprojects', '*')) if path.isdir(file) ] @@ -193,15 +208,44 @@ def find_subproject(self, file: str) -> Subproject: raise ValueError('File "' + file + '" does not belong to a Python subproject') -class CppModule(Module): +class CppModule(SourceModule): """ Provides access to directories and files that belong to the project's C++ code. """ + class Subproject(SourceModule.Subproject): + """ + Provides access to directories and files that belong to an individual subproject that is part of the project's + C++ code. + """ + + def find_source_files(self) -> List[str]: + """ + Finds and returns all source files that are contained by the subproject. + + :return: A list that contains the paths of the source files that have been found + """ + + def file_filter(file) -> bool: + return file.endswith('.hpp') or file.endswith('.cpp') + + return find_files_recursively(self.root_dir, file_filter=file_filter) + @property def root_dir(self) -> str: return 'cpp' + def find_subprojects(self) -> List[Subproject]: + """ + Finds and returns all subprojects that are part of the C++ code. + + :return: A list that contains all subrojects that have been found + """ + return [ + CppModule.Subproject(self, file) for file in glob(path.join(self.root_dir, 'subprojects', '*')) + if path.isdir(file) + ] + class BuildModule(Module): """ @@ -213,8 +257,184 @@ def root_dir(self) -> str: return 'scons' +class DocumentationModule(Module): + """ + Provides access to directories and files that belong to the project's documentation. + """ + + class ApidocSubproject(ABC): + """ + An abstract base class for all classes that provide access to directories and files that are needed for building + the API documentation of a certain C++ or Python subproject. + """ + + def __init__(self, parent_module: 'DocumentationModule', source_subproject: SourceModule.Subproject): + """ + :param parent_module: The `DocumentationModule` this subproject belongs to + :param source_subproject: The subproject of which the API documentation should be built + """ + self.parent_module = parent_module + self.source_subproject = source_subproject + + @property + def name(self) -> str: + """ + The name of the subproject of which the API documentation should be built. + """ + return self.source_subproject.name + + @property + def apidoc_dir(self) -> str: + """ + The directory, where the API documentation should be stored. + """ + return path.join(self.parent_module.apidoc_dir, 'api', self.source_subproject.parent_module.root_dir, + self.name) + + def find_apidoc_files(self) -> List[str]: + """ + Finds and returns all files that belong to the API documentation that has been built. + + :return: A list that contains the paths of the build files that have been found + """ + return find_files_recursively(self.apidoc_dir) + + class CppApidocSubproject(ApidocSubproject): + """ + Provides access to the directories and files that are necessary for building the API documentation of a certain + C++ subproject. + """ + + @property + def config_file(self) -> str: + """ + The config file, which should be used for building the API documentation. + """ + return path.join(self.parent_module.root_dir, 'Doxyfile_' + self.name) + + class PythonApidocSubproject(ApidocSubproject): + """ + Provides access to the directories and files that are necessary for building the API documentation of a certain + Python subproject. + """ + + @property + def config_file(self) -> str: + """ + The config file, which should be used for building the API documentation. + """ + return path.join(self.build_dir, 'conf.py') + + @property + def build_dir(self) -> str: + """ + The directory, where build files should be stored. + """ + return path.join(self.parent_module.root_dir, 'python', self.name) + + @property + def root_dir(self) -> str: + return 'doc' + + @property + def config_file(self) -> str: + """ + The config file that should be used for building the documentation. + """ + return path.join(self.root_dir, 'conf.py') + + @property + def apidoc_dir(self) -> str: + """ + The directory, where API documentations should be stored. + """ + return path.join(self.root_dir, 'apidoc') + + @property + def build_dir(self) -> str: + """ + The directory, where the documentation should be stored. + """ + return path.join(self.root_dir, '_build') + + def find_build_files(self) -> List[str]: + """ + Finds and returns all files that belong to the documentation that has been built. + + :return: A list that contains the paths of the build files that have been found + """ + return find_files_recursively(self.build_dir) + + def find_source_files(self) -> List[str]: + """ + Finds and returns all source files from which the documentation is built. + + :return: A list that contains the paths of the source files that have been found + """ + + def directory_filter(directory: str) -> bool: + return directory != path.basename(self.build_dir) \ + and directory != path.basename(self.apidoc_dir) \ + and directory != 'python' + + def file_filter(file: str) -> bool: + return not file.startswith('Doxyfile') and not file == 'requirements.txt' and not file == 'conf.py' + + return find_files_recursively(self.root_dir, directory_filter=directory_filter, file_filter=file_filter) + + def get_cpp_apidoc_subproject(self, cpp_subproject: CppModule.Subproject) -> CppApidocSubproject: + """ + Returns a `CppApidocSubproject` for building the API documentation of a given C++ subproject. + + :param cpp_subproject: The C++ subproject of which the API documentation should be built + :return: A `CppApidocSubproject` + """ + return DocumentationModule.CppApidocSubproject(self, cpp_subproject) + + def get_python_apidoc_subproject(self, python_subproject: PythonModule.Subproject) -> PythonApidocSubproject: + """ + Returns a `PythonApidocSubproject` for building the API documentation of a given Python subproject. + + :param python_subproject: The Python subproject of which the API documentation should be built + :return: A `PythonApidocSubproject` + """ + return DocumentationModule.PythonApidocSubproject(self, python_subproject) + + def find_cpp_apidoc_subproject(self, file: str) -> CppApidocSubproject: + """ + Finds and returns the `CppApidocSubproject` to which a given file belongs. + + :param file: The path of the file + :return: The `CppApiSubproject` to which the given file belongs + """ + for subproject in CPP_MODULE.find_subprojects(): + apidoc_subproject = self.get_cpp_apidoc_subproject(subproject) + + if file.startswith(apidoc_subproject.apidoc_dir): + return apidoc_subproject + + raise ValueError('File "' + file + '" does not belong to a C++ API documentation subproject') + + def find_python_apidoc_subproject(self, file: str) -> PythonApidocSubproject: + """ + Finds and returns the `PythonApidocSubproject` to which a given file belongs. + + :param file: The path of the file + :return: The `PythonApidocSubproject` to which the given file belongs + """ + for subproject in PYTHON_MODULE.find_subprojects(): + apidoc_subproject = self.get_python_apidoc_subproject(subproject) + + if file.startswith(apidoc_subproject.apidoc_dir): + return apidoc_subproject + + raise ValueError('File "' + file + '" does not belong to a Python API documentation subproject') + + BUILD_MODULE = BuildModule() PYTHON_MODULE = PythonModule() CPP_MODULE = CppModule() + +DOC_MODULE = DocumentationModule() diff --git a/scons/sconstruct.py b/scons/sconstruct.py index fb20f2b3ed..d7d896d8fd 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -10,7 +10,8 @@ from code_style import check_cpp_code_style, check_python_code_style, enforce_cpp_code_style, enforce_python_code_style from compilation import compile_cpp, compile_cython, install_cpp, install_cython, setup_cpp, setup_cython -from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE +from documentation import apidoc_cpp, apidoc_python, doc +from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels from run import install_runtime_dependencies from testing import run_tests @@ -45,12 +46,17 @@ def __print_if_clean(environment, message: str): TARGET_NAME_BUILD_WHEELS = 'build_wheels' TARGET_NAME_INSTALL_WHEELS = 'install_wheels' TARGET_NAME_TESTS = 'tests' +TARGET_NAME_APIDOC = 'apidoc' +TARGET_NAME_APIDOC_CPP = TARGET_NAME_APIDOC + '_cpp' +TARGET_NAME_APIDOC_PYTHON = TARGET_NAME_APIDOC + '_python' +TARGET_NAME_DOC = 'doc' VALID_TARGETS = { TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_FORMAT, TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, - TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS + TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, + TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC } DEFAULT_TARGET = 'undefined' @@ -165,3 +171,46 @@ def __print_if_clean(environment, message: str): # Define targets for running automated tests... target_test = __create_phony_target(env, TARGET_NAME_TESTS, action=run_tests) env.Depends(target_test, target_install_wheels) + +# Define targets for generating the documentation... +commands_apidoc_cpp = [] +commands_apidoc_python = [] + +for subproject in CPP_MODULE.find_subprojects(): + apidoc_subproject = DOC_MODULE.get_cpp_apidoc_subproject(subproject) + config_file = apidoc_subproject.config_file + + if path.isfile(config_file): + apidoc_files = apidoc_subproject.find_apidoc_files() + targets_apidoc_cpp = apidoc_files if apidoc_files else apidoc_subproject.apidoc_dir + source_files = [config_file] + subproject.find_source_files() + command_apidoc_cpp = env.Command(targets_apidoc_cpp, source_files, action=apidoc_cpp) + commands_apidoc_cpp.append(command_apidoc_cpp) + +target_apidoc_cpp = env.Alias(TARGET_NAME_APIDOC_CPP, None, None) +env.Depends(target_apidoc_cpp, commands_apidoc_cpp) + +for subproject in PYTHON_MODULE.find_subprojects(): + apidoc_subproject = DOC_MODULE.get_python_apidoc_subproject(subproject) + config_file = apidoc_subproject.config_file + + if path.isfile(config_file): + apidoc_files = apidoc_subproject.find_apidoc_files() + targets_apidoc_python = apidoc_files if apidoc_files else apidoc_subproject.apidoc_dir + source_files = [config_file] + subproject.find_source_files() + command_apidoc_python = env.Command(targets_apidoc_python, source_files, action=apidoc_python) + env.Depends(command_apidoc_python, target_install_wheels) + commands_apidoc_python.append(command_apidoc_python) + +target_apidoc_python = env.Alias(TARGET_NAME_APIDOC_PYTHON, None, None) +env.Depends(target_apidoc_python, commands_apidoc_python) + +target_apidoc = env.Alias(TARGET_NAME_APIDOC, None, None) +env.Depends(target_apidoc, [target_apidoc_cpp, target_apidoc_python]) + +doc_files = DOC_MODULE.find_build_files() +targets_doc = doc_files if doc_files else DOC_MODULE.build_dir +command_doc = env.Command(targets_doc, [DOC_MODULE.config_file] + DOC_MODULE.find_source_files(), action=doc) +env.Depends(command_doc, target_apidoc) +target_doc = env.Alias(TARGET_NAME_DOC, None, None) +env.Depends(target_doc, command_doc) From 55f3d7a707f018ca210a9152ad330001726945fd Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 6 Sep 2023 20:21:36 +0200 Subject: [PATCH 23/39] A build targets for removing the documentation. --- scons/modules.py | 15 +++++++++++++++ scons/sconstruct.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/scons/modules.py b/scons/modules.py index 90cfd0e0c5..470756d98a 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -299,6 +299,14 @@ def find_apidoc_files(self) -> List[str]: """ return find_files_recursively(self.apidoc_dir) + def find_build_files(self) -> List[str]: + """ + Finds and returns all build files that have been created when building the API documentation. + + :return: A list that contains the paths of all build files that have been found + """ + return [self.apidoc_dir] + class CppApidocSubproject(ApidocSubproject): """ Provides access to the directories and files that are necessary for building the API documentation of a certain @@ -332,6 +340,13 @@ def build_dir(self) -> str: """ return path.join(self.parent_module.root_dir, 'python', self.name) + def find_build_files(self) -> List[str]: + + def file_filter(file) -> bool: + return file.endswith('.rst') + + return find_files_recursively(self.build_dir, file_filter=file_filter) + super().find_build_files() + @property def root_dir(self) -> str: return 'doc' diff --git a/scons/sconstruct.py b/scons/sconstruct.py index d7d896d8fd..0119ec8cb3 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -214,3 +214,31 @@ def __print_if_clean(environment, message: str): env.Depends(command_doc, target_apidoc) target_doc = env.Alias(TARGET_NAME_DOC, None, None) env.Depends(target_doc, command_doc) + +# Define target for cleaning up the documentation and associated build directories... +if not COMMAND_LINE_TARGETS \ + or TARGET_NAME_APIDOC_CPP in COMMAND_LINE_TARGETS \ + or TARGET_NAME_APIDOC in COMMAND_LINE_TARGETS \ + or TARGET_NAME_DOC in COMMAND_LINE_TARGETS: + __print_if_clean(env, 'Removing C++ API documentation...') + + for subproject in CPP_MODULE.find_subprojects(): + apidoc_subproject = DOC_MODULE.get_cpp_apidoc_subproject(subproject) + env.Clean([target_apidoc_cpp, DEFAULT_TARGET], apidoc_subproject.find_build_files()) + +if not COMMAND_LINE_TARGETS \ + or TARGET_NAME_APIDOC_PYTHON in COMMAND_LINE_TARGETS \ + or TARGET_NAME_APIDOC in COMMAND_LINE_TARGETS \ + or TARGET_NAME_DOC in COMMAND_LINE_TARGETS: + __print_if_clean(env, 'Removing Python API documentation...') + + for subproject in PYTHON_MODULE.find_subprojects(): + apidoc_subproject = DOC_MODULE.get_python_apidoc_subproject(subproject) + env.Clean([target_apidoc_python, DEFAULT_TARGET], apidoc_subproject.find_build_files()) + +if not COMMAND_LINE_TARGETS or TARGET_NAME_APIDOC in COMMAND_LINE_TARGETS or TARGET_NAME_DOC in COMMAND_LINE_TARGETS: + env.Clean([target_apidoc, DEFAULT_TARGET], DOC_MODULE.apidoc_dir) + +if not COMMAND_LINE_TARGETS or TARGET_NAME_DOC in COMMAND_LINE_TARGETS: + __print_if_clean(env, 'Removing documentation...') + env.Clean([target_doc, DEFAULT_TARGET], DOC_MODULE.build_dir) From 23a92bf3dceddfcb71bfbac84968b6b0aebde0d1 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 6 Sep 2023 20:22:24 +0200 Subject: [PATCH 24/39] Specify the default target. --- scons/sconstruct.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 0119ec8cb3..e494a7dbed 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -59,7 +59,7 @@ def __print_if_clean(environment, message: str): TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC } -DEFAULT_TARGET = 'undefined' +DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS # Raise an error if any invalid targets are given... invalid_targets = [target for target in COMMAND_LINE_TARGETS if target not in VALID_TARGETS] @@ -242,3 +242,6 @@ def __print_if_clean(environment, message: str): if not COMMAND_LINE_TARGETS or TARGET_NAME_DOC in COMMAND_LINE_TARGETS: __print_if_clean(env, 'Removing documentation...') env.Clean([target_doc, DEFAULT_TARGET], DOC_MODULE.build_dir) + +# Set the default target... +env.Default(DEFAULT_TARGET) From 3206fe959ffb1ea0781ceea7a051f5de1ffcbfc6 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 6 Sep 2023 20:25:15 +0200 Subject: [PATCH 25/39] Remove build dependencies from requirements.txt file. --- python/requirements.txt | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/python/requirements.txt b/python/requirements.txt index ebe2cdb075..46658ec398 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,13 +1,5 @@ -cython >= 3.0, < 3.1 -meson >= 1.2, < 1.3 -ninja >= 1.11, < 1.12 -build >= 1.0, < 1.1 -pylint >= 2.17, < 2.18 -yapf >= 0.40, < 0.41 -isort >= 5.12, < 5.13 -clang-format >= 16.0, < 16.1 +liac-arff >= 2.5, < 2.6 numpy >= 1.25, < 1.26 -scipy >= 1.11, < 1.12 scikit-learn >=1.3, < 1.4 -liac-arff >= 2.5, < 2.6 +scipy >= 1.11, < 1.12 tabulate >= 0.9, < 0.10 From 5027982058f6ebc17ccc6263a606061c7fb82454 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 6 Sep 2023 20:25:25 +0200 Subject: [PATCH 26/39] Edit .gitignore file. --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index e9b7d96c8c..bb6f86eee4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ python/**/cython/*.lib python/**/cython/*.pyd python/**/tests/res/tmp/ cpp/build/ -.cpp_files.tmp # Documentation files doc/_build/ From 7cb17479ed1689c3d3819f58530720ea759e1ce2 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Sun, 10 Sep 2023 14:50:53 +0200 Subject: [PATCH 27/39] Use new build system for Linux and MacOS builds in Github workflows. --- .github/workflows/test_build.yml | 12 ++++++------ .github/workflows/test_format.yml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 3a0eaac994..8beb662149 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -8,8 +8,8 @@ on: - '**/*.pyx' - '**/*.py' - '**/*.build' - - 'Makefile' - - 'python/requirements.txt' + - 'build' + - '**/requirements.txt' - 'doc/**' - 'python/subprojects/testbed/tests/**' - '.github/workflows/test_build.yml' @@ -26,15 +26,15 @@ jobs: sudo apt install -y opencl-headers ocl-icd-opencl-dev - name: Compile via GCC run: | - make compile + ./build compile - name: Run Tests run: | - make tests + ./build tests - name: Install Doxygen uses: ssciwr/doxygen-install@v1 - name: Generate Documentation run: | - make doc + ./build doc macos_build: name: Test MacOS build runs-on: macos-latest @@ -50,7 +50,7 @@ jobs: brew install opencl-clhpp-headers - name: Compile via Clang run: | - CPLUS_INCLUDE_PATH=/usr/local/opt/opencl-clhpp-headers/include make compile + CPLUS_INCLUDE_PATH=/usr/local/opt/opencl-clhpp-headers/include ./build compile windows_build: name: Test Windows build runs-on: windows-latest diff --git a/.github/workflows/test_format.yml b/.github/workflows/test_format.yml index 104d6a9d34..b736bfeddc 100644 --- a/.github/workflows/test_format.yml +++ b/.github/workflows/test_format.yml @@ -5,7 +5,7 @@ on: - '**/*.hpp' - '**/*.cpp' - '**/*.py' - - 'Makefile' + - 'build' - '.clang-format' - '.isort.cfg' - '.style.yapf' @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 - name: Check C++ code style run: | - make test_format_cpp + ./build test_format_cpp - name: Check Python code style run: | - make test_format_python + ./build test_format_python From b7efef41b8cf873295b84c812b2119ae779d4384 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Sun, 10 Sep 2023 14:55:23 +0200 Subject: [PATCH 28/39] Fix command for activating virtual environment. --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 8ba0ee5610..6249fabbd1 100755 --- a/build +++ b/build @@ -19,7 +19,7 @@ if [ ! -d $VENV_DIR ] && [ $CLEAN = false ]; then fi if [ -d "$VENV_DIR" ]; then - source $VENV_DIR/bin/activate \ + . $VENV_DIR/bin/activate \ && python -c "import sys; sys.path.append('$SCONS_DIR'); import run; run.install_build_dependencies('scons')" \ && scons --enable-virtualenv --silent --file $SCONS_DIR/sconstruct.py $@ \ && deactivate From 168d87efff76fd40bf943bdf4fa3adfa2c8dbf8e Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Sun, 10 Sep 2023 15:04:07 +0200 Subject: [PATCH 29/39] Explicitly specify Python version. --- build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build b/build index 6249fabbd1..3b4a086584 100755 --- a/build +++ b/build @@ -15,12 +15,12 @@ fi if [ ! -d $VENV_DIR ] && [ $CLEAN = false ]; then echo "Creating virtual Python environment..." - python -m venv ${VENV_DIR} + python3 -m venv ${VENV_DIR} fi if [ -d "$VENV_DIR" ]; then . $VENV_DIR/bin/activate \ - && python -c "import sys; sys.path.append('$SCONS_DIR'); import run; run.install_build_dependencies('scons')" \ + && python3 -c "import sys; sys.path.append('$SCONS_DIR'); import run; run.install_build_dependencies('scons')" \ && scons --enable-virtualenv --silent --file $SCONS_DIR/sconstruct.py $@ \ && deactivate fi From 2dd49e08b597097f502bacf07a3c75b6727901fc Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Sun, 10 Sep 2023 16:32:16 +0200 Subject: [PATCH 30/39] Install Roboto font before generating the documentation. --- .github/workflows/test_build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 8beb662149..6318aba264 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -32,6 +32,9 @@ jobs: ./build tests - name: Install Doxygen uses: ssciwr/doxygen-install@v1 + - name: Install Roboto font + run: | + sudo apt install -y fonts-roboto - name: Generate Documentation run: | ./build doc From 7ba4faa7bd887730be3cb598f5c097b03e54cb29 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 13 Sep 2023 15:10:22 +0200 Subject: [PATCH 31/39] Add bash script for running the build system on Windows --- build.bat | 35 +++++++++++++++++++++++++++++++++++ scons/run.py | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 build.bat diff --git a/build.bat b/build.bat new file mode 100644 index 0000000000..2d391c43b8 --- /dev/null +++ b/build.bat @@ -0,0 +1,35 @@ +@echo off + +set "VENV_DIR=venv" +set "SCONS_DIR=scons" +set "CLEAN=false" + +if not "%1"=="" if "%2"=="" ( + if "%1"=="--clean" ( + set "CLEAN=true" + ) + if "%1"=="-c" ( + set "CLEAN=true" + ) +) + +if not exist "%VENV_DIR%" if "%CLEAN%"=="false" ( + echo Creating virtual Python environment... + python -m venv "%VENV_DIR%" +) + +if exist "%VENV_DIR%" ( + call %VENV_DIR%\Scripts\activate ^ + && python -c "import sys;sys.path.append('%SCONS_DIR%'); import run; run.install_build_dependencies('scons')" ^ + && scons --enable-virtualenv --silent --file %SCONS_DIR%\sconstruct.py %* ^ + && call deactivate +) + +if "%CLEAN%"=="true" if exist "%VENV_DIR%" ( + echo Removing virtual Python environment... + rd /s /q "%VENV_DIR%" + + if exist "%SCONS_DIR%\build" ( + rd /s /q "%SCONS_DIR%\build" + ) +) diff --git a/scons/run.py b/scons/run.py index a28428e971..f31550266b 100644 --- a/scons/run.py +++ b/scons/run.py @@ -17,7 +17,7 @@ def __run_command(cmd: str, *args, print_args: bool = False): cmd_formatted = cmd + (reduce(lambda aggr, argument: aggr + ' ' + argument, args, '') if print_args else '') print('Running external command "' + cmd_formatted + '"...') - cmd_args = [cmd] + cmd_args = [path.join(path.dirname(sys.executable), cmd)] for arg in args: cmd_args.append(str(arg)) From f2216f75ca70342adca3f8cd9205bfdcdad4e6d5 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 13 Sep 2023 15:16:27 +0200 Subject: [PATCH 32/39] Use new build system for Windows builds in Github workflows. --- .github/workflows/test_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 6318aba264..326ef9525c 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -76,4 +76,4 @@ jobs: run: | $env:INCLUDE += ";$($pwd.Path)\vcpkg\packages\opencl_x64-windows\include" $env:LIB += ";$($pwd.Path)\vcpkg\packages\opencl_x64-windows\lib" - make compile + ./build.bat compile From ec5fb2dad329a0f18da5b02c3406bc33d886ec62 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 13 Sep 2023 15:26:23 +0200 Subject: [PATCH 33/39] Omit argument "--enable-virtualenv" when running SCons. --- build | 2 +- build.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build b/build index 3b4a086584..b451e68ce1 100755 --- a/build +++ b/build @@ -21,7 +21,7 @@ fi if [ -d "$VENV_DIR" ]; then . $VENV_DIR/bin/activate \ && python3 -c "import sys; sys.path.append('$SCONS_DIR'); import run; run.install_build_dependencies('scons')" \ - && scons --enable-virtualenv --silent --file $SCONS_DIR/sconstruct.py $@ \ + && scons --silent --file $SCONS_DIR/sconstruct.py $@ \ && deactivate fi diff --git a/build.bat b/build.bat index 2d391c43b8..15000b1557 100644 --- a/build.bat +++ b/build.bat @@ -21,7 +21,7 @@ if not exist "%VENV_DIR%" if "%CLEAN%"=="false" ( if exist "%VENV_DIR%" ( call %VENV_DIR%\Scripts\activate ^ && python -c "import sys;sys.path.append('%SCONS_DIR%'); import run; run.install_build_dependencies('scons')" ^ - && scons --enable-virtualenv --silent --file %SCONS_DIR%\sconstruct.py %* ^ + && scons --silent --file %SCONS_DIR%\sconstruct.py %* ^ && call deactivate ) From 9decf142728710179cbf8dfe3135e38cf1c8d3c0 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 13 Sep 2023 15:50:37 +0200 Subject: [PATCH 34/39] Set exit code in build scripts. --- build | 12 ++++++++---- build.bat | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/build b/build index b451e68ce1..3223efbcba 100755 --- a/build +++ b/build @@ -3,6 +3,7 @@ VENV_DIR="venv" SCONS_DIR="scons" CLEAN=false +EXIT_CODE=0 if [ $# -eq 1 ]; then if [ $1 = "--clean" ]; then @@ -19,10 +20,11 @@ if [ ! -d $VENV_DIR ] && [ $CLEAN = false ]; then fi if [ -d "$VENV_DIR" ]; then - . $VENV_DIR/bin/activate \ - && python3 -c "import sys; sys.path.append('$SCONS_DIR'); import run; run.install_build_dependencies('scons')" \ - && scons --silent --file $SCONS_DIR/sconstruct.py $@ \ - && deactivate + . $VENV_DIR/bin/activate + python3 -c "import sys; sys.path.append('$SCONS_DIR'); import run; run.install_build_dependencies('scons')" + scons --silent --file $SCONS_DIR/sconstruct.py $@ + EXIT_CODE=$? + deactivate fi if [ $CLEAN = true ] && [ -d $VENV_DIR ]; then @@ -30,3 +32,5 @@ if [ $CLEAN = true ] && [ -d $VENV_DIR ]; then rm -rf $VENV_DIR rm -rf $SCONS_DIR/build fi + +exit $EXIT_CODE diff --git a/build.bat b/build.bat index 15000b1557..36c8b9d15c 100644 --- a/build.bat +++ b/build.bat @@ -3,6 +3,7 @@ set "VENV_DIR=venv" set "SCONS_DIR=scons" set "CLEAN=false" +set "EXIT_CODE=0" if not "%1"=="" if "%2"=="" ( if "%1"=="--clean" ( @@ -19,10 +20,11 @@ if not exist "%VENV_DIR%" if "%CLEAN%"=="false" ( ) if exist "%VENV_DIR%" ( - call %VENV_DIR%\Scripts\activate ^ - && python -c "import sys;sys.path.append('%SCONS_DIR%'); import run; run.install_build_dependencies('scons')" ^ - && scons --silent --file %SCONS_DIR%\sconstruct.py %* ^ - && call deactivate + call %VENV_DIR%\Scripts\activate + python -c "import sys;sys.path.append('%SCONS_DIR%'); import run; run.install_build_dependencies('scons')" + scons --silent --file %SCONS_DIR%\sconstruct.py %* + set "EXIT_CODE=%ERRORLEVEL%" + call deactivate ) if "%CLEAN%"=="true" if exist "%VENV_DIR%" ( @@ -33,3 +35,5 @@ if "%CLEAN%"=="true" if exist "%VENV_DIR%" ( rd /s /q "%SCONS_DIR%\build" ) ) + +exit /b "%EXIT_CODE%" From 8f8ab8c079c8a5a67faade9646e4dd24f146c04e Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 13 Sep 2023 15:57:23 +0200 Subject: [PATCH 35/39] Update changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4fa2a0658..c09a353867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ A major update to the BOOMER algorithm that introduces the following changes. * The documentation has been updated to a more modern theme supporting light and dark theme variants. * A build option that allows to disable multi-threading support via OpenMP at compile-time has been added. * The Python code is now checked for common issues by applying `pylint` via continuous integration. +* The Makefile has been replaced with wrapper scripts triggering a [SCons](https://scons.org/) build. ## Version 0.9.0 (Jul. 2nd, 2023) From 57ae7ac7d936da1b26475931833ce52812155679 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 13 Sep 2023 16:06:42 +0200 Subject: [PATCH 36/39] Update Github workflow for publishing packages on PyPi. --- .github/workflows/publish.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 12c5dd1e21..f6bf8674a3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -41,6 +41,20 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + - name: Install OpenCL + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y opencl-headers ocl-icd-opencl-dev + - name: Install OpenMP + if: matrix.os == 'macos-latest' + run: | + brew install libomp + brew link libomp --force + - name: Install OpenCL + if: matrix.os == 'macos-latest' + run: | + brew install opencl-clhpp-headers - name: Prepare MSVC if: matrix.os == 'windows-latest' uses: ilammy/msvc-dev-cmd@v1 @@ -52,8 +66,9 @@ jobs: - name: Build package mlrl-common uses: pypa/cibuildwheel@v2 env: - CIBW_BEFORE_ALL_MACOS: brew install libomp && brew link libomp --force - CIBW_BEFORE_BUILD: make clean install_cpp install_cython + CIBW_BEFORE_BUILD_LINUX: ./build --clean && ./build install + CIBW_BEFORE_BUILD_MACOS: ./build --clean && CPLUS_INCLUDE_PATH=/usr/local/opt/opencl-clhpp-headers/include ./build install + CIBW_BEFORE_BUILD_WINDOWS: ./build.bat --clean && ./build.bat install CIBW_BUILD_FRONTEND: build CIBW_ARCHS: auto64 CIBW_SKIP: 'pp* *musllinux*' @@ -62,8 +77,9 @@ jobs: - name: Build package mlrl-boosting uses: pypa/cibuildwheel@v2 env: - CIBW_BEFORE_ALL_MACOS: brew install libomp && brew link libomp --force - CIBW_BEFORE_BUILD: make clean install_cpp install_cython + CIBW_BEFORE_BUILD_LINUX: ./build --clean && ./build install + CIBW_BEFORE_BUILD_MACOS: ./build --clean && CPLUS_INCLUDE_PATH=/usr/local/opt/opencl-clhpp-headers/include ./build install + CIBW_BEFORE_BUILD_WINDOWS: ./build.bat --clean && ./build.bat install CIBW_BUILD_FRONTEND: build CIBW_ARCHS: auto64 CIBW_SKIP: 'pp* *musllinux*' @@ -101,7 +117,7 @@ jobs: - name: Build package mlrl-common uses: pypa/cibuildwheel@v2 env: - CIBW_BEFORE_BUILD: make clean install_cpp install_cython + CIBW_BEFORE_BUILD: ./build --clean && ./build install CIBW_BUILD_FRONTEND: build CIBW_ARCHS_LINUX: aarch64 CIBW_SKIP: 'pp* *musllinux*' @@ -110,7 +126,7 @@ jobs: - name: Build package mlrl-boosting uses: pypa/cibuildwheel@v2 env: - CIBW_BEFORE_BUILD: make clean install_cpp install_cython + CIBW_BEFORE_BUILD: ./build --clean && ./build install CIBW_BUILD_FRONTEND: build CIBW_ARCHS_LINUX: aarch64 CIBW_SKIP: 'pp* *musllinux*' From 1390882c9e52d9f6191e9d6c618c93e21466ddc7 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 13 Sep 2023 16:08:50 +0200 Subject: [PATCH 37/39] Remove Makefile. --- Makefile | 246 ------------------------------------------------------- 1 file changed, 246 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index f81172a2e1..0000000000 --- a/Makefile +++ /dev/null @@ -1,246 +0,0 @@ -default_target: install -.PHONY: clean_venv clean_cpp clean_cython clean_compile clean_cpp_install clean_cython_install clean_wheel \ - clean_install clean_doc clean test_format_cpp test_format_python test_format format_cpp format_python format \ - compile_cpp compile_cython compile install_cpp install_cython wheel install apidoc_cpp apidoc_python doc - -UNAME = $(if $(filter Windows_NT,${OS}),Windows,$(shell uname)) -IS_WIN = $(filter Windows,${UNAME}) - -VENV_DIR = venv -CPP_SRC_DIR = cpp -CPP_BUILD_DIR = ${CPP_SRC_DIR}/build -CPP_PACKAGE_DIR = ${CPP_SRC_DIR}/subprojects -PYTHON_SRC_DIR = python -PYTHON_BUILD_DIR = ${PYTHON_SRC_DIR}/build -PYTHON_PACKAGE_DIR = ${PYTHON_SRC_DIR}/subprojects -DIST_DIR = dist -DOC_DIR = doc -DOC_API_DIR = ${DOC_DIR}/apidoc -DOC_TMP_DIR = ${DOC_DIR}/python -DOC_BUILD_DIR = ${DOC_DIR}/_build - -PS = powershell -Command -PYTHON = $(if ${IS_WIN},python,python3) -VENV_CREATE = ${PYTHON} -m venv ${VENV_DIR} -VENV_ACTIVATE = $(if ${IS_WIN},${PS} "${VENV_DIR}/Scripts/activate.bat;",. ${VENV_DIR}/bin/activate) -VENV_DEACTIVATE = $(if ${IS_WIN},${PS} "${VENV_DIR}/Scripts/deactivate.bat;",deactivate) -PIP_INSTALL = python -m pip install --prefer-binary -ISORT = isort --settings-path . --virtual-env ${VENV_DIR} --skip-gitignore -ISORT_DRYRUN = ${ISORT} --check -ISORT_INPLACE = ${ISORT} --overwrite-in-place -YAPF = yapf -r -p --style=.style.yapf --verbose --exclude '**/build/*.py' -YAPF_DRYRUN = ${YAPF} --diff -YAPF_INPLACE = ${YAPF} -i -PYLINT = pylint --jobs=0 --recursive=y --ignore=build --rcfile=.pylintrc -CLANG_FORMAT = clang-format --style=file --verbose -CLANG_FORMAT_DRYRUN = ${CLANG_FORMAT} -n --Werror -CLANG_FORMAT_INPLACE = ${CLANG_FORMAT} -i -MESON_SETUP = meson setup -MESON_COMPILE = meson compile -MESON_INSTALL = meson install -WHEEL_BUILD = python -m build --wheel -WHEEL_INSTALL = python -m pip install --force-reinstall --no-deps -PYTHON_UNITTEST = python -m unittest discover -v -f -s -DOXYGEN = $(if ${IS_WIN},for /f %%i in (./../VERSION) do set PROJECT_NUMBER=%%i && doxygen,PROJECT_NUMBER=${file < VERSION} doxygen) -SPHINX_APIDOC = sphinx-apidoc --tocfile index -f -SPHINX_BUILD = sphinx-build -M html - -define delete_dir - $(if ${IS_WIN},\ - ${PS} "if (Test-Path ${1}) {rm ${1} -Recurse -Force}",\ - rm -rf ${1}) -endef - -define delete_files_recursively - $(if ${IS_WIN},\ - ${PS} "rm ${1} -Recurse -Force -Include ${2}",\ - rm -f ${1}/**/${2}) -endef - -define delete_dirs_recursively - $(if ${IS_WIN},\ - ${PS} "rm ${1} -Recurse -Force -Include ${2}",\ - rm -rf ${1}/**/${2}) -endef - -define install_wheels - $(if ${IS_WIN},\ - ${PS} "${WHEEL_INSTALL} (Get-ChildItem -Path ${1} | Where Name -Match '\.whl' | Select-Object -ExpandProperty FullName);",\ - ${WHEEL_INSTALL} ${1}/*.whl) -endef - -define create_dir - $(if ${IS_WIN},\ - ${PS} "New-Item -Path ${1} -ItemType "directory" -Force",\ - mkdir -p ${1}) -endef - -define clang_format_dryrun_recursively - $(if ${IS_WIN},\ - (${PS} "Get-ChildItem -Path ${1} -Recurse | Where Name -Match '\.hpp|\.cpp' | Select-Object -ExpandProperty FullName | Out-File .cpp_files.tmp -Encoding utf8";\ - ${CLANG_FORMAT_DRYRUN} --files=.cpp_files.tmp;\ - ${PS} "rm .cpp_files.tmp -Force"),\ - find ${1} -type f \( -iname "*.hpp" -o -iname "*.cpp" \) -exec ${CLANG_FORMAT_DRYRUN} {} +) -endef - -define clang_format_inplace_recursively - $(if ${IS_WIN},\ - (${PS} "Get-ChildItem -Path ${1} -Recurse | Where Name -Match '\.hpp|\.cpp' | Select-Object -ExpandProperty FullName | Out-File .cpp_files.tmp -Encoding utf8";\ - ${CLANG_FORMAT_INPLACE} --files=.cpp_files.tmp;\ - ${PS} "rm .cpp_files.tmp -Force"),\ - find ${1} -type f \( -iname "*.hpp" -o -iname "*.cpp" \) -exec ${CLANG_FORMAT_INPLACE} {} +) -endef - -clean_venv: - @echo Removing virtual Python environment... - $(call delete_dir,${VENV_DIR}) - -clean_cpp: - @echo Removing C++ compilation files... - $(call delete_dir,${CPP_BUILD_DIR}) - -clean_cython: - @echo Removing Cython compilation files... - $(call delete_dir,${PYTHON_BUILD_DIR}) - -clean_compile: clean_cpp clean_cython - -clean_install: - @echo Removing shared libraries and extension modules from source tree... - $(call delete_files_recursively,${PYTHON_PACKAGE_DIR},*.so*) - $(call delete_files_recursively,${PYTHON_PACKAGE_DIR},*.dylib) - $(call delete_files_recursively,${PYTHON_PACKAGE_DIR},*.dll) - $(call delete_files_recursively,${PYTHON_PACKAGE_DIR},*.lib) - $(call delete_files_recursively,${PYTHON_PACKAGE_DIR},*.pyd) - -clean_wheel: - @echo Removing Python build files... - $(call delete_dirs_recursively,${PYTHON_PACKAGE_DIR},build) - $(call delete_dirs_recursively,${PYTHON_PACKAGE_DIR},${DIST_DIR}) - $(call delete_dirs_recursively,${PYTHON_PACKAGE_DIR},*.egg-info) - -clean_doc: - @echo Removing documentation... - $(call delete_dir,${DOC_BUILD_DIR}) - $(call delete_dir,${DOC_API_DIR}) - $(call delete_files_recursively,${DOC_TMP_DIR},*.rst) - -clean: clean_doc clean_wheel clean_compile clean_install clean_venv - -venv: - @echo Creating virtual Python environment... - ${VENV_CREATE} - ${VENV_ACTIVATE} \ - && ${PIP_INSTALL} -r ${PYTHON_SRC_DIR}/requirements.txt \ - && ${VENV_DEACTIVATE} - -test_format_python: venv - @echo Checking Python code style... - ${VENV_ACTIVATE} \ - && ${ISORT_DRYRUN} ${PYTHON_PACKAGE_DIR} \ - && ${YAPF_DRYRUN} ${PYTHON_PACKAGE_DIR} \ - && ${PYLINT} ${PYTHON_PACKAGE_DIR} \ - && ${VENV_DEACTIVATE} - -test_format_cpp: venv - @echo Checking C++ code style... - ${VENV_ACTIVATE} \ - && $(call clang_format_dryrun_recursively,${CPP_PACKAGE_DIR}) \ - && ${VENV_DEACTIVATE} - -test_format: test_format_python test_format_cpp - -format_python: venv - @echo Formatting Python code... - ${VENV_ACTIVATE} \ - && ${ISORT_INPLACE} ${PYTHON_PACKAGE_DIR} \ - && ${YAPF_INPLACE} ${PYTHON_PACKAGE_DIR} \ - && ${VENV_DEACTIVATE} - -format_cpp: venv - @echo Formatting C++ code... - ${VENV_ACTIVATE} \ - && $(call clang_format_inplace_recursively,${CPP_PACKAGE_DIR}) \ - && ${VENV_DEACTIVATE} - -format: format_python format_cpp - -compile_cpp: venv - @echo Compiling C++ code... - ${VENV_ACTIVATE} \ - && ${MESON_SETUP} ${CPP_BUILD_DIR} ${CPP_SRC_DIR} \ - && ${MESON_COMPILE} -C ${CPP_BUILD_DIR} \ - && ${VENV_DEACTIVATE} - -compile_cython: venv - @echo Compiling Cython code... - ${VENV_ACTIVATE} \ - && ${MESON_SETUP} ${PYTHON_BUILD_DIR} ${PYTHON_SRC_DIR} \ - && ${MESON_COMPILE} -C ${PYTHON_BUILD_DIR} \ - && ${VENV_DEACTIVATE} - -compile: compile_cpp compile_cython - -install_cpp: compile_cpp - @echo Installing shared libraries into source tree... - ${VENV_ACTIVATE} \ - && ${MESON_INSTALL} -C ${CPP_BUILD_DIR} \ - && ${VENV_DEACTIVATE} - -install_cython: compile_cython - @echo Installing extension modules into source tree... - ${VENV_ACTIVATE} \ - && ${MESON_INSTALL} -C ${PYTHON_BUILD_DIR} \ - && ${VENV_DEACTIVATE} - -wheel: install_cpp install_cython - @echo Building wheel packages... - ${VENV_ACTIVATE} \ - && ${WHEEL_BUILD} ${PYTHON_PACKAGE_DIR}/common \ - && ${WHEEL_BUILD} ${PYTHON_PACKAGE_DIR}/boosting \ - && ${WHEEL_BUILD} ${PYTHON_PACKAGE_DIR}/seco \ - && ${WHEEL_BUILD} ${PYTHON_PACKAGE_DIR}/testbed \ - && ${VENV_DEACTIVATE} - -install: wheel - @echo Installing wheel packages into virtual environment... - ${VENV_ACTIVATE} \ - && $(call install_wheels,${PYTHON_PACKAGE_DIR}/common/${DIST_DIR}) \ - && $(call install_wheels,${PYTHON_PACKAGE_DIR}/boosting/${DIST_DIR}) \ - && $(call install_wheels,${PYTHON_PACKAGE_DIR}/seco/${DIST_DIR}) \ - && $(call install_wheels,${PYTHON_PACKAGE_DIR}/testbed/${DIST_DIR}) \ - && ${VENV_DEACTIVATE} - -tests: install - @echo Running integration tests... - ${VENV_ACTIVATE} \ - && ${PYTHON_UNITTEST} ${PYTHON_PACKAGE_DIR}/testbed/tests \ - && ${VENV_DEACTIVATE} - -apidoc_cpp: - @echo Generating C++ API documentation via Doxygen... - $(call create_dir,${DOC_API_DIR}/api/cpp/common) - cd ${DOC_DIR} && ${DOXYGEN} Doxyfile_common - $(call create_dir,${DOC_API_DIR}/api/cpp/boosting) - cd ${DOC_DIR} && ${DOXYGEN} Doxyfile_boosting - -apidoc_python: install - @echo Installing documentation dependencies into virtual environment... - ${VENV_ACTIVATE} \ - && ${PIP_INSTALL} -r ${DOC_DIR}/requirements.txt \ - && ${VENV_DEACTIVATE} - @echo Generating Python API documentation via Sphinx-Apidoc... - ${VENV_ACTIVATE} \ - && ${SPHINX_APIDOC} -o ${DOC_TMP_DIR}/common ${PYTHON_PACKAGE_DIR}/common/mlrl **/cython \ - && ${SPHINX_BUILD} ${DOC_TMP_DIR}/common ${DOC_API_DIR}/api/python/common \ - && ${SPHINX_APIDOC} -o ${DOC_TMP_DIR}/boosting ${PYTHON_PACKAGE_DIR}/boosting/mlrl **/cython \ - && ${SPHINX_BUILD} ${DOC_TMP_DIR}/boosting ${DOC_API_DIR}/api/python/boosting \ - && ${SPHINX_APIDOC} -o ${DOC_TMP_DIR}/testbed ${PYTHON_PACKAGE_DIR}/testbed/mlrl \ - && ${SPHINX_BUILD} ${DOC_TMP_DIR}/testbed ${DOC_API_DIR}/api/python/testbed \ - && ${VENV_DEACTIVATE} - -doc: apidoc_cpp apidoc_python - @echo Generating Sphinx documentation... - ${VENV_ACTIVATE} \ - && ${SPHINX_BUILD} ${DOC_DIR} ${DOC_BUILD_DIR} \ - && ${VENV_DEACTIVATE} From 5c3a45c0cac7a9ff1f365ecc7b0c9136619478b0 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Wed, 13 Sep 2023 16:31:04 +0200 Subject: [PATCH 38/39] Fix invokation of programs not installed in the virtual environment. --- scons/code_style.py | 10 +++++----- scons/compilation.py | 8 ++++---- scons/documentation.py | 40 ++++++++++++++++++++-------------------- scons/run.py | 32 ++++++++++++++++++++++++++++---- 4 files changed, 57 insertions(+), 33 deletions(-) diff --git a/scons/code_style.py b/scons/code_style.py index 51dd8a729b..cf115982ff 100644 --- a/scons/code_style.py +++ b/scons/code_style.py @@ -7,7 +7,7 @@ from os import path from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE -from run import run_program +from run import run_venv_program def __isort(directory: str, enforce_changes: bool = False): @@ -16,17 +16,17 @@ def __isort(directory: str, enforce_changes: bool = False): if not enforce_changes: args.append('--check') - run_program('isort', *args, directory) + run_venv_program('isort', *args, directory) def __yapf(directory: str, enforce_changes: bool = False): args = ['-r', '-p', '--style=.style.yapf', '--exclude', '**/build/*.py', '-i' if enforce_changes else '--diff'] - run_program('yapf', *args, directory) + run_venv_program('yapf', *args, directory) def __pylint(directory: str): args = ['--jobs=0', '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', '--score=n'] - run_program('pylint', *args, directory) + run_venv_program('pylint', *args, directory) def __clang_format(directory: str, enforce_changes: bool = True): @@ -40,7 +40,7 @@ def __clang_format(directory: str, enforce_changes: bool = True): args.append('-n') args.append('--Werror') - run_program('clang-format', *args, *cpp_header_files, *cpp_source_files) + run_venv_program('clang-format', *args, *cpp_header_files, *cpp_source_files) def check_python_code_style(**_): diff --git a/scons/compilation.py b/scons/compilation.py index fb3f9b558c..5b3dcaff73 100644 --- a/scons/compilation.py +++ b/scons/compilation.py @@ -6,20 +6,20 @@ from typing import List, Optional from modules import CPP_MODULE, PYTHON_MODULE -from run import run_program +from run import run_venv_program def __meson_setup(root_dir: str, build_dir: str, dependencies: Optional[List[str]] = None): print('Setting up build directory "' + build_dir + '"...') - run_program('meson', 'setup', build_dir, root_dir, print_args=True, additional_dependencies=dependencies) + run_venv_program('meson', 'setup', build_dir, root_dir, print_args=True, additional_dependencies=dependencies) def __meson_compile(build_dir: str): - run_program('meson', 'compile', '-C', build_dir, print_args=True) + run_venv_program('meson', 'compile', '-C', build_dir, print_args=True) def __meson_install(build_dir: str): - run_program('meson', 'install', '--no-rebuild', '--only-changed', '-C', build_dir, print_args=True) + run_venv_program('meson', 'install', '--no-rebuild', '--only-changed', '-C', build_dir, print_args=True) def setup_cpp(**_): diff --git a/scons/documentation.py b/scons/documentation.py index 911721d365..8558a2ccd9 100644 --- a/scons/documentation.py +++ b/scons/documentation.py @@ -7,7 +7,7 @@ from typing import List, Optional from modules import DOC_MODULE -from run import run_program +from run import run_program, run_venv_program def __doxygen(config_file: str, output_dir: str): @@ -16,28 +16,28 @@ def __doxygen(config_file: str, output_dir: str): def __sphinx_apidoc(source_dir: str, output_dir: str): - run_program('sphinx-apidoc', - '--tocfile', - 'index', - '-f', - '-o', - output_dir, - source_dir, - '**/cython', - print_args=True, - additional_dependencies=['sphinx', 'furo'], - requirements_file=DOC_MODULE.requirements_file) + run_venv_program('sphinx-apidoc', + '--tocfile', + 'index', + '-f', + '-o', + output_dir, + source_dir, + '**/cython', + print_args=True, + additional_dependencies=['sphinx', 'furo'], + requirements_file=DOC_MODULE.requirements_file) def __sphinx_build(source_dir: str, output_dir: str, additional_dependencies: Optional[List[str]] = None): - run_program('sphinx-build', - '-M', - 'html', - source_dir, - output_dir, - print_args=True, - additional_dependencies=additional_dependencies, - requirements_file=DOC_MODULE.requirements_file) + run_venv_program('sphinx-build', + '-M', + 'html', + source_dir, + output_dir, + print_args=True, + additional_dependencies=additional_dependencies, + requirements_file=DOC_MODULE.requirements_file) # pylint: disable=unused-argument diff --git a/scons/run.py b/scons/run.py index f31550266b..4b607a7d25 100644 --- a/scons/run.py +++ b/scons/run.py @@ -15,9 +15,10 @@ def __run_command(cmd: str, *args, print_args: bool = False): - cmd_formatted = cmd + (reduce(lambda aggr, argument: aggr + ' ' + argument, args, '') if print_args else '') + cmd_formatted = path.basename(cmd) + (reduce(lambda aggr, argument: aggr + ' ' + argument, args, '') + if print_args else '') print('Running external command "' + cmd_formatted + '"...') - cmd_args = [path.join(path.dirname(sys.executable), cmd)] + cmd_args = [cmd] for arg in args: cmd_args.append(str(arg)) @@ -113,7 +114,7 @@ def run_program(program: str, additional_dependencies: Optional[List[str]] = None, requirements_file: str = BUILD_MODULE.requirements_file): """ - Runs an external program. + Runs an external program that has been installed into the virtual environment. :param program: The name of the program to be run :param args: Optional arguments that should be passed to the program @@ -130,6 +131,29 @@ def run_program(program: str, __run_command(program, *args, print_args=print_args) +def run_venv_program(program: str, + *args, + print_args: bool = False, + additional_dependencies: Optional[List[str]] = None, + requirements_file: str = BUILD_MODULE.requirements_file): + """ + Runs an external program that has been installed into the virtual environment. + + :param program: The name of the program to be run + :param args: Optional arguments that should be passed to the program + :param print_args: True, if the arguments should be included in log statements, False otherwise + :param additional_dependencies: The names of dependencies that should be installed before running the program + :param requirements_file: The path of the requirements.txt file that specifies the dependency versions + """ + dependencies = [program] + + if additional_dependencies: + dependencies.extend(additional_dependencies) + + __install_dependencies(requirements_file, *dependencies) + __run_command(path.join(path.dirname(sys.executable), program), *args, print_args=print_args) + + def run_python_program(program: str, *args, print_args: bool = False, @@ -150,4 +174,4 @@ def run_python_program(program: str, dependencies.extend(additional_dependencies) __install_dependencies(requirements_file, *dependencies) - __run_command('python', '-m', program, *args, print_args=print_args) + __run_command(path.join(path.dirname(sys.executable), 'python'), '-m', program, *args, print_args=print_args) From aed82c22423ad22e048a236b49cb38ddead68b75 Mon Sep 17 00:00:00 2001 From: Michael Rapp <michael.rapp90@googlemail.com> Date: Sun, 17 Sep 2023 17:11:42 +0200 Subject: [PATCH 39/39] Update documentation. --- doc/api/codestyle.inc.rst | 40 ++++- doc/api/compilation.inc.rst | 272 ++++++++++++++++++++++++++++------ doc/api/documentation.inc.rst | 81 +++++++++- doc/api/testing.inc.rst | 19 ++- 4 files changed, 352 insertions(+), 60 deletions(-) diff --git a/doc/api/codestyle.inc.rst b/doc/api/codestyle.inc.rst index c8193537ef..eec38971dc 100644 --- a/doc/api/codestyle.inc.rst +++ b/doc/api/codestyle.inc.rst @@ -5,20 +5,48 @@ Code Style We aim to enforce a consistent code style across the entire project. For formatting the C++ code, we employ `clang-format <https://clang.llvm.org/docs/ClangFormat.html>`__. The desired C++ code style is defined in the file ``.clang-format`` in project's root directory. Accordingly, we use `YAPF <https://github.com/google/yapf>`__ to enforce the Python code style defined in the file ``.style.yapf``. In addition, `isort <https://github.com/PyCQA/isort>`__ is used to keep the ordering of imports in Python and Cython source files consistent according to the configuration file ``.isort.cfg`` and `pylint <https://pylint.org/>`__ is used to check for common issues in the Python code according to the configuration file ``.pylintrc``. If you have modified the project's source code, you can check whether it adheres to our coding standards via the following command: -.. code-block:: text +.. tab:: Linux - make test_format + .. code-block:: text + + ./build test_format + +.. tab:: MacOS + + .. code-block:: text + + ./build test_format + +.. tab:: Windows + + .. code-block:: text + + build.bat test_format .. note:: - If you want to check for compliance with the C++ or Python code style independently, you can alternatively use the command ``make test_format_cpp`` or ``make test_format_python``. + If you want to check for compliance with the C++ or Python code style independently, you can use the build target ``test_format_cpp`` or ``test_format_python`` instead of ``test_format``. In order to automatically format the project's source files according to our style guidelines, the following command can be used: -.. code-block:: text +.. tab:: Linux + + .. code-block:: text + + ./build format + +.. tab:: MacOS + + .. code-block:: text + + ./build format + +.. tab:: Windows + + .. code-block:: text - make format + build.bat format .. note:: - If you want to format only the C++ source files, you can run the command ``make format_cpp`` instead. Accordingly, the command ``make format_python`` may be used to format only the Python source files. + If you want to format only the C++ source files, you can specify the build target ``format_cpp`` instead of ``format``. Accordingly, the target ``format_python`` may be used to format only the Python source files. Whenever any source files have been modified, a `Github Action <https://docs.github.com/en/actions>`__ is run automatically to verify if they adhere to our code style guidelines. The result of these runs can be found in the `Github repository <https://github.com/mrapp-ke/Boomer/actions>`__. diff --git a/doc/api/compilation.inc.rst b/doc/api/compilation.inc.rst index 54ca621e85..54f86f8bda 100644 --- a/doc/api/compilation.inc.rst +++ b/doc/api/compilation.inc.rst @@ -3,106 +3,288 @@ Building from Source -------------------- -As discussed in the previous section :ref:`structure`, the algorithm that is provided by this project is mostly implemented in `C++ <https://en.wikipedia.org/wiki/C%2B%2B>`__ to ensure maximum efficiency (requires C++ 14 or newer). In addition, a `Python <https://en.wikipedia.org/wiki/Python_(programming_language)>`__ wrapper that integrates the algorithm with the `scikit-learn <https://scikit-learn.org>`__ framework is provided (requires Python 3.8 or newer). To make the underlying C++ implementation accessible from within the Python code, `Cython <https://en.wikipedia.org/wiki/Cython>`__ is used (requires Cython 3.0 or newer). +As discussed in the previous section :ref:`structure`, the algorithm that is provided by this project is implemented in `C++ <https://en.wikipedia.org/wiki/C%2B%2B>`__ to ensure maximum efficiency (requires C++ 14 or newer). In addition, a `Python <https://en.wikipedia.org/wiki/Python_(programming_language)>`__ wrapper that integrates the algorithm with the `scikit-learn <https://scikit-learn.org>`__ framework is provided (requires Python 3.8 or newer). To make the underlying C++ implementation accessible from within the Python code, `Cython <https://en.wikipedia.org/wiki/Cython>`__ is used (requires Cython 3.0 or newer). -Unlike pure Python programs, the C++ and Cython source files must be compiled for a particular target platform. To ease the process of compiling the source code, the project comes with a `Makefile <https://en.wikipedia.org/wiki/Make_(software)>`__ that automates the necessary steps. In the following, we discuss the individual steps that are necessary for building the project from scratch. This is necessary if you intend to modify the library's source code. If you want to use the algorithm without any custom modifications, the :ref:`installation` of pre-built packages is usually a better choice. +Unlike pure Python programs, the C++ and Cython source files must be compiled for a particular target platform. To ease the process of compiling the source code, the project comes with a `SCons <https://scons.org/>`__ build that automates the necessary steps. In the following, we discuss the individual steps that are necessary for building the project from scratch. This is necessary if you intend to modify the library's source code. If you want to use the algorithm without any custom modifications, the :ref:`installation` of pre-built packages is usually a better choice. **Prerequisites** -As a prerequisite, a supported version of Python, a suitable C++ compiler, an implementation of the Make build automation tool, as well as libraries for multi-threading and GPU support, must be installed on the host system. The installation of these software components depends on the operation system at hand. In the following, we provide installation instructions for the supported platforms. +As a prerequisite, a supported version of Python, a suitable C++ compiler, as well as optional libraries for multi-threading and GPU support, must be available on the host system. The installation of these software components depends on the operation system at hand. In the following, we provide installation instructions for the supported platforms. -* **Linux:** Nowadays, most Linux distributions include a pre-installed version of Python 3. If this is not the case, instructions on how to install a recent Python version can be found in Python's `Beginners Guide <https://wiki.python.org/moin/BeginnersGuide/Download>`__. As noted in this guide, Python should be installed via the distribution's package manager if possible. The most common Linux distributions do also ship with `GNU Make <https://www.gnu.org/software/make/>`__ and the `GNU Compiler Collection <https://gcc.gnu.org/>`__ (GCC) by default. If this is not the case, these software packages can typically be installed via the distribution's default package manager. `OpenMP <https://en.wikipedia.org/wiki/OpenMP>`__ and `OpenCL <https://www.khronos.org/opencl/>`__, which are optionally required for multi-threading and GPU support, should be installable via the package manager as well. -* **MacOS:** Recent versions of MacOS do not include Python by default. A suitable Python version can manually be downloaded from the `project's website <https://www.python.org/downloads/macos/>`__. Alternatively, the package manager `Homebrew <https://en.wikipedia.org/wiki/Homebrew_(package_manager)>`__ can be used for installation via the command ``brew install python``. MacOS relies on the `Clang <https://en.wikipedia.org/wiki/Clang>`__ compiler for building C++ code. It is part of the `Xcode <https://developer.apple.com/support/xcode/>`__ developer toolset. In addition, if the project should be compiled with multi-threading support enabled, the `OpenMP <https://en.wikipedia.org/wiki/OpenMP>`__ library must be installed. We recommend to install it via Homebrew by running the command ``brew install libomp``. The `Xcode <https://developer.apple.com/support/xcode/>`__ developer toolset should also include `OpenCL <https://www.khronos.org/opencl/>`__, which is needed for GPU support. However, the `OpenCL C++ headers <https://github.com/KhronosGroup/OpenCL-Headers>`__ must be installed manually. The easiest way to do so is via the Homebrew command ``brew install opencl-clhpp-headers``. -* **Windows:** Python releases for Windows are available at the `project's website <https://www.python.org/downloads/windows/>`__. In addition, an implementation of the Make tool must be installed. We recommend to use `GNU Make for Windows <http://gnuwin32.sourceforge.net/>`__. For the compilation of the project's source code, the MSVC compiler must be used. It is included in the `Build Tools for Visual Studio <https://visualstudio.microsoft.com/downloads/>`__, which also includes the `OpenMP <https://en.wikipedia.org/wiki/OpenMP>`__ library. Finally, `Powershell <https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/powershell>`__ must be used to run the project's Makefile. It should be included by default on modern Windows systems. If you intend to compile the project with GPU support enabled, `OpenCL <https://www.khronos.org/opencl/>`__ must be installed manually. In order to do so, we recommend to install the package ``opencl`` via the package manager `vcpkg <https://github.com/microsoft/vcpkg>`__. +.. tab:: Linux -Additional compile- or build-time dependencies will automatically be installed when following the instructions below and must not be installed manually. + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **Python** | Nowadays, most Linux distributions include a pre-installed version of Python 3. If this is not the case, instructions on how to install a recent Python version can be found in Python’s `Beginners Guide <https://wiki.python.org/moin/BeginnersGuide/Download>`__. As noted in this guide, Python should be installed via the distribution’s package manager if possible. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **C++ compiler** | Most Linux distributions provide the `GNU Compiler Collection <https://gcc.gnu.org/>`__ (GCC), which includes a C++ compiler, as part of their software repositories. If this is the case, it can be installed via the distribution's package manager. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **OpenMP** | `OpenMP <https://en.wikipedia.org/wiki/OpenMP>`__, which is optionally required for multi-threading support, should be installable via your Linux distribution's package manager. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **OpenCL** | If the project should be compiled with GPU support enabled, `OpenCL <https://www.khronos.org/opencl/>`__ must be available. On Linux, it should be installable via your distribution's package manager. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +.. tab:: MacOS + + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **Python** | Recent versions of MacOS do not include Python by default. A suitable Python version can manually be downloaded from the `project's website <https://www.python.org/downloads/macos/>`__. Alternatively, the package manager `Homebrew <https://en.wikipedia.org/wiki/Homebrew_(package_manager)>`__ can be used for installation via the command ``brew install python``. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **C++ compiler** | MacOS relies on the `Clang <https://en.wikipedia.org/wiki/Clang>`__ compiler for building C++ code. It is part of the `Xcode <https://developer.apple.com/support/xcode/>`__ developer toolset. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **OpenMP** | If the project should be compiled with multi-threading support enabled, the `OpenMP <https://en.wikipedia.org/wiki/OpenMP>`__ library must be installed. We recommend to install it via Homebrew by running the command ``brew install libomp``. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **OpenCL** | The `Xcode <https://developer.apple.com/support/xcode/>`__ developer toolset should include `OpenCL <https://www.khronos.org/opencl/>`__, which is needed for GPU support. However, the `OpenCL C++ headers <https://github.com/KhronosGroup/OpenCL-Headers>`__ must be installed manually. The easiest way to do so is via the Homebrew command ``brew install opencl-clhpp-headers``. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +.. tab:: Windows + + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **Python** | Python releases for Windows are available at the `project's website <https://www.python.org/downloads/windows/>`__, where you can download an installer. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **C++ compiler** | For the compilation of the project's source code, the MSVC compiler must be used. It is included in the `Build Tools for Visual Studio <https://visualstudio.microsoft.com/downloads/>`__. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **OpenMP** | The `Build Tools for Visual Studio <https://visualstudio.microsoft.com/downloads/>`__ also include the `OpenMP <https://en.wikipedia.org/wiki/OpenMP>`__ library, which is utilized by the project for multi-theading support. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **OpenCL** | If you intend to compile the project with GPU support enabled, `OpenCL <https://www.khronos.org/opencl/>`__ must be installed manually. In order to do so, we recommend to install the package ``opencl`` via the package manager `vcpkg <https://github.com/microsoft/vcpkg>`__. | + +------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Additional build- or run-time dependencies will automatically be installed when following the instructions below and must not be installed manually. **Step 1: Creating a virtual environment** -The build process is based on creating a virtual Python environment that allows to install build-time dependencies in an isolated manner and independently from the host system. Once all packages have successfully been built, they are installed into the virtual environment. To create new virtual environment and install all necessarily build-time dependencies, the following command must be executed: +The build process is based on an virtual Python environment that allows to install build- and run-time dependencies in an isolated manner and independently from the host system. Once the build process was completed, the resulting Python packages are installed into the virtual environment. To create new virtual environment and install all necessarily run-time dependencies, the following command must be executed: + +.. tab:: Linux + + .. code-block:: text + + ./build venv + +.. tab:: MacOS + + .. code-block:: text -.. code-block:: text + ./build venv - make venv +.. tab:: Windows -All compile-time dependencies (`numpy`, `scipy`, `cython`, `meson`, `ninja`, etc.) that are required for building the project should automatically be installed into the virtual environment when executing the above command. As a result, a subdirectory `venv/` should have been created in the project's root directory. + .. code-block:: text + + build.bat venv + +All run-time dependencies (`numpy`, `scipy`, etc.) that are required for running the algorithms that are provided by the project should automatically be installed into the virtual environment when executing the above command. As a result, a subdirectory `venv/` should have been created in the project's root directory. **Step 2: Compiling the C++ code** Once a new virtual environment has successfully been created, the compilation of the C++ code can be started by executing the following command: -.. code-block:: text +.. tab:: Linux + + .. code-block:: text + + ./build compile_cpp + +.. tab:: MacOS - make compile_cpp + .. code-block:: text -Compilation is based on the build system `Meson <https://mesonbuild.com/>`_ and uses `Ninja <https://ninja-build.org/>`_ as a backend. After the above command has been completed, a new directory `cpp/build/` should have been created. It contains the shared libraries ("libmlrlcommon", "libmlrlboosting" and possibly others) that provide the basic functionality of the project's algorithms. + ./build compile_cpp + +.. tab:: Windows + + .. code-block:: text + + build.bat compile_cpp + +The compilation is based on the build system `Meson <https://mesonbuild.com/>`_ and uses `Ninja <https://ninja-build.org/>`_ as a backend. After the above command has terminated, a new directory `cpp/build/` should have been created. It contains the shared libraries ("libmlrlcommon", "libmlrlboosting" and possibly others) that provide the basic functionality of the project's algorithms. **Step 3: Compiling the Cython code** -Once the compilation of the C++ code has completed, the Cython code that allows to access the corresponding shared libraries from within Python can be compiled in the next step. Again, Meson and Ninja are used for compilation. It can be started via the following command: +Once the compilation of the C++ code has completed, the Cython code, which allows to access the corresponding shared libraries from within Python, can be compiled in the next step. Again, Meson and Ninja are used for compilation. It can be started via the following command: + +.. tab:: Linux -.. code-block:: text + .. code-block:: text - make compile_cython + ./build compile_cython + +.. tab:: MacOS + + .. code-block:: text + + ./build compile_cython + +.. tab:: Windows + + .. code-block:: text + + build.bat compile_cython As a result of executing the above command, the directory `python/build` should have been created. It contains Python extension modules for the respective target platform. .. note:: - Instead of performing the previous steps one after the other, the command ``make compile`` can be used to compile the C++ and Cython source files in a single step. + Instead of performing the previous steps one after the other, the build target ``compile`` can be specfied instead of ``compile_cpp`` and ``compile_cython`` to build the C++ and Cython source files in a single step. + +**Step 4: Copying shared libraries into the Python source tree** + +The shared libraries that have been created in the previous steps from the C++ source files must afterwards be copied into the Python source tree. This can be achieved by executing the following command: + +.. tab:: Linux + + .. code-block:: text + + ./build install_cpp + +.. tab:: MacOS + + .. code-block:: text + + ./build install_cpp + +.. tab:: Windows + + .. code-block:: text + + build.bat install_cpp + +This should result in the compilation files, which were previously located in the `cpp/build/` directory, to be copied into the `cython/` subdirectories that are contained by each Python module (e.g., into the directory `python/subprojects/common/mlrl/common/cython/`). + +**Step 5: Copying extension modules into the Python source tree** + +Similar to the previous step, the Python extension modules that have been built from the project's Cython code must be copied into the Python source tree via the following command: + +.. tab:: Linux -**Step 4: Copying compilation files into the Python source tree** + .. code-block:: text -The shared library files and Python extension modules that have been created in the previous steps must afterwards be copied into the source tree that contains the Python code. This can be achieved by executing the following commands: + ./build install_cython -.. code-block:: text +.. tab:: MacOS - make install_cpp - make install_cython + .. code-block:: text -This should result in the compilation files, which were previously located in the `cpp/build/` and `python/build/` directories, to be copied into the `cython/` subdirectories that are contained by each Python module (e.g., into the directory `python/subprojects/common/mlrl/common/cython/`). + ./build install_cython -**Step 5: Building wheel packages** +.. tab:: Windows + + .. code-block:: text + + build.bat install_cython + +As a result, the compilation files that can be found in the `python/build/` directories should have been copied into the `cython/` subdirectories of each Python module. + +.. note:: + Instead of executing the above commands one after the other, the build target ``install`` can be used instead of ``install_cpp`` and ``install_cython`` to copy both, the shared libraries and the extension modules, into the source tree. + +**Step 6: Building wheel packages** Once the compilation files have been copied into the Python source tree, wheel packages can be built for the individual Python modules via the following command: -.. code-block:: text +.. tab:: Linux + + .. code-block:: text + + ./build build_wheels + +.. tab:: MacOS + + .. code-block:: text + + ./build build_wheels - make wheel +.. tab:: Windows + + .. code-block:: text + + build.bat build_wheels This should result in .whl files being created in a new `dist/` subdirectory inside the directories that correspond to the individual Python modules (e.g., in the directory `python/subprojects/common/dist/`). -**Step 6: Installing the wheel packages into the virtual environment** +**Step 7: Installing the wheel packages into the virtual environment** + +The wheel packages that have previously been created can finally be installed into the virtual environment via the following command: + +.. tab:: Linux + + .. code-block:: text -The wheel packages that have previously been created, as well as its runtime-dependencies (e.g., `scikit-learn` or `liac-arff`), can finally be installed into the virtual environment via the following command: + ./build install_wheels -.. code-block:: text +.. tab:: MacOS - make install + .. code-block:: text -After this final step has completed, the Python packages can be used from within the virtual environment. To ensure that the installation of the wheel packages was successful, check if a `mlrl/` directory has been created in the `lib/` directory of the virtual environment (depending on the Python version, it should be located at `venv/lib/python3.9/site-packages/mlrl/` or similar). If this is the case, the algorithm can be used from within your own Python code. Alternatively, the command line API can be used to start an experiment (see :ref:`experiments`). + ./build install_wheels -.. warning:: - Whenever any C++, Cython or Python source files have been modified, they must be recompiled and updated wheel packages must be installed into the virtual environment by executing the command ``make install``. If any compilation files do already exist, this will only result in the affected parts of the code to be rebuilt. +.. tab:: Windows + + .. code-block:: text + + build.bat install_wheels + +After this final step has completed, the Python packages can be used from within the virtual environment once it has been `activated <https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#activating-a-virtual-environment>`__. To ensure that the installation of the wheel packages was successful, check if a `mlrl/` directory has been created in the `lib/` directory of the virtual environment (depending on the Python version, it should be located at `venv/lib/python3.9/site-packages/mlrl/` or similar). If this is the case, the algorithm can be used from within your own Python code. Alternatively, the command line API can be used to start an experiment (see :ref:`experiments`). + +.. note:: + Instead of following the above instructions step by step, the following command, which automatically executes all necessary steps, can be used for simplicity: + + .. tab:: Linux + + .. code-block:: text + + ./build + + .. tab:: MacOS + + .. code-block:: text + + ./build + + .. tab:: Windows + + .. code-block:: text + + build.bat + + Whenever any C++, Cython or Python source files have been modified, the above command must be run again in order to rebuild modified files and install updated wheel packages into the virtual environment. If any compilation files do already exist, this will only result in the affected parts of the code to be rebuilt. **Cleanup** -The Makefile allows to delete the files that result from the individual steps that have been described above. To delete the wheel packages that have been created via the command ``make wheel`` the following command can be used: +It is possible to delete the compilation files that result from an individual step of the build process mentioned above by using the command libe argument ``--clean`` or ``-c``. This may be useful if you want to repeat a single or multiple steps of the build process from scratch in case anything went wrong. For example, to delete the C++ compilation files, the following command can be used: -.. code-block:: text +.. tab:: Linux - make clean_wheel + .. code-block:: text -The following command allows to remove the shared library files and Python extension modules that have been copied into the Python source tree via the commands ``make install_cpp`` and ``make install_cython``: + ./build --clean compile_cpp -.. code-block:: text +.. tab:: MacOS - make clean_install + .. code-block:: text -The commands ``make clean_cython`` and ``make clean_cpp`` remove the Cython or C++ compilation files that have been created via the command ``make compile_cython`` or ``make compile_cpp`` from the respective `build/` directories. If you want to delete both, the Cython and C++ compilation files, the following command can be used: + ./build --clean compile_cpp -.. code-block:: text +.. tab:: Windows - make clean_compile + .. code-block:: text -.. note:: - If you want to delete all compilation files that have been created via the Makefile, including the virtual environment, you should use the command ``make clean``. + build.bat --clean compile_cpp + +If you want to delete all compilation files that have previously been created, including the virtual environment, you should use the following command, where no build target is specified: + +.. tab:: Linux + + .. code-block:: text + + ./build --clean + +.. tab:: MacOS + + .. code-block:: text + + ./build --clean + +.. tab:: Windows + + .. code-block:: text + + build.bat --clean diff --git a/doc/api/documentation.inc.rst b/doc/api/documentation.inc.rst index 3b4f8fb5ba..74f1084f89 100644 --- a/doc/api/documentation.inc.rst +++ b/doc/api/documentation.inc.rst @@ -3,16 +3,83 @@ Generating the Documentation ---------------------------- -In order to generate the documentation (this document), `Doxygen <https://sourceforge.net/projects/doxygen/>`_ must be installed on the host system beforehand. It is used to generate an API documentation from the C++ source files. By running the following command, the C++ API documentation is generated via Doxygen, the Python API documentation is created via `sphinx-apidoc <https://www.sphinx-doc.org/en/master/man/sphinx-apidoc.html>`_ and the documentation's HTML files are generated via `sphinx <https://www.sphinx-doc.org/en/master/>`_: +**Prerequisites** -.. code-block:: text +In order to generate the documentation (this document), `Doxygen <https://sourceforge.net/projects/doxygen/>`__ must be installed on the host system beforehand. It is used to generate an API documentation from the C++ source files. In addition, the `Roboto <https://fonts.google.com/specimen/Roboto>`__ font should be available on your system. If this is not the case, another font will be used as a fallback. - make doc +**Step 1: Generating the C++ API documentation** -Afterwards, the generated files can be found in the directory `doc/build_/html/`. +By running the following command, the C++ API documentation is generated via Doxygen: + +.. tab:: Linux + + .. code-block:: text + + ./build apidoc_cpp + +.. tab:: MacOS + + .. code-block:: text + + ./build apidoc_cpp + +.. tab:: Windows + + .. code-block:: text + + build.bat apidoc_cpp + +The resulting HTML files should be located in the directory `doc/apidoc/api/cpp/`. + +**Step 2: Generating the Python API documentation** + +Similarly, the following command generates an API documentation from the project's Python code via `sphinx-apidoc <https://www.sphinx-doc.org/en/master/man/sphinx-apidoc.html>`__: + +.. tab:: Linux + + .. code-block:: text + + ./build apidoc_python + +.. tab:: MacOS + + .. code-block:: text -To clean up the generated documentation files, the following command can be used: + ./build apidoc_python + +.. tab:: Windows + + .. code-block:: text + + build.bat apidoc_python + +.. note:: + If you want to generate the API documentation for the C++ and Python code simulatenously, it is possible to use the build target ``apidoc`` instead of ``apidoc_cpp`` and ``apidoc_python``. + +**Step 3: Generating the final documentation** + +To generate the final documentation's HTML files via `sphinx <https://www.sphinx-doc.org/en/master/>`__, the following command can be used: + +.. tab:: Linux + + .. code-block:: text + + ./build doc + +.. tab:: MacOS + + .. code-block:: text + + ./build doc + +.. tab:: Windows + + .. code-block:: text + + build.bat doc + +Afterwards, the generated files can be found in the directory `doc/build_/html/`. -.. code-block:: text +It should further be noted that it is not necessary to run the above steps one after the other. Executing a single command with the build target ``doc`` should suffice to create the entire documentation, including files that describe the C++ and Python API. - make clean_doc +Files that have been generated via the above steps can be removed by invoking the respective commands with the command line argument ``--clean``. A more detailed description can be found under :ref:`compilation`. diff --git a/doc/api/testing.inc.rst b/doc/api/testing.inc.rst index 76510dd8c8..9b6714d113 100644 --- a/doc/api/testing.inc.rst +++ b/doc/api/testing.inc.rst @@ -5,8 +5,23 @@ Testing the Code To be able to detect problems with the project's source code early during development, it comes with a large number of integration tests. Each of these tests runs a different configuration of the project's algorithms via the command line API and checks for unexpected results. If you want to execute the integrations tests on your own system, you can use the following command: -.. code-block:: text +.. tab:: Linux + + .. code-block:: text + + ./build tests + +.. tab:: MacOS + + .. code-block:: text + + ./build tests + +.. tab:: Windows + + .. code-block:: text + + build.bat tests - make tests The integration tests are also run automatically on a `CI server <https://en.wikipedia.org/wiki/Continuous_integration>`__ whenever relevant parts of the source code have been modified. For this purpose, we rely on the infrastructure provided by `Github Actions <https://docs.github.com/en/actions>`__. A track record of past test runs can be found in the `Github repository <https://github.com/mrapp-ke/Boomer/actions>`__.